Case 1 — Build a Production-Ready RAG Chatbot

# Case 1 — Build a Production-Ready RAG Chatbot > *"Build a chatbot that answers questions from a corpus of 500 support documents. Production-ready: hybrid search, reranking, citations, 'I don't know' fallback, evaluation."* --- ## Step 1 — Architecture overview ``` User question ↓ [Embed query] ↓ [Hybrid search (dense + BM25)] → top 20 candidates ↓ [Cross-encoder rerank] → top 5 ↓ [LLM with retrieved context + citations + "I don't know" fallback] ↓ Answer with citations ``` 5 components. Let's build each. ## Step 2 — Document ingestion ```python from langchain.text_splitter import RecursiveCharacterTextSplitter def ingest_documents(docs): splitter = RecursiveCharacterTextSplitter( chunk_size=500, chunk_overlap=50, separators=["\n\n", "\n", ". ", " ", ""], ) chunks = [] for doc in docs: for chunk_text in splitter.split_text(doc.text): chunks.append({ "text": chunk_text, "metadata": { "source_id": doc.id, "source_title": doc.title, "source_url": doc.url, "ingested_at": datetime.utcnow().isoformat(), } }) return chunks # Embed and index from openai import OpenAI client = OpenAI() def embed_chunks(chunks): texts = [c["text"] for c in chunks] response = client.embeddings.create( model="text-embedding-3-large", input=texts, ) for chunk, embedding_data in zip(chunks, response.data): chunk["embedding"] = embedding_data.embedding return chunks # Store in Qdrant from qdrant_client import QdrantClient from qdrant_client.models import VectorParams, Distance, PointStruct qdrant = QdrantClient(host="localhost", port=6333) qdrant.recreate_collection( collection_name="support_docs", vectors_config=VectorParams(size=3072, distance=Distance.COSINE), ) def index_chunks(chunks): points = [ PointStruct( id=i, vector=chunk["embedding"], payload={"text": chunk["text"], **chunk["metadata"]} ) for i, chunk in enumerate(chunks) ] qdrant.upsert(collection_name="support_docs", points=points) ``` ## Step 3 — Hybrid search ```python from rank_bm25 import BM25Okapi import re # Build BM25 index tokenized_corpus = [re.findall(r"\w+", c["text"].lower()) for c in chunks] bm25 = BM25Okapi(tokenized_corpus) def hybrid_search(query, top_k=20): # Dense query_emb = client.embeddings.create( model="text-embedding-3-large", input=[query] ).data[0].embedding dense_results = qdrant.search( collection_name="support_docs", query_vector=query_emb, limit=top_k * 2, ) # Sparse (BM25) tokens = re.findall(r"\w+", query.lower()) bm25_scores = bm25.get_scores(tokens) sparse_top_idx = bm25_scores.argsort()[::-1][:top_k * 2] # Reciprocal rank fusion rrf_scores = {} k = 60 # standard RRF constant for rank, hit in enumerate(dense_results): rrf_scores[hit.id] = rrf_scores.get(hit.id, 0) + 1 / (k + rank + 1) for rank, idx in enumerate(sparse_top_idx): rrf_scores[int(idx)] = rrf_scores.get(int(idx), 0) + 1 / (k + rank + 1) # Top by combined score top_ids = sorted(rrf_scores.keys(), key=lambda i: -rrf_scores[i])[:top_k] return [chunks[i] for i in top_ids] ``` ## Step 4 — Reranking ```python import cohere co = cohere.Client(api_key=os.environ["COHERE_API_KEY"]) def rerank(query, candidates, top_k=5): docs = [c["text"] for c in candidates] response = co.rerank( model="rerank-english-v3.0", query=query, documents=docs, top_n=top_k, ) return [candidates[r.index] for r in response.results] ``` ## Step 5 — Generation with citations ```python import anthropic claude = anthropic.Anthropic() def answer_with_rag(user_question): candidates = hybrid_search(user_question, top_k=20) top_chunks = rerank(user_question, candidates, top_k=5) context_block = "\n\n".join([ f"[Source {i+1}: {c['source_title']}]\n{c['text']}" for i, c in enumerate(top_chunks) ]) response = claude.messages.create( model="claude-sonnet-4-6", system=[{ "type": "text", "text": '''You are a support assistant. Answer the user's question using ONLY the provided context. Rules: - Cite sources for each claim using [Source X] format. - If the context doesn't contain the answer, say "I don't have information about that in our docs. Could you rephrase or contact support?" - Keep responses under 200 words. - Don't make up information not in the context.''', "cache_control": {"type": "ephemeral"} }], messages=[{ "role": "user", "content": f"Context:\n{context_block}\n\nQuestion: {user_question}" }], max_tokens=400, ) return {