Retrievers

import warnings; warnings.filterwarnings("ignore")
from standins import KeywordEmbeddings, RuleModel
from langchain_core.documents import Document
from langchain_core.vectorstores import InMemoryVectorStore
from langchain_community.vectorstores import FAISS
from langchain_community.retrievers import BM25Retriever
from langchain_classic.retrievers import EnsembleRetriever, MultiQueryRetriever, ParentDocumentRetriever
from langchain_classic.storage import InMemoryStore
from langchain_text_splitters import RecursiveCharacterTextSplitter
embeddings = KeywordEmbeddings()

texts = ["Report a lost card within 3 days for zero liability.",
         "Card-not-present payments above Rs 5,000 need OTP.",
         "Credit card annual fee is Rs 500, waived on spends above Rs 1 lakh.",
         "UPI limit for a new payee is Rs 25,000 in the first 24 hours.",
         "Never share your UPI PIN with anyone.",
         "Error code UPI-U30 means the debit failed at the bank.",
         "Video KYC is available from 9 AM to 9 PM.",
         "Home loan interest starts at 8.5% per year."]
docs = [Document(page_content=t, metadata={"id": i}) for i, t in enumerate(texts)]
show = lambda ds: [d.metadata["id"] for d in ds]

retriever = FAISS.from_documents(docs, embeddings).as_retriever(search_kwargs={"k": 2})
print("similarity   'my card is stolen' ->", show(retriever.invoke("my card is stolen")))
bm25 = BM25Retriever.from_documents(docs, k=2)            # default: splits on spaces only
print("BM25 default 'U30'               ->", show(bm25.invoke("U30")), " <- misses: the text has 'UPI-U30' as one word")
import re
bm25 = BM25Retriever.from_documents(docs, k=2, preprocess_func=lambda t: re.findall(r"\w+", t.lower()))
print("BM25 + regex 'U30'          ->", show(bm25.invoke("U30")), " <- found doc 5")
hybrid = EnsembleRetriever(retrievers=[bm25, retriever], weights=[0.5, 0.5])
print("hybrid       'what does U30 error mean for my UPI payment?' ->", show(hybrid.invoke("what does U30 error mean for my UPI payment?")))

llm = RuleModel(default="How much money can I transfer to a new payee?\nWhat is the UPI limit for a new beneficiary?\nIs there a limit on the first payment to a new person?")
multi = MultiQueryRetriever.from_llm(retriever=retriever, llm=llm)
print("multi-query  'how much can I send to someone new?' ->", show(multi.invoke("how much can I send to someone new?")))

parent = ParentDocumentRetriever(vectorstore=InMemoryVectorStore(embeddings), docstore=InMemoryStore(),
                                 child_splitter=RecursiveCharacterTextSplitter(chunk_size=60, chunk_overlap=0))
parent.add_documents([Document(" ".join(texts[:3]), metadata={"section": "cards"}), Document(" ".join(texts[3:6]), metadata={"section": "upi"})])
r = parent.invoke("annual fee")
print("parent-doc   'annual fee'        ->", r[0].metadata, f"({len(r[0].page_content)} chars: the whole section)")