Callbacks, Caching, Rate Limits & LangSmith

os.environ["LANGSMITH_TRACING"] = "true"
os.environ["LANGSMITH_API_KEY"] = userdata.get("langsmith_key")     # free at smith.langchain.com
os.environ["LANGSMITH_PROJECT"] = "langchain-handson"

chain.invoke({"topic": "vector database"},
             config={"run_name": "explain-topic", "tags": ["demo"], "metadata": {"user": "ajeet"}})
# open smith.langchain.com → every step with inputs, outputs, tokens, latency and errors

import time, asyncio
from standins import RuleModel
from langchain_core.callbacks import BaseCallbackHandler
from langchain_core.caches import InMemoryCache
from langchain_core.globals import set_llm_cache
from langchain_core.rate_limiters import InMemoryRateLimiter

# a stand-in with 0.8 s of fake network delay, so timings look like a real API
llm = RuleModel(default="India has 22 official languages listed in the Constitution.", delay=0.8)

class MyLogger(BaseCallbackHandler):
    def on_chat_model_start(self, serialized, messages, **kwargs):
        self.start = time.time(); print("🚀 LLM started")
    def on_llm_end(self, response, **kwargs):
        usage = response.generations[0][0].message.usage_metadata
        print(f"✅ LLM finished in {time.time() - self.start:.1f}s | tokens: {usage}")
llm.invoke("Tell me a fun fact about India", config={"callbacks": [MyLogger()]})

class PrintTokens(BaseCallbackHandler):
    def on_llm_new_token(self, token, **kwargs): print(token, end="|")
print("tokens: ", end="")
for _ in RuleModel(default="Snow peaks touch the sky.").stream("Himalayas", config={"callbacks": [PrintTokens()]}): pass
print()

set_llm_cache(InMemoryCache())
for i in (1, 2):
    t = time.time(); llm.invoke("What is the capital of Japan?")
    print(f"call {i}: {time.time() - t:.1f} s" + ("  (from cache, no API cost)" if i == 2 else ""))
set_llm_cache(None)

limited = RuleModel(default="hi", rate_limiter=InMemoryRateLimiter(requests_per_second=0.5, check_every_n_seconds=0.1, max_bucket_size=1))
start = time.time()
for i in range(3):
    limited.invoke(f"hi {i}"); print(f"rate-limited call {i + 1} finished at {time.time() - start:.0f} s")

questions = [f"Give one fact about number {i}" for i in range(1, 6)]
t = time.time(); [llm.invoke(q) for q in questions]; seq = time.time() - t
async def main(): return await asyncio.gather(*[llm.ainvoke(q) for q in questions])
t = time.time(); asyncio.run(main()); par = time.time() - t
t = time.time(); llm.batch(questions, config={"max_concurrency": 2}); b2 = time.time() - t
print(f"5 questions: one by one {seq:.0f} s | async gather {par:.0f} s | batch(max_concurrency=2) {b2:.0f} s")