betterdb-agent-cache 0.4.0__tar.gz

betterdb_agent_cache-0.4.0/.gitignore

@@ -0,0 +1,6 @@
+__pycache__/
+*.pyc
+*.pyo
+.venv/
+dist/
+*.egg-info/

betterdb_agent_cache-0.4.0/PKG-INFO

@@ -0,0 +1,34 @@
+Metadata-Version: 2.4
+Name: betterdb-agent-cache
+Version: 0.4.0
+Summary: Multi-tier exact-match cache for AI agent workloads backed by Valkey. LLM responses, tool results, and session state with built-in OpenTelemetry and Prometheus instrumentation.
+Project-URL: Repository, https://github.com/BetterDB-inc/monitor
+License: MIT
+Keywords: agent,cache,langchain,langgraph,llm,opentelemetry,prometheus,redis,valkey
+Requires-Python: >=3.11
+Requires-Dist: opentelemetry-api>=1.20.0
+Requires-Dist: prometheus-client>=0.19.0
+Requires-Dist: valkey>=6.0.0
+Provides-Extra: all
+Requires-Dist: anthropic>=0.20.0; extra == 'all'
+Requires-Dist: langchain-core>=0.1.0; extra == 'all'
+Requires-Dist: langgraph>=0.1.0; extra == 'all'
+Requires-Dist: llama-index-core>=0.10.0; extra == 'all'
+Requires-Dist: openai>=1.0.0; extra == 'all'
+Requires-Dist: posthog>=3.0.0; extra == 'all'
+Provides-Extra: analytics
+Requires-Dist: posthog>=3.0.0; extra == 'analytics'
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.20.0; extra == 'anthropic'
+Provides-Extra: dev
+Requires-Dist: fakeredis[aioredis]>=2.20.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.1.0; extra == 'langchain'
+Provides-Extra: langgraph
+Requires-Dist: langgraph>=0.1.0; extra == 'langgraph'
+Provides-Extra: llamaindex
+Requires-Dist: llama-index-core>=0.10.0; extra == 'llamaindex'
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == 'openai'

betterdb_agent_cache-0.4.0/RELEASE_NOTES.md

@@ -0,0 +1,103 @@
+# betterdb-agent-cache v0.3.0
+
+Python port of `@betterdb/agent-cache`. Multi-tier exact-match cache for AI agent
+workloads backed by Valkey — LLM responses, tool results, and session state, with
+built-in OpenTelemetry and Prometheus instrumentation.
+
+Runs on vanilla Valkey 7+. No modules, no RedisJSON, no RediSearch. Works on
+ElastiCache for Valkey, Memorystore for Valkey, and MemoryDB.
+
+---
+
+## Installation
+
+```sh
+pip install betterdb-agent-cache
+```
+
+Optional extras install the provider SDKs alongside the library:
+
+```sh
+pip install "betterdb-agent-cache[openai]"
+pip install "betterdb-agent-cache[anthropic]"
+pip install "betterdb-agent-cache[langchain]"
+pip install "betterdb-agent-cache[langgraph]"
+pip install "betterdb-agent-cache[llamaindex]"
+```
+
+---
+
+## What's included
+
+### Three cache tiers
+
+| Tier | Use for |
+|---|---|
+| `cache.llm` | LLM API responses — `check` / `store` / `store_multipart` / `invalidate_by_model` |
+| `cache.tool` | Tool call results — `check` / `store` / `set_policy` / `invalidate_by_tool` |
+| `cache.session` | Agent session state — `get` / `set` / `get_all` / `destroy_thread` / `touch` |
+
+### Provider adapters
+
+| Import | Provider |
+|---|---|
+| `betterdb_agent_cache.adapters.openai` | OpenAI Chat Completions |
+| `betterdb_agent_cache.adapters.openai_responses` | OpenAI Responses API |
+| `betterdb_agent_cache.adapters.anthropic` | Anthropic Messages |
+| `betterdb_agent_cache.adapters.llamaindex` | LlamaIndex |
+| `betterdb_agent_cache.adapters.langchain` | LangChain `BaseCache` |
+| `betterdb_agent_cache.adapters.langgraph` | LangGraph `BaseCheckpointSaver` |
+
+### Pluggable binary normalizer
+
+```python
+from betterdb_agent_cache import compose_normalizer, hash_base64
+
+normalizer = compose_normalizer({"base64": hash_base64})
+```
+
+### Observability
+
+- OpenTelemetry spans on every cache operation
+- Prometheus counters, histograms, and gauges (`requests_total`, `operation_duration_seconds`,
+  `cost_saved_total`, `stored_bytes_total`, `active_sessions`)
+
+---
+
+## Quick start
+
+```python
+import asyncio
+import valkey.asyncio as valkey_client
+from betterdb_agent_cache import AgentCache, ModelCost, TierDefaults
+from betterdb_agent_cache.adapters.openai import prepare_params
+from betterdb_agent_cache.types import AgentCacheOptions
+
+client = valkey_client.Valkey(host="localhost", port=6379)
+cache = AgentCache(AgentCacheOptions(
+    client=client,
+    tier_defaults={"llm": TierDefaults(ttl=3600)},
+    cost_table={"gpt-4o-mini": ModelCost(input_per_1k=0.00015, output_per_1k=0.0006)},
+))
+
+async def main():
+    params = await prepare_params({
+        "model": "gpt-4o-mini",
+        "messages": [{"role": "user", "content": "What is 2+2?"}],
+    })
+
+    result = await cache.llm.check(params)
+    if result.hit:
+        print("Cache hit:", result.response)
+    else:
+        # ... call OpenAI ...
+        await cache.llm.store(params, "Four")
+
+asyncio.run(main())
+```
+
+---
+
+## Full changelog
+
+See [CHANGELOG.md](./CHANGELOG.md) for detailed history.

betterdb_agent_cache-0.4.0/betterdb_agent_cache/__init__.py

@@ -0,0 +1,89 @@
+from .agent_cache import AgentCache
+from .default_cost_table import DEFAULT_COST_TABLE
+from .analytics import Analytics
+from .errors import AgentCacheError, AgentCacheUsageError, ValkeyCommandError
+from .normalizer import (
+    BinaryNormalizer,
+    BinaryRef,
+    NormalizerConfig,
+    compose_normalizer,
+    default_normalizer,
+    fetch_and_hash,
+    hash_base64,
+    hash_bytes,
+    hash_url,
+    passthrough,
+)
+from .types import (
+    AgentCacheOptions,
+    AgentCacheStats,
+    BinaryBlock,
+    BlockHints,
+    ContentBlock,
+    LlmCacheMessage,
+    LlmCacheParams,
+    LlmCacheResult,
+    LlmStoreOptions,
+    ModelCost,
+    ReasoningBlock,
+    SessionStats,
+    TextBlock,
+    TierDefaults,
+    TierStats,
+    ToolCallBlock,
+    ToolCacheResult,
+    ToolDefinition,
+    ToolEffectivenessEntry,
+    ToolPolicy,
+    ToolResultBlock,
+    ToolStats,
+    ToolStoreOptions,
+)
+
+__all__ = [
+    # Main class
+    "AgentCache",
+    "DEFAULT_COST_TABLE",
+    # Types
+    "AgentCacheOptions",
+    "AgentCacheStats",
+    "LlmCacheParams",
+    "LlmCacheMessage",
+    "LlmStoreOptions",
+    "LlmCacheResult",
+    "ToolStoreOptions",
+    "ToolPolicy",
+    "ToolCacheResult",
+    "ToolDefinition",
+    "ModelCost",
+    "TierDefaults",
+    "TierStats",
+    "SessionStats",
+    "ToolStats",
+    "ToolEffectivenessEntry",
+    # Content blocks
+    "ContentBlock",
+    "TextBlock",
+    "BinaryBlock",
+    "ToolCallBlock",
+    "ToolResultBlock",
+    "ReasoningBlock",
+    "BlockHints",
+    # Normalizer
+    "BinaryRef",
+    "BinaryNormalizer",
+    "NormalizerConfig",
+    "hash_base64",
+    "hash_bytes",
+    "hash_url",
+    "fetch_and_hash",
+    "passthrough",
+    "compose_normalizer",
+    "default_normalizer",
+    # Errors
+    "AgentCacheError",
+    "AgentCacheUsageError",
+    "ValkeyCommandError",
+    # Analytics
+    "Analytics",
+]

betterdb_agent_cache-0.4.0/betterdb_agent_cache/adapters/anthropic.py

@@ -0,0 +1,248 @@
+"""Anthropic Messages adapter.
+
+Converts params for ``anthropic.messages.create()`` into ``LlmCacheParams``.
+
+Usage::
+
+    from betterdb_agent_cache.adapters.anthropic import prepare_params
+
+    params = await prepare_params({"model": "claude-opus-4-5", "messages": [...], "max_tokens": 1024})
+    result = await cache.llm.check(params)
+"""
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+from dataclasses import dataclass, field
+from typing import Any
+
+from ..normalizer import BinaryNormalizer, default_normalizer
+from ..types import (
+    BinaryBlock,
+    BlockHints,
+    ContentBlock,
+    LlmCacheParams,
+    ReasoningBlock,
+    TextBlock,
+    ToolCallBlock,
+    ToolResultBlock,
+)
+
+
+@dataclass
+class AnthropicPrepareOptions:
+    normalizer: BinaryNormalizer = field(default_factory=lambda: default_normalizer)
+
+
+def _build_cache_hints(cache_control: dict[str, Any] | None) -> BlockHints | None:
+    if not cache_control:
+        return None
+    hint: BlockHints = {"anthropicCacheControl": {"type": "ephemeral"}}
+    if cache_control.get("ttl"):
+        hint["anthropicCacheControl"]["ttl"] = cache_control["ttl"]  # type: ignore[typeddict-item]
+    return hint
+
+
+async def _normalize_block(
+    block: dict[str, Any],
+    normalizer: BinaryNormalizer,
+) -> ContentBlock | None:
+    t = block.get("type")
+
+    if t == "text":
+        result: TextBlock = {"type": "text", "text": block["text"]}
+        hints = _build_cache_hints(block.get("cache_control"))
+        if hints:
+            result["hints"] = hints
+        return result
+
+    if t == "image":
+        src = block.get("source", {})
+        src_type = src.get("type")
+        media_type = "image/*"
+
+        if src_type == "base64":
+            source: dict[str, Any] = {"type": "base64", "data": src["data"]}
+            media_type = src.get("media_type", "image/*")
+        elif src_type == "url":
+            source = {"type": "url", "url": src["url"]}
+        elif src_type == "file":
+            source = {"type": "file_id", "file_id": src["file_id"], "provider": "anthropic"}
+        else:
+            return None
+
+        ref = await normalizer({"kind": "image", "source": source})
+        img_block: BinaryBlock = {"type": "binary", "kind": "image", "mediaType": media_type, "ref": ref}
+        hints = _build_cache_hints(block.get("cache_control"))
+        if hints:
+            img_block["hints"] = hints
+        return img_block
+
+    if t == "document":
+        src = block.get("source", {})
+        src_type = src.get("type")
+
+        if src_type == "content":
+            full_json = json.dumps(src.get("content"), sort_keys=True)
+            digest = hashlib.sha256(full_json.encode()).hexdigest()
+            ref = f"nested:sha256:{digest}"
+            doc_block: BinaryBlock = {
+                "type": "binary", "kind": "document",
+                "mediaType": "application/x-nested-content", "ref": ref,
+            }
+            hints = _build_cache_hints(block.get("cache_control"))
+            if hints:
+                doc_block["hints"] = hints
+            return doc_block
+
+        media_type = "application/octet-stream"
+        if src_type == "base64":
+            source = {"type": "base64", "data": src["data"]}
+            media_type = src.get("media_type", "application/pdf")
+        elif src_type == "text":
+            encoded = base64.b64encode(src["text"].encode()).decode()
+            source = {"type": "base64", "data": encoded}
+            media_type = "text/plain"
+        elif src_type == "url":
+            source = {"type": "url", "url": src["url"]}
+            media_type = "application/pdf"
+        elif src_type == "file":
+            source = {"type": "file_id", "file_id": src["file_id"], "provider": "anthropic"}
+        else:
+            return None
+
+        ref = await normalizer({"kind": "document", "source": source})
+        doc_b: BinaryBlock = {"type": "binary", "kind": "document", "mediaType": media_type, "ref": ref}
+        hints = _build_cache_hints(block.get("cache_control"))
+        if hints:
+            doc_b["hints"] = hints
+        return doc_b
+
+    if t == "tool_use":
+        return {
+            "type": "tool_call",
+            "id": block["id"],
+            "name": block["name"],
+            "args": block.get("input", {}),
+        }
+
+    if t == "thinking":
+        result_r: ReasoningBlock = {"type": "reasoning", "text": block["thinking"]}
+        if block.get("signature"):
+            result_r["opaqueSignature"] = block["signature"]
+        return result_r
+
+    if t == "redacted_thinking":
+        return {
+            "type": "reasoning",
+            "text": "",
+            "redacted": True,
+            "opaqueSignature": block.get("data", ""),
+        }
+
+    return None
+
+
+async def _normalize_tool_result_content(
+    content: str | list[dict[str, Any]],
+    normalizer: BinaryNormalizer,
+) -> list[TextBlock | BinaryBlock]:
+    if isinstance(content, str):
+        return [{"type": "text", "text": content}]
+    blocks: list[TextBlock | BinaryBlock] = []
+    for item in content:
+        if item.get("type") == "text":
+            blocks.append({"type": "text", "text": item["text"]})
+        elif item.get("type") == "image":
+            b = await _normalize_block(item, normalizer)
+            if b and b.get("type") == "binary":
+                blocks.append(b)  # type: ignore[arg-type]
+    return blocks
+
+
+async def prepare_params(
+    params: dict[str, Any],
+    opts: AnthropicPrepareOptions | None = None,
+) -> LlmCacheParams:
+    """Normalise Anthropic Messages params to ``LlmCacheParams``."""
+    normalizer = opts.normalizer if opts else default_normalizer
+    messages: list[Any] = []
+
+    # System message
+    system = params.get("system")
+    if system:
+        if isinstance(system, str):
+            messages.append({"role": "system", "content": system})
+        else:
+            sys_blocks: list[TextBlock] = []
+            for b in system:
+                tb: TextBlock = {"type": "text", "text": b["text"]}
+                hints = _build_cache_hints(b.get("cache_control"))
+                if hints:
+                    tb["hints"] = hints
+                sys_blocks.append(tb)
+            messages.append({"role": "system", "content": sys_blocks})
+
+    for msg in params.get("messages", []):
+        role = msg["role"]
+        content = msg["content"]
+
+        if role == "assistant":
+            if isinstance(content, str):
+                messages.append({
+                    "role": "assistant",
+                    "content": [{"type": "text", "text": content}],
+                })
+            else:
+                blocks: list[ContentBlock] = []
+                for blk in content:
+                    b = await _normalize_block(blk, normalizer)
+                    if b is not None:
+                        blocks.append(b)
+                messages.append({"role": "assistant", "content": blocks})
+
+        else:  # user
+            if isinstance(content, str):
+                messages.append({
+                    "role": "user",
+                    "content": [{"type": "text", "text": content}],
+                })
+                continue
+
+            tool_results = [p for p in content if p.get("type") == "tool_result"]
+            others = [p for p in content if p.get("type") != "tool_result"]
+
+            for tr in tool_results:
+                tr_content = await _normalize_tool_result_content(
+                    tr.get("content") or "", normalizer
+                )
+                messages.append({
+                    "role": "tool",
+                    "toolCallId": tr["tool_use_id"],
+                    "content": tr_content,
+                })
+
+            if others:
+                user_blocks: list[ContentBlock] = []
+                for blk in others:
+                    b = await _normalize_block(blk, normalizer)
+                    if b is not None:
+                        user_blocks.append(b)
+                messages.append({"role": "user", "content": user_blocks})
+
+    result: LlmCacheParams = {"model": params["model"], "messages": messages}
+    if params.get("temperature") is not None:
+        result["temperature"] = params["temperature"]
+    if params.get("top_p") is not None:
+        result["top_p"] = params["top_p"]
+    if params.get("max_tokens") is not None:
+        result["max_tokens"] = params["max_tokens"]
+    if params.get("tools") is not None:
+        result["tools"] = params["tools"]
+    if params.get("tool_choice") is not None:
+        result["tool_choice"] = params["tool_choice"]
+    if params.get("stop_sequences") is not None:
+        result["stop"] = params["stop_sequences"]
+
+    return result

betterdb_agent_cache-0.4.0/betterdb_agent_cache/adapters/langchain.py

@@ -0,0 +1,189 @@
+"""LangChain cache adapter.
+
+Implements LangChain's ``BaseCache`` interface backed by the AgentCache LLM tier.
+
+Usage::
+
+    from langchain_openai import ChatOpenAI
+    from betterdb_agent_cache.adapters.langchain import BetterDBLlmCache
+
+    lc_cache = BetterDBLlmCache(cache=agent_cache)
+    llm = ChatOpenAI(model="gpt-4o", cache=lc_cache)
+"""
+from __future__ import annotations
+
+import json
+import re
+from typing import TYPE_CHECKING, Any, Optional, Sequence
+
+try:
+    from langchain_core.caches import BaseCache
+    from langchain_core.messages import AIMessage
+    from langchain_core.outputs import ChatGeneration, Generation
+    _LANGCHAIN_AVAILABLE = True
+except ImportError:
+    _LANGCHAIN_AVAILABLE = False
+    BaseCache = object  # type: ignore[assignment,misc]
+    Generation = Any  # type: ignore[misc,assignment]
+    ChatGeneration = Any  # type: ignore[misc,assignment]
+
+if TYPE_CHECKING:
+    from ..agent_cache import AgentCache
+
+
+_KV_RE = re.compile(r'\b(\w+):"([^"]*)"')
+
+_NUMERIC_PARAMS = {"temperature", "top_p"}
+_INT_PARAMS = {"max_tokens"}
+
+
+def _parse_llm_params(llm_string: str) -> dict[str, Any]:
+    """Extract model and sampling parameters from LangChain's serialised llm_string.
+
+    Handles two formats:
+    - Comma-separated key:value pairs: ``model_name:"gpt-4o-mini",temperature:"0.7",...``
+    - LangChain JSON serialisation: ``{"kwargs": {"model_name": "gpt-4o-mini", ...}, ...}---[...]``
+
+    Returns a dict suitable for use as ``LlmCacheParams`` (without ``messages``).
+    Falls back to using the raw ``llm_string`` as the model name if extraction fails.
+    """
+    # Format 2: JSON serialisation with optional ---[...] suffix
+    try:
+        json_part = llm_string.split("---")[0].strip()
+        data = json.loads(json_part)
+        kwargs = data.get("kwargs", data)
+        params: dict[str, Any] = {}
+        model = kwargs.get("model_name") or kwargs.get("model")
+        if model:
+            params["model"] = str(model)
+        for field in ("temperature", "top_p", "max_tokens"):
+            if (v := kwargs.get(field)) is not None:
+                params[field] = v
+        if params.get("model"):
+            return params
+    except (json.JSONDecodeError, TypeError, AttributeError):
+        pass
+
+    # Format 1: comma-separated key:"value" pairs
+    params = {}
+    for m in _KV_RE.finditer(llm_string):
+        k, v = m.group(1), m.group(2)
+        if k in ("model_name", "model"):
+            params["model"] = v
+        elif k in _NUMERIC_PARAMS:
+            try:
+                params[k] = float(v)
+            except ValueError:
+                pass
+        elif k in _INT_PARAMS:
+            try:
+                params[k] = int(v)
+            except ValueError:
+                pass
+
+    if not params.get("model"):
+        params["model"] = llm_string
+    return params
+
+
+class BetterDBLlmCache(BaseCache):
+    """LangChain ``BaseCache`` implementation backed by AgentCache.
+
+    This cache is async-only. LangChain calls ``alookup`` / ``aupdate`` in
+    async pipelines. The synchronous ``lookup`` / ``update`` methods raise
+    ``RuntimeError`` — use an async LangChain invocation (``ainvoke`` /
+    ``astream``) to avoid hitting them.
+    """
+
+    def __init__(self, cache: "AgentCache") -> None:
+        if not _LANGCHAIN_AVAILABLE:
+            raise ImportError(
+                "langchain-core is required for BetterDBLlmCache. "
+                "Install it with: pip install betterdb-agent-cache[langchain]"
+            )
+        super().__init__()
+        self._cache = cache
+
+    # ── Async interface (primary) ──────────────────────────────────────────
+
+    async def alookup(
+        self, prompt: str, llm_string: str
+    ) -> Optional[list[Generation]]:
+        result = await self._cache.llm.check({
+            **_parse_llm_params(llm_string),
+            "messages": [{"role": "user", "content": prompt}],
+        })
+        if not result.hit or not result.response:
+            return None
+        try:
+            parsed: list[dict[str, Any]] = json.loads(result.response)
+            return [
+                ChatGeneration(text=g.get("text", ""), message=AIMessage(g.get("text", "")))
+                for g in parsed
+            ]
+        except (json.JSONDecodeError, TypeError):
+            text = result.response
+            return [ChatGeneration(text=text, message=AIMessage(text))]
+
+    async def aupdate(
+        self, prompt: str, llm_string: str, return_val: Sequence[Any]
+    ) -> None:
+        if not return_val:
+            return
+        stripped = [{"text": g.text if hasattr(g, "text") else g.get("text", "")} for g in return_val]
+        text = json.dumps(stripped)
+
+        # Extract token counts — try usage_metadata first, fall back to response_metadata
+        first = return_val[0] if return_val else None
+        tokens = None
+        if first is not None:
+            msg = getattr(first, "message", None) or (
+                first.get("message") if isinstance(first, dict) else None
+            )
+            if msg is not None:
+                # Newer LangChain: AIMessage.usage_metadata = {"input_tokens": N, "output_tokens": M}
+                usage = getattr(msg, "usage_metadata", None)
+                if isinstance(usage, dict) and usage:
+                    inp = usage.get("input_tokens")
+                    out = usage.get("output_tokens")
+                    if inp is not None and out is not None:
+                        tokens = {"input": inp, "output": out}
+
+                # Fallback: AIMessage.response_metadata["token_usage"] = {"prompt_tokens": N, "completion_tokens": M}
+                if tokens is None:
+                    rm = getattr(msg, "response_metadata", None) or {}
+                    tu = rm.get("token_usage") if isinstance(rm, dict) else None
+                    if isinstance(tu, dict) and tu:
+                        inp = tu.get("prompt_tokens")
+                        out = tu.get("completion_tokens")
+                        if inp is not None and out is not None:
+                            tokens = {"input": inp, "output": out}
+
+        from ..types import LlmStoreOptions
+        await self._cache.llm.store(
+            {**_parse_llm_params(llm_string), "messages": [{"role": "user", "content": prompt}]},
+            text,
+            LlmStoreOptions(tokens=tokens) if tokens else None,
+        )
+
+    # ── Sync interface (not supported) ────────────────────────────────────
+
+    def lookup(self, prompt: str, llm_string: str) -> Optional[list[Generation]]:
+        raise RuntimeError(
+            "BetterDBLlmCache is async-only. "
+            "Use an async LangChain invocation (ainvoke / astream)."
+        )
+
+    def update(self, prompt: str, llm_string: str, return_val: Sequence[Any]) -> None:
+        raise RuntimeError(
+            "BetterDBLlmCache is async-only. "
+            "Use an async LangChain invocation (ainvoke / astream)."
+        )
+
+    def clear(self, **kwargs: Any) -> None:
+        raise RuntimeError(
+            "BetterDBLlmCache is async-only. Use aclear() instead."
+        )
+
+    async def aclear(self, **kwargs: Any) -> None:
+        await self._cache.llm.clear()