cachellm-proxy 0.1.0__py3-none-any.whl

cachellm/cache/entry.py

@@ -0,0 +1,92 @@
+"""The record stored in Redis for one cached answer."""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+import numpy as np
+
+
+@dataclass
+class CacheEntry:
+    entry_id: str
+    namespace: str
+    category: str
+    model: str
+    provider: str
+    prompt: str
+    response_text: str
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    finish_reason: str = "stop"
+    created_at: float = field(default_factory=time.time)
+    ttl: int = 86_400
+    hits: int = 0
+    exact_hash: str = ""
+    embedding: np.ndarray | None = None
+    tool_calls: list[dict[str, Any]] | None = None
+
+    # ------------------------------------------------------------- serialisation
+    def to_redis_mapping(self) -> dict[str, str | bytes | int | float]:
+        """Flatten to a Redis hash. The vector goes in as raw float32 bytes."""
+        mapping: dict[str, str | bytes | int | float] = {
+            "entry_id": self.entry_id,
+            "namespace": self.namespace,
+            "category": self.category,
+            "model": self.model,
+            "provider": self.provider,
+            "prompt": self.prompt,
+            "response_text": self.response_text,
+            "prompt_tokens": int(self.prompt_tokens),
+            "completion_tokens": int(self.completion_tokens),
+            "finish_reason": self.finish_reason,
+            "created_at": float(self.created_at),
+            "ttl": int(self.ttl),
+            "hits": int(self.hits),
+            "exact_hash": self.exact_hash,
+            "tool_calls": json.dumps(self.tool_calls) if self.tool_calls else "",
+        }
+        if self.embedding is not None:
+            mapping["embedding"] = np.asarray(self.embedding, dtype=np.float32).tobytes()
+        return mapping
+
+    @staticmethod
+    def _s(raw: dict[Any, Any], key: str, default: str = "") -> str:
+        value = raw.get(key, raw.get(key.encode(), default))
+        if isinstance(value, bytes):
+            return value.decode("utf-8", "replace")
+        return str(value) if value is not None else default
+
+    @classmethod
+    def from_redis_mapping(cls, raw: dict[Any, Any]) -> CacheEntry:
+        get = cls._s
+        tool_calls_raw = get(raw, "tool_calls")
+        return cls(
+            entry_id=get(raw, "entry_id"),
+            namespace=get(raw, "namespace"),
+            category=get(raw, "category", "default"),
+            model=get(raw, "model"),
+            provider=get(raw, "provider"),
+            prompt=get(raw, "prompt"),
+            response_text=get(raw, "response_text"),
+            prompt_tokens=int(float(get(raw, "prompt_tokens", "0") or 0)),
+            completion_tokens=int(float(get(raw, "completion_tokens", "0") or 0)),
+            finish_reason=get(raw, "finish_reason", "stop"),
+            created_at=float(get(raw, "created_at", "0") or 0),
+            ttl=int(float(get(raw, "ttl", "0") or 0)),
+            hits=int(float(get(raw, "hits", "0") or 0)),
+            exact_hash=get(raw, "exact_hash"),
+            tool_calls=json.loads(tool_calls_raw) if tool_calls_raw else None,
+        )
+
+    def age_seconds(self) -> float:
+        return max(0.0, time.time() - self.created_at)
+
+    def summary(self) -> dict[str, Any]:
+        data = asdict(self)
+        data.pop("embedding", None)
+        data["age_seconds"] = round(self.age_seconds(), 1)
+        return data

cachellm/cache/exact_store.py

@@ -0,0 +1,33 @@
+"""Level 1: exact-match lookup.
+
+A literal repeat should never pay for an embedding. This tier is a plain
+``GET`` that resolves a normalised-prompt hash to an entry id, so it answers in
+about a millisecond and takes the pressure off the vector index.
+"""
+
+from __future__ import annotations
+
+import redis.asyncio as aioredis
+
+from cachellm.settings import Settings
+
+
+class ExactStore:
+    def __init__(self, redis: aioredis.Redis, settings: Settings) -> None:
+        self._redis = redis
+        self._prefix = settings.exact_prefix
+
+    def key(self, namespace: str, exact_hash: str) -> str:
+        return f"{self._prefix}{namespace}:{exact_hash}"
+
+    async def get(self, namespace: str, exact_hash: str) -> str | None:
+        raw = await self._redis.get(self.key(namespace, exact_hash))
+        if raw is None:
+            return None
+        return raw.decode("utf-8") if isinstance(raw, bytes) else str(raw)
+
+    async def put(self, namespace: str, exact_hash: str, entry_id: str, ttl: int) -> None:
+        await self._redis.set(self.key(namespace, exact_hash), entry_id, ex=ttl)
+
+    async def delete(self, namespace: str, exact_hash: str) -> None:
+        await self._redis.delete(self.key(namespace, exact_hash))

cachellm/cache/keys.py

@@ -0,0 +1,124 @@
+"""Cache key derivation.
+
+Two ideas do the heavy lifting here.
+
+**Namespace.** Everything that changes what a *correct* answer looks like goes
+into a namespace hash: provider, model, system prompt, temperature bucket,
+max_tokens, response format. Two identical user questions asked under different
+system prompts must never share an answer, and a model upgrade must not serve
+stale text from the old model. Making that a tag on the index means
+invalidation is one query, not a scan.
+
+**Two levels.** The exact hash catches literal repeats in about a millisecond
+without touching the embedding model. Only genuine near misses pay for a vector
+search. In replayed traffic a large share of repeats are literal, so this tier
+does real work.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import unicodedata
+
+from cachellm.models import ChatCompletionRequest
+
+_WS = re.compile(r"\s+")
+_TRAILING_PUNCT = re.compile(r"[\s\.\?!,;:]+$")
+
+# Words that carry no meaning for retrieval. Stripping them raises hit rate by
+# collapsing polite phrasing onto the same point in vector space. Off by default
+# because it also erases some genuine distinctions.
+FILLER_WORDS = frozenset(
+    {
+        "please",
+        "could",
+        "would",
+        "can",
+        "you",
+        "kindly",
+        "hey",
+        "hi",
+        "hello",
+        "thanks",
+        "thank",
+        "sorry",
+        "just",
+        "actually",
+        "basically",
+        "really",
+        "simply",
+        "quick",
+        "quickly",
+        "tell",
+        "me",
+        "explain",
+        "about",
+        "i",
+        "want",
+        "to",
+        "know",
+        "wondering",
+        "help",
+    }
+)
+
+
+def normalise_text(text: str) -> str:
+    """Unicode-normalise, lowercase, collapse whitespace, drop trailing punctuation."""
+    out = unicodedata.normalize("NFKC", text or "").strip().lower()
+    out = _WS.sub(" ", out)
+    return _TRAILING_PUNCT.sub("", out)
+
+
+_WORD = re.compile(r"[\w']+", re.UNICODE)
+
+
+def strip_filler(text: str) -> str:
+    """Drop filler words, punctuation included, keeping the meaningful tokens."""
+    tokens = [t for t in _WORD.findall(normalise_text(text)) if t not in FILLER_WORDS]
+    return " ".join(tokens) if tokens else normalise_text(text)
+
+
+def embedding_text(raw: str, *, strip_fillers: bool) -> str:
+    return strip_filler(raw) if strip_fillers else normalise_text(raw)
+
+
+def temperature_bucket(temperature: float) -> str:
+    """Bucket to one decimal place.
+
+    0.0 and 0.05 produce effectively the same answer distribution and should
+    share a cache entry; 0.2 and 0.9 should not.
+    """
+    return f"{round(float(temperature), 1):.1f}"
+
+
+def namespace_for(request: ChatCompletionRequest, provider: str) -> str:
+    """Stable 16-hex-char id for "requests that may share answers"."""
+    response_format = request.response_format or {}
+    payload = {
+        "provider": provider,
+        "model": request.model.strip(),
+        "system": normalise_text(request.system_text()),
+        "temperature": temperature_bucket(request.effective_temperature),
+        "top_p": request.top_p,
+        "max_tokens": request.effective_max_tokens,
+        "response_format": response_format.get("type"),
+        "stop": request.stop,
+    }
+    blob = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
+    return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:16]
+
+
+def exact_hash(text: str) -> str:
+    return hashlib.sha256(normalise_text(text).encode("utf-8")).hexdigest()[:32]
+
+
+def entry_id(namespace: str, exact: str) -> str:
+    return f"{namespace}-{exact[:20]}"
+
+
+def prompt_fingerprint(text: str) -> str:
+    """Short, non-reversible id for a prompt, safe to put in logs and metrics."""
+    return hashlib.blake2b(normalise_text(text).encode("utf-8"), digest_size=6).hexdigest()

cachellm/cache/policy.py

@@ -0,0 +1,134 @@
+"""Decides whether a request may be cached, in which category, for how long.
+
+This module is the honesty layer. A semantic cache that caches everything will
+eventually serve yesterday's stock price or one user's order status to another
+user. Each rule below exists because of a specific way that goes wrong.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from cachellm.models import ChatCompletionRequest
+from cachellm.settings import Settings
+
+# Anything that pins an answer to *now*. These still get cached, but with a
+# short TTL, because "what is the weather in Jaipur" repeats a lot within an hour.
+VOLATILE_PATTERNS = re.compile(
+    r"\b(today|tonight|right now|currently|current|latest|breaking|this (week|month|year|morning)|"
+    r"yesterday|tomorrow|as of|so far|up to date|live|news|stock price|share price|weather|"
+    r"forecast|who won|score|trending|recent|nowadays|these days)\b",
+    re.IGNORECASE,
+)
+
+# Open-ended generation: two different answers are both correct, so a cache hit
+# that returns the same poem twice is a product bug, not a saving.
+CREATIVE_PATTERNS = re.compile(
+    r"\b(write|compose|draft|generate|create|invent|imagine|brainstorm|come up with)\b.{0,40}"
+    r"\b(poem|story|song|lyrics|joke|essay|article|blog|caption|tagline|slogan|name|names|ideas?|"
+    r"script|email|letter|post)\b",
+    re.IGNORECASE,
+)
+
+# Constrained answer space, so near-duplicates are safe at a lower threshold.
+CLASSIFICATION_PATTERNS = re.compile(
+    r"\b(classify|categorise|categorize|label|sentiment|is this|does this|yes or no|true or false|"
+    r"which category|tag this|detect|extract|rate this|score this)\b",
+    re.IGNORECASE,
+)
+
+# Personal data. If any of this is present the answer is about one person and
+# must never be served to another, so we do not store it at all.
+PII_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
+    ("email", re.compile(r"\b[\w.+-]+@[\w-]+\.[\w.]{2,}\b")),
+    ("long_digits", re.compile(r"\b\d{9,}\b")),  # phone, Aadhaar, account, order id
+    ("card", re.compile(r"\b(?:\d[ -]*?){13,16}\b")),
+    ("api_key", re.compile(r"\b(sk|pk|ghp|gho|AKIA|ASIA)[-_A-Za-z0-9]{12,}\b")),
+    (
+        "possessive",
+        re.compile(
+            r"\bmy (order|account|booking|policy|invoice|ticket|card|salary|"
+            r"password|address|phone|otp|pan|aadhaar|ssn)\b",
+            re.IGNORECASE,
+        ),
+    ),
+)
+
+
+@dataclass(frozen=True)
+class PolicyDecision:
+    cacheable: bool
+    category: str
+    threshold: float
+    ttl: int
+    reason: str = ""
+
+    @property
+    def bypass_reason(self) -> str:
+        return "" if self.cacheable else self.reason
+
+
+def _classify(text: str, request: ChatCompletionRequest) -> str:
+    if VOLATILE_PATTERNS.search(text):
+        return "volatile"
+    if CREATIVE_PATTERNS.search(text):
+        return "creative"
+    max_tokens = request.effective_max_tokens
+    if CLASSIFICATION_PATTERNS.search(text) or (max_tokens is not None and max_tokens <= 32):
+        return "classification"
+    if request.user_turn_count() > 1:
+        return "conversational"
+    return "factual"
+
+
+def detect_pii(text: str) -> str | None:
+    for label, pattern in PII_PATTERNS:
+        if pattern.search(text):
+            return label
+    return None
+
+
+def decide(
+    request: ChatCompletionRequest,
+    settings: Settings,
+    *,
+    cache_control: str = "",
+) -> PolicyDecision:
+    """Return the caching decision for one request."""
+    text = request.last_user_text()
+    category = _classify(text, request)
+    threshold = settings.threshold_for(category)
+    ttl = settings.ttl_for(category)
+
+    def no(reason: str) -> PolicyDecision:
+        return PolicyDecision(False, category, threshold, ttl, reason)
+
+    control = (cache_control or "").lower()
+    if "no-store" in control or "no-cache" in control:
+        return no("client_requested_bypass")
+    if not settings.enabled:
+        return no("cache_disabled")
+    if not text:
+        return no("no_user_message")
+    if len(text) > settings.max_prompt_chars:
+        return no("prompt_too_long")
+    if request.effective_temperature > settings.max_cacheable_temperature:
+        return no("temperature_too_high")
+    if (request.n or 1) > 1:
+        return no("multiple_completions_requested")
+    if request.tools and not settings.cache_tool_calls:
+        return no("tool_calls")
+    if any(m.has_non_text_parts() for m in request.messages):
+        return no("non_text_content")
+    fmt = (request.response_format or {}).get("type")
+    if fmt in ("json_object", "json_schema") and not settings.cache_json_mode:
+        return no("json_mode")
+    if request.user_turn_count() > 1 and not settings.cache_multi_turn:
+        return no("multi_turn_conversation")
+    if settings.pii_guard:
+        found = detect_pii(text)
+        if found:
+            return no(f"pii:{found}")
+
+    return PolicyDecision(True, category, threshold, ttl, "")

cachellm/cache/redis_client.py

@@ -0,0 +1,22 @@
+"""Redis connection factory.
+
+``decode_responses`` is deliberately False. Embeddings are stored as raw
+float32 bytes and a decoding client would corrupt them on the way back out;
+text fields are decoded explicitly where they are read.
+"""
+
+from __future__ import annotations
+
+import redis.asyncio as aioredis
+
+from cachellm.settings import Settings
+
+
+def build_redis(settings: Settings) -> aioredis.Redis:
+    return aioredis.from_url(
+        settings.redis_url,
+        decode_responses=False,
+        health_check_interval=30,
+        socket_connect_timeout=5,
+        socket_keepalive=True,
+    )