cachellm-proxy 0.1.0__py3-none-any.whl

cachellm/__init__.py

@@ -0,0 +1,13 @@
+"""CacheLLM: a drop-in semantic cache for OpenAI-compatible LLM APIs."""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__", "main"]
+
+
+def main() -> None:  # pragma: no cover - thin console-script shim
+    from cachellm.cli import app
+
+    app()

cachellm/__main__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from cachellm.cli import app
+
+if __name__ == "__main__":  # pragma: no cover
+    app()

cachellm/api/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from cachellm.api.app import create_app
+
+__all__ = ["create_app"]

cachellm/api/app.py

@@ -0,0 +1,143 @@
+"""FastAPI application assembly."""
+
+from __future__ import annotations
+
+import contextlib
+from collections.abc import AsyncIterator
+from typing import Any
+
+import structlog
+from fastapi import FastAPI, Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse, Response
+
+from cachellm import __version__
+from cachellm.api import routes_admin, routes_chat
+from cachellm.api.deps import AppState, build_state, shutdown_state
+from cachellm.errors import CacheLLMError
+from cachellm.logging_setup import configure_logging
+from cachellm.observability.tracing import setup_tracing
+from cachellm.settings import Settings, get_settings
+
+log = structlog.get_logger(__name__)
+
+DESCRIPTION = """
+A drop-in semantic cache for OpenAI-compatible LLM APIs.
+
+Point your client's `base_url` at this service and repeated or reworded
+questions are served from cache instead of the model. Every response carries
+`X-Cache` headers explaining what happened.
+""".strip()
+
+
+def create_app(settings: Settings | None = None, state: AppState | None = None) -> FastAPI:
+    settings = settings or get_settings()
+    configure_logging(settings)
+
+    @contextlib.asynccontextmanager
+    async def lifespan(app: FastAPI) -> AsyncIterator[None]:
+        app.state.cachellm = state or await build_state(settings)
+        current: AppState = app.state.cachellm
+        if not current.settings.auth_enabled:
+            log.warning("auth_disabled", hint="set CACHELLM_API_KEYS before exposing this proxy")
+        if current.settings.shadow_mode:
+            log.warning("shadow_mode_on", hint="cache hits are logged, never served")
+        if not current.settings.is_calibrated and current.settings.threshold_default <= 0:
+            log.warning(
+                "threshold_not_calibrated",
+                embedding_model=current.settings.embedding_model,
+                using=current.settings.calibrated_threshold,
+                hint="measured safe thresholds span 0.89-0.98 across models; run "
+                "`python -m bench.compare_models` on your own data before trusting this",
+            )
+        log.info(
+            "cachellm_started",
+            version=__version__,
+            cache_available=current.cache_available,
+            embedding_model=current.embedder.name,
+            default_provider=current.settings.default_provider,
+        )
+        try:
+            yield
+        finally:
+            if state is None:
+                await shutdown_state(current)
+
+    app = FastAPI(
+        title="CacheLLM",
+        version=__version__,
+        description=DESCRIPTION,
+        lifespan=lifespan,
+        docs_url="/docs",
+        openapi_url="/openapi.json",
+    )
+
+    setup_tracing(settings, app)
+
+    app.include_router(routes_chat.router, prefix="/v1", tags=["openai"])
+    app.include_router(routes_admin.router, prefix="/admin", tags=["admin"])
+
+    # --------------------------------------------------------------- errors
+    @app.exception_handler(CacheLLMError)
+    async def _cachellm_error(_request: Request, exc: CacheLLMError) -> JSONResponse:
+        return JSONResponse(status_code=exc.status_code, content=exc.envelope())
+
+    @app.exception_handler(RequestValidationError)
+    async def _validation_error(_request: Request, exc: RequestValidationError) -> JSONResponse:
+        first = exc.errors()[0] if exc.errors() else {}
+        location = ".".join(str(p) for p in first.get("loc", ())[1:]) or None
+        return JSONResponse(
+            status_code=400,
+            content={
+                "error": {
+                    "message": first.get("msg", "Invalid request."),
+                    "type": "invalid_request_error",
+                    "param": location,
+                    "code": None,
+                }
+            },
+        )
+
+    # --------------------------------------------------------------- health
+    @app.get("/healthz", tags=["ops"])
+    async def healthz() -> dict[str, Any]:
+        return {"status": "ok", "version": __version__}
+
+    @app.get("/readyz", tags=["ops"])
+    async def readyz(request: Request) -> JSONResponse:
+        current: AppState = request.app.state.cachellm
+        ready = current.cache_available
+        body = {
+            "status": "ready" if ready else "degraded",
+            "cache_available": current.cache_available,
+            "degraded_reason": current.degraded_reason or None,
+            "embedding_model": current.embedder.name,
+        }
+        # Degraded still returns 200: the proxy is serving, just without cache.
+        return JSONResponse(body, status_code=200)
+
+    @app.get("/metrics", tags=["ops"])
+    async def metrics(request: Request) -> Response:
+        current: AppState = request.app.state.cachellm
+        if not current.settings.metrics_enabled:
+            return Response(status_code=404)
+        if current.vectors is not None:
+            with contextlib.suppress(Exception):
+                current.metrics.entries.set(await current.vectors.count())
+        body, content_type = current.metrics.render()
+        return Response(content=body, media_type=content_type)
+
+    @app.get("/", include_in_schema=False)
+    async def root() -> dict[str, Any]:
+        return {
+            "name": "CacheLLM",
+            "version": __version__,
+            "docs": "/docs",
+            "openai_base_url": "/v1",
+            "endpoints": ["/v1/chat/completions", "/v1/models", "/admin/stats", "/metrics"],
+        }
+
+    return app
+
+
+app = create_app  # uvicorn factory entry point

cachellm/api/auth.py

@@ -0,0 +1,34 @@
+"""Client authentication.
+
+The proxy holds provider credentials, so an unauthenticated proxy on a public
+address is an open wallet. Keys are compared in constant time and never logged.
+Auth is off only when no keys are configured, which is the local-dev case and
+is reported loudly at startup.
+"""
+
+from __future__ import annotations
+
+import hmac
+
+from fastapi import Request
+
+from cachellm.errors import AuthError
+
+
+def extract_key(request: Request) -> str:
+    header = request.headers.get("authorization", "")
+    if header.lower().startswith("bearer "):
+        return header[7:].strip()
+    return request.headers.get("api-key", "").strip()
+
+
+def verify(request: Request, keys: set[str]) -> None:
+    if not keys:
+        return
+    presented = extract_key(request)
+    if not presented:
+        raise AuthError("Missing API key. Pass it as `Authorization: Bearer <key>`.")
+    for known in keys:
+        if hmac.compare_digest(presented, known):
+            return
+    raise AuthError()

cachellm/api/deps.py

@@ -0,0 +1,106 @@
+"""Application state and its lifecycle.
+
+One deliberate design choice here: the cache **fails open**. If Redis is
+unreachable or the index cannot be built, the proxy keeps serving by forwarding
+every request upstream and reports itself degraded. A cache that takes an
+application down when it breaks is worse than no cache, and this is the first
+thing a reviewer looks for.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import structlog
+from fastapi import Request
+
+from cachellm.cache.analytics import Analytics
+from cachellm.cache.coalesce import SingleFlight
+from cachellm.cache.exact_store import ExactStore
+from cachellm.cache.redis_client import build_redis
+from cachellm.cache.service import CacheService
+from cachellm.cache.vector_store import VectorStore
+from cachellm.embeddings import build_embedder
+from cachellm.embeddings.base import Embedder
+from cachellm.observability.metrics import Metrics, get_metrics
+from cachellm.providers.registry import ProviderRegistry
+from cachellm.settings import Settings
+
+log = structlog.get_logger(__name__)
+
+
+@dataclass
+class AppState:
+    settings: Settings
+    metrics: Metrics
+    providers: ProviderRegistry
+    embedder: Embedder
+    singleflight: SingleFlight = field(default_factory=SingleFlight)
+    redis: Any = None
+    vectors: VectorStore | None = None
+    exact: ExactStore | None = None
+    analytics: Analytics | None = None
+    cache: CacheService | None = None
+    cache_available: bool = False
+    degraded_reason: str = ""
+
+    @property
+    def caching_on(self) -> bool:
+        return self.cache_available and self.cache is not None and self.settings.enabled
+
+
+async def build_state(
+    settings: Settings,
+    *,
+    embedder: Embedder | None = None,
+    providers: ProviderRegistry | None = None,
+) -> AppState:
+    state = AppState(
+        settings=settings,
+        metrics=get_metrics(),
+        providers=providers or ProviderRegistry(settings),
+        embedder=embedder or build_embedder(settings),
+    )
+    try:
+        redis = build_redis(settings)
+        await redis.ping()
+        vectors = VectorStore(redis, settings)
+        await vectors.connect()
+        exact = ExactStore(redis, settings)
+        analytics = Analytics(redis, settings)
+        state.redis = redis
+        state.vectors = vectors
+        state.exact = exact
+        state.analytics = analytics
+        state.cache = CacheService(
+            settings=settings,
+            embedder=state.embedder,
+            vectors=vectors,
+            exact=exact,
+            analytics=analytics,
+        )
+        state.cache_available = True
+    except Exception as exc:  # degrade, never crash the proxy
+        state.cache_available = False
+        state.degraded_reason = f"{type(exc).__name__}: {exc}"
+        log.error("cache_unavailable_failing_open", error=state.degraded_reason)
+
+    try:
+        await state.embedder.warmup()
+    except Exception as exc:  # embeddings are optional for pass-through
+        log.error("embedder_warmup_failed", error=str(exc)[:300])
+        state.cache_available = False
+        state.degraded_reason = f"embedder unavailable: {exc}"
+
+    return state
+
+
+async def shutdown_state(state: AppState) -> None:
+    await state.providers.close()
+    if state.redis is not None:
+        await state.redis.aclose()
+
+
+def get_state(request: Request) -> AppState:
+    return request.app.state.cachellm  # type: ignore[no-any-return]

cachellm/api/routes_admin.py

@@ -0,0 +1,265 @@
+"""Operator endpoints: stats, invalidation, near-miss analysis, threshold tuning.
+
+These are what turn the cache from a black box into something a team is willing
+to leave switched on. Every one of them answers a question an operator actually
+asks: what is it doing, what would a different threshold do, and how do I make
+it forget something right now.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import structlog
+from fastapi import APIRouter, Body, Query, Request
+from pydantic import BaseModel, Field
+
+from cachellm.api.auth import verify
+from cachellm.api.deps import get_state
+from cachellm.errors import CacheLLMError
+
+log = structlog.get_logger(__name__)
+router = APIRouter()
+
+DEFAULT_SWEEP = [0.80, 0.84, 0.86, 0.88, 0.90, 0.92, 0.94, 0.96, 0.98, 0.99]
+
+
+class InvalidateRequest(BaseModel):
+    namespace: str | None = None
+    model: str | None = None
+    all: bool = False
+
+
+class LabelledPair(BaseModel):
+    a: str
+    b: str
+    duplicate: bool
+
+
+class SweepRequest(BaseModel):
+    pairs: list[LabelledPair] = Field(min_length=1)
+    thresholds: list[float] | None = None
+
+
+def _require_cache(request: Request) -> Any:
+    state = get_state(request)
+    if not state.cache_available or state.cache is None:
+        raise CacheLLMError(
+            503,
+            f"Cache backend unavailable: {state.degraded_reason or 'not connected'}",
+            "api_error",
+            code="cache_unavailable",
+        )
+    return state
+
+
+@router.get("/stats")
+async def stats(request: Request) -> dict[str, Any]:
+    state = get_state(request)
+    if state.settings.require_auth_for_admin:
+        verify(request, state.settings.client_keys)
+    if not state.cache_available or state.cache is None:
+        return {
+            "cache_available": False,
+            "caching_enabled": state.settings.enabled,
+            "degraded_reason": state.degraded_reason or "backend not connected",
+        }
+    data = await state.cache.stats()
+    data["cache_available"] = True
+    data["caching_enabled"] = state.settings.enabled
+    data["shadow_mode"] = state.settings.shadow_mode
+    data["in_flight"] = state.singleflight.in_flight
+    state.metrics.entries.set(data.get("entries", 0))
+    return data
+
+
+@router.get("/config")
+async def config(request: Request) -> dict[str, Any]:
+    state = get_state(request)
+    if state.settings.require_auth_for_admin:
+        verify(request, state.settings.client_keys)
+    s = state.settings
+    return {
+        "enabled": s.enabled,
+        "shadow_mode": s.shadow_mode,
+        "embedding": {
+            "backend": s.embedding_backend,
+            "model": s.embedding_model,
+            "dim": s.expected_dim(),
+        },
+        "thresholds": {
+            c: s.threshold_for(c)
+            for c in (
+                "factual",
+                "classification",
+                "creative",
+                "volatile",
+                "conversational",
+                "default",
+            )
+        },
+        "ttl_seconds": {
+            c: s.ttl_for(c)
+            for c in (
+                "factual",
+                "classification",
+                "creative",
+                "volatile",
+                "conversational",
+                "default",
+            )
+        },
+        "rules": {
+            "max_cacheable_temperature": s.max_cacheable_temperature,
+            "cache_multi_turn": s.cache_multi_turn,
+            "cache_json_mode": s.cache_json_mode,
+            "cache_tool_calls": s.cache_tool_calls,
+            "pii_guard": s.pii_guard,
+            "strip_filler_words": s.strip_filler_words,
+            "max_prompt_chars": s.max_prompt_chars,
+        },
+        "default_provider": s.default_provider,
+        "auth_enabled": s.auth_enabled,
+    }
+
+
+@router.post("/invalidate")
+async def invalidate(request: Request, body: InvalidateRequest) -> dict[str, Any]:
+    state = _require_cache(request)
+    if state.settings.require_auth_for_admin:
+        verify(request, state.settings.client_keys)
+    if not (body.namespace or body.model or body.all):
+        raise CacheLLMError(400, "Pass one of `namespace`, `model` or `all`.", param="namespace")
+    removed = await state.cache.invalidate(
+        namespace=body.namespace, model=body.model, drop_all=body.all
+    )
+    return {
+        "removed_keys": removed,
+        "namespace": body.namespace,
+        "model": body.model,
+        "all": body.all,
+    }
+
+
+@router.get("/near-misses")
+async def near_misses(request: Request, limit: int = Query(50, ge=1, le=500)) -> dict[str, Any]:
+    state = _require_cache(request)
+    if state.settings.require_auth_for_admin:
+        verify(request, state.settings.client_keys)
+    rows = await state.analytics.near_misses(limit=limit)
+    return {"count": len(rows), "near_misses": rows}
+
+
+@router.get("/near-miss-histogram")
+async def near_miss_histogram(
+    request: Request, buckets: int = Query(20, ge=5, le=100)
+) -> dict[str, Any]:
+    """Cumulative view: how many more requests each lower threshold would serve."""
+    state = _require_cache(request)
+    if state.settings.require_auth_for_admin:
+        verify(request, state.settings.client_keys)
+    return {"histogram": await state.analytics.near_miss_histogram(buckets=buckets)}
+
+
+@router.get("/entries")
+async def entries(request: Request, limit: int = Query(20, ge=1, le=200)) -> dict[str, Any]:
+    state = _require_cache(request)
+    if state.settings.require_auth_for_admin:
+        verify(request, state.settings.client_keys)
+    keys = await state.vectors.scan_keys()
+    rows = []
+    for key in keys[:limit]:
+        entry_id = key.split(state.settings.entry_prefix, 1)[-1]
+        entry = await state.vectors.get(entry_id)
+        if entry is None:
+            continue
+        summary = entry.summary()
+        if not state.settings.log_prompts:
+            summary["prompt"] = summary["prompt"][:120]
+            summary["response_text"] = summary["response_text"][:160]
+        rows.append(summary)
+    return {"total_keys": len(keys), "returned": len(rows), "entries": rows}
+
+
+@router.post("/reset-stats")
+async def reset_stats(request: Request) -> dict[str, str]:
+    state = _require_cache(request)
+    if state.settings.require_auth_for_admin:
+        verify(request, state.settings.client_keys)
+    await state.analytics.reset()
+    return {"status": "counters reset"}
+
+
+@router.post("/threshold-sweep")
+async def threshold_sweep(request: Request, body: SweepRequest = Body(...)) -> dict[str, Any]:
+    """Score labelled prompt pairs at several thresholds.
+
+    This is the answer to the only question that really matters when tuning a
+    semantic cache: at threshold X, how many extra requests do I serve, and how
+    many of those are actually different questions wearing similar words?
+
+    ``precision`` is the share of would-be cache hits that were genuine
+    duplicates. ``false_positive_rate`` is the share of *non*-duplicate pairs
+    that would wrongly hit. Read them together: hit rate you can only spend if
+    precision holds.
+    """
+    state = get_state(request)
+    if state.settings.require_auth_for_admin:
+        verify(request, state.settings.client_keys)
+
+    thresholds = sorted(body.thresholds or DEFAULT_SWEEP)
+    texts_a = [p.a for p in body.pairs]
+    texts_b = [p.b for p in body.pairs]
+    labels = np.array([p.duplicate for p in body.pairs], dtype=bool)
+
+    vectors_a = await state.embedder.embed_batch(texts_a)
+    vectors_b = await state.embedder.embed_batch(texts_b)
+    sims = np.array(
+        [float(np.dot(a, b)) for a, b in zip(vectors_a, vectors_b, strict=True)],
+        dtype=np.float32,
+    )
+
+    positives = int(labels.sum())
+    negatives = int((~labels).sum())
+    rows: list[dict[str, Any]] = []
+    for threshold in thresholds:
+        predicted = sims >= threshold
+        tp = int((predicted & labels).sum())
+        fp = int((predicted & ~labels).sum())
+        fn = int((~predicted & labels).sum())
+        precision = tp / (tp + fp) if (tp + fp) else 1.0
+        recall = tp / positives if positives else 0.0
+        f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0
+        rows.append(
+            {
+                "threshold": round(threshold, 4),
+                "would_hit": tp + fp,
+                "hit_rate": round((tp + fp) / len(body.pairs), 4),
+                "true_hits": tp,
+                "false_hits": fp,
+                "missed_duplicates": fn,
+                "precision": round(precision, 4),
+                "recall": round(recall, 4),
+                "f1": round(f1, 4),
+                "false_positive_rate": round(fp / negatives, 4) if negatives else 0.0,
+            }
+        )
+
+    best = max(rows, key=lambda r: r["f1"]) if rows else None
+    safe = [r for r in rows if r["false_positive_rate"] <= 0.01]
+    return {
+        "pairs": len(body.pairs),
+        "duplicates": positives,
+        "non_duplicates": negatives,
+        "embedding_model": state.embedder.name,
+        "similarity": {
+            "duplicates_mean": round(float(sims[labels].mean()), 4) if positives else None,
+            "non_duplicates_mean": round(float(sims[~labels].mean()), 4) if negatives else None,
+        },
+        "sweep": rows,
+        "best_f1": best,
+        "lowest_threshold_under_1pct_false_positives": (
+            min(safe, key=lambda r: r["threshold"]) if safe else None
+        ),
+    }