cachellm-proxy 0.2.0__tar.gz → 0.2.1__tar.gz

{cachellm_proxy-0.2.0 → cachellm_proxy-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cachellm-proxy
-Version: 0.2.0
+Version: 0.2.1
 Summary: A drop-in semantic cache for OpenAI-compatible LLM APIs: match requests by meaning, serve repeats instantly, cut spend.
 Keywords: llm,cache,semantic-cache,openai,bedrock,proxy,vector-search,redis,fastapi,llmops,cost-optimization
 Author: Adarsh Dwivedi
@@ -53,7 +53,7 @@ Description-Content-Type: text/markdown
 [![CI](https://github.com/adarshcod30/CacheLLM/actions/workflows/ci.yml/badge.svg)](https://github.com/adarshcod30/CacheLLM/actions/workflows/ci.yml)
 [![Python 3.11+](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-blue)](https://www.python.org/)
 [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
-[![Tests](https://img.shields.io/badge/tests-261%20passing-brightgreen)](tests/)
+[![Tests](https://img.shields.io/badge/tests-265%20passing-brightgreen)](tests/)
 [![PyPI](https://img.shields.io/pypi/v/cachellm-proxy)](https://pypi.org/project/cachellm-proxy/)
 [![Hit rate](https://img.shields.io/badge/hit%20rate-77%25%20on%20Bedrock-orange)](docs/evaluation.md)
 
@@ -447,6 +447,7 @@ Clients can steer per request with `X-Cache-Control`:
 | `GET /admin/providers` | Every host, its base URL, its pip extra, example model ids |
 | `GET /admin/route/{model}` | Where one model name would go, and why |
 | `POST /admin/invalidate` | Drop by `namespace`, by `model`, or `all` |
+| `GET /admin/requests` | Recent request log: what the cache did with each one |
 | `GET /admin/near-misses` | Recent lookups that landed just below threshold |
 | `GET /admin/near-miss-histogram` | Cumulative view of what a lower threshold would buy |
 | `GET /admin/entries` | Inspect what is stored |

{cachellm_proxy-0.2.0 → cachellm_proxy-0.2.1}/README.md

@@ -3,7 +3,7 @@
 [![CI](https://github.com/adarshcod30/CacheLLM/actions/workflows/ci.yml/badge.svg)](https://github.com/adarshcod30/CacheLLM/actions/workflows/ci.yml)
 [![Python 3.11+](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-blue)](https://www.python.org/)
 [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
-[![Tests](https://img.shields.io/badge/tests-261%20passing-brightgreen)](tests/)
+[![Tests](https://img.shields.io/badge/tests-265%20passing-brightgreen)](tests/)
 [![PyPI](https://img.shields.io/pypi/v/cachellm-proxy)](https://pypi.org/project/cachellm-proxy/)
 [![Hit rate](https://img.shields.io/badge/hit%20rate-77%25%20on%20Bedrock-orange)](docs/evaluation.md)
 
@@ -397,6 +397,7 @@ Clients can steer per request with `X-Cache-Control`:
 | `GET /admin/providers` | Every host, its base URL, its pip extra, example model ids |
 | `GET /admin/route/{model}` | Where one model name would go, and why |
 | `POST /admin/invalidate` | Drop by `namespace`, by `model`, or `all` |
+| `GET /admin/requests` | Recent request log: what the cache did with each one |
 | `GET /admin/near-misses` | Recent lookups that landed just below threshold |
 | `GET /admin/near-miss-histogram` | Cumulative view of what a lower threshold would buy |
 | `GET /admin/entries` | Inspect what is stored |

{cachellm_proxy-0.2.0 → cachellm_proxy-0.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cachellm-proxy"
-version = "0.2.0"
+version = "0.2.1"
 description = "A drop-in semantic cache for OpenAI-compatible LLM APIs: match requests by meaning, serve repeats instantly, cut spend."
 readme = "README.md"
 requires-python = ">=3.11"

{cachellm_proxy-0.2.0 → cachellm_proxy-0.2.1}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "cachellm-proxy"
-version = "0.2.0"
+version = "0.2.1"
 description = "A drop-in semantic cache for OpenAI-compatible LLM APIs: match requests by meaning, serve repeats instantly, cut spend."
 readme = "README.md"
 authors = [

{cachellm_proxy-0.2.0 → cachellm_proxy-0.2.1}/src/cachellm/__init__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 
 __all__ = ["__version__", "main"]
 

{cachellm_proxy-0.2.0 → cachellm_proxy-0.2.1}/src/cachellm/api/routes_admin.py

@@ -212,6 +212,21 @@ async def invalidate(request: Request, body: InvalidateRequest) -> dict[str, Any
     }
 
 
+@router.get("/requests")
+async def requests_log(request: Request, limit: int = Query(50, ge=1, le=500)) -> dict[str, Any]:
+    """The recent request log: what the cache did with each one.
+
+    This is what `cachellm stats` reads. It has to come over HTTP because with
+    the in-memory backend the cache lives inside the serving process, and a CLI
+    building its own state would report on an empty cache of its own.
+    """
+    state = _require_cache(request)
+    if state.settings.require_auth_for_admin:
+        verify(request, state.settings.client_keys)
+    rows = await state.analytics.recent_requests(limit=limit)
+    return {"count": len(rows), "requests": rows}
+
+
 @router.get("/near-misses")
 async def near_misses(request: Request, limit: int = Query(50, ge=1, le=500)) -> dict[str, Any]:
     state = _require_cache(request)

{cachellm_proxy-0.2.0 → cachellm_proxy-0.2.1}/src/cachellm/cli.py

@@ -3,10 +3,10 @@
 from __future__ import annotations
 
 import asyncio
-import contextlib
 import json
+import time
 from pathlib import Path
-from typing import Annotated
+from typing import Annotated, Any
 
 import typer
 
@@ -57,75 +57,97 @@ def config() -> None:
     typer.echo(json.dumps(data, indent=2, default=str))
 
 
+def _proxy_url(url: str) -> str:
+    settings = get_settings()
+    return (url or f"http://127.0.0.1:{settings.port}").rstrip("/")
+
+
+def _fetch(url: str, path: str, params: dict[str, Any] | None = None) -> Any:
+    """Read from a running proxy, or None when nothing is listening.
+
+    Talking to the server rather than building our own state is not an
+    optimisation: with the in-memory backend the cache lives inside the serving
+    process, so a CLI that built its own would report on an empty cache of its
+    own and always say "no requests yet".
+    """
+    import httpx
+
+    settings = get_settings()
+    headers = {}
+    if keys := sorted(settings.client_keys):
+        headers["Authorization"] = f"Bearer {keys[0]}"
+    try:
+        response = httpx.get(f"{url}{path}", params=params, headers=headers, timeout=5.0)
+        response.raise_for_status()
+        return response.json()
+    except Exception:
+        return None
+
+
+def _not_running(url: str) -> None:
+    typer.secho(f"No proxy answering at {url}.", fg="yellow")
+    typer.echo("  Start one with `cachellm serve`, or pass --url if it is elsewhere.")
+    typer.echo(
+        f"  {report.DIM}The cache lives inside the serving process unless you "
+        f"use the redis backend, so there is nothing to read without it."
+        f"{report.RESET}"
+    )
+
+
 @app.command()
 def stats(
+    url: Annotated[str, typer.Option(help="Proxy to read from.")] = "",
     limit: Annotated[int, typer.Option(help="How many recent requests to show.")] = 15,
     json_out: Annotated[bool, typer.Option("--json", help="Raw JSON instead.")] = False,
 ) -> None:
     """Show what the cache has been doing: hit rate, savings, latency, recent requests."""
-
-    async def run() -> None:
-        from cachellm.api.deps import build_state, shutdown_state
-
-        settings = get_settings()
-        state = await build_state(settings)
-        try:
-            if state.cache is None or state.analytics is None:
-                typer.secho(f"cache unavailable: {state.degraded_reason}", fg="red")
-                raise typer.Exit(1)
-            data = await state.cache.stats()
-            data["backend"] = state.backend
-            data["cache_available"] = True
-            data["caching_enabled"] = settings.enabled
-            recent = await state.analytics.recent_requests(limit=limit)
-            if json_out:
-                typer.echo(json.dumps({"stats": data, "recent": recent}, indent=2))
-                return
-            typer.echo(report.summary(data, settings.embedding_model))
-            typer.echo(report.request_table(recent, limit=limit))
-        finally:
-            await shutdown_state(state)
-
-    asyncio.run(run())
+    base = _proxy_url(url)
+    data = _fetch(base, "/admin/stats")
+    if data is None:
+        _not_running(base)
+        raise typer.Exit(1)
+    log = _fetch(base, "/admin/requests", {"limit": limit}) or {}
+    recent = log.get("requests", [])
+    if json_out:
+        typer.echo(json.dumps({"stats": data, "recent": recent}, indent=2))
+        return
+    typer.echo(report.summary(data, get_settings().embedding_model))
+    typer.echo(report.request_table(recent, limit=limit))
 
 
 @app.command()
 def watch(
+    url: Annotated[str, typer.Option(help="Proxy to follow.")] = "",
     interval: Annotated[float, typer.Option(help="Seconds between refreshes.")] = 2.0,
 ) -> None:
     """Follow requests as they happen, like `tail -f` for the cache."""
-
-    async def run() -> None:
-        from cachellm.api.deps import build_state, shutdown_state
-
-        settings = get_settings()
-        state = await build_state(settings)
-        if state.cache is None or state.analytics is None:
-            typer.secho(f"cache unavailable: {state.degraded_reason}", fg="red")
-            raise typer.Exit(1)
-        seen: set[tuple[float, str]] = set()
-        typer.echo(f"watching {state.backend} backend, ctrl-c to stop\n")
-        try:
-            while True:
-                for record in reversed(await state.analytics.recent_requests(limit=50)):
-                    marker = (record.get("at", 0.0), record.get("prompt", ""))
-                    if marker in seen:
-                        continue
-                    seen.add(marker)
+    base = _proxy_url(url)
+    if _fetch(base, "/admin/stats") is None:
+        _not_running(base)
+        raise typer.Exit(1)
+
+    seen: set[tuple[float, str]] = set()
+    first = True
+    typer.echo(f"following {base}, ctrl-c to stop\n")
+    try:
+        while True:
+            payload = _fetch(base, "/admin/requests", {"limit": 50}) or {}
+            for record in reversed(payload.get("requests", [])):
+                marker = (record.get("at", 0.0), record.get("prompt", ""))
+                if marker in seen:
+                    continue
+                seen.add(marker)
+                if not first:
                     typer.echo(report.request_line(record))
-                if len(seen) > 5_000:
-                    seen.clear()
-                await asyncio.sleep(interval)
-        except (KeyboardInterrupt, asyncio.CancelledError):
-            data = await state.cache.stats()
-            data["backend"] = state.backend
-            data["cache_available"] = True
-            typer.echo(report.summary(data, settings.embedding_model))
-        finally:
-            await shutdown_state(state)
-
-    with contextlib.suppress(KeyboardInterrupt):
-        asyncio.run(run())
+            first = False
+            if len(seen) > 5_000:
+                seen.clear()
+            time.sleep(interval)
+    except KeyboardInterrupt:
+        data = _fetch(base, "/admin/stats")
+        if data:
+            typer.echo("")
+            typer.echo(report.summary(data, get_settings().embedding_model))
 
 
 @app.command()