cachellm-proxy 0.1.0__tar.gz → 0.2.0__tar.gz

{cachellm_proxy-0.1.0 → cachellm_proxy-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cachellm-proxy
-Version: 0.1.0
+Version: 0.2.0
 Summary: A drop-in semantic cache for OpenAI-compatible LLM APIs: match requests by meaning, serve repeats instantly, cut spend.
 Keywords: llm,cache,semantic-cache,openai,bedrock,proxy,vector-search,redis,fastapi,llmops,cost-optimization
 Author: Adarsh Dwivedi
@@ -14,19 +14,17 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Typing :: Typed
-Requires-Dist: boto3>=1.43.91
 Requires-Dist: fastapi>=0.141.1
+Requires-Dist: uvicorn[standard]>=0.52.4
 Requires-Dist: fastembed>=0.8.0
-Requires-Dist: httpx>=0.28.1
 Requires-Dist: numpy>=2.4.6
-Requires-Dist: openai>=3.11.0
-Requires-Dist: prometheus-client>=0.26.0
 Requires-Dist: pydantic-settings>=2.15.0
-Requires-Dist: redis>=8.1.0
-Requires-Dist: redisvl>=0.27.1
+Requires-Dist: prometheus-client>=0.26.0
 Requires-Dist: structlog>=26.1.0
 Requires-Dist: typer>=0.27.2
-Requires-Dist: uvicorn[standard]>=0.52.4
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: cachellm-proxy[aws,redis,observability,datasets] ; extra == 'all'
+Requires-Dist: boto3>=1.43.91 ; extra == 'aws'
 Requires-Dist: botocore[crt]>=1.43.91 ; extra == 'aws'
 Requires-Dist: datasets>=5.0.1 ; extra == 'datasets'
 Requires-Dist: pandas>=3.0.5 ; extra == 'datasets'
@@ -37,13 +35,17 @@ Requires-Dist: opentelemetry-instrumentation-fastapi>=0.65b0 ; extra == 'observa
 Requires-Dist: opentelemetry-instrumentation-httpx>=0.65b0 ; extra == 'observability'
 Requires-Dist: opentelemetry-instrumentation-redis>=0.65b0 ; extra == 'observability'
 Requires-Dist: opentelemetry-sdk>=1.44.0 ; extra == 'observability'
+Requires-Dist: redis>=8.1.0 ; extra == 'redis'
+Requires-Dist: redisvl>=0.27.1 ; extra == 'redis'
 Requires-Python: >=3.11
 Project-URL: Homepage, https://github.com/adarshcod30/CacheLLM
 Project-URL: Repository, https://github.com/adarshcod30/CacheLLM
 Project-URL: Issues, https://github.com/adarshcod30/CacheLLM/issues
+Provides-Extra: all
 Provides-Extra: aws
 Provides-Extra: datasets
 Provides-Extra: observability
+Provides-Extra: redis
 Description-Content-Type: text/markdown
 
 # CacheLLM
@@ -51,7 +53,7 @@ Description-Content-Type: text/markdown
 [![CI](https://github.com/adarshcod30/CacheLLM/actions/workflows/ci.yml/badge.svg)](https://github.com/adarshcod30/CacheLLM/actions/workflows/ci.yml)
 [![Python 3.11+](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-blue)](https://www.python.org/)
 [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
-[![Tests](https://img.shields.io/badge/tests-128%20passing-brightgreen)](tests/)
+[![Tests](https://img.shields.io/badge/tests-261%20passing-brightgreen)](tests/)
 [![PyPI](https://img.shields.io/pypi/v/cachellm-proxy)](https://pypi.org/project/cachellm-proxy/)
 [![Hit rate](https://img.shields.io/badge/hit%20rate-77%25%20on%20Bedrock-orange)](docs/evaluation.md)
 
@@ -111,6 +113,7 @@ The same benchmark runs without any cloud credentials against the built-in fake
 | Feature | What it does | Why it exists |
 | --- | --- | --- |
 | **Drop-in OpenAI API** | Same request and response shape, streaming included, verified against the official `openai` Python SDK in CI | Adoption has to cost one line, or nobody adopts it |
+| **Runs with nothing installed** | Defaults to an in-process numpy store; uses Redis automatically when it can reach one | A cache you have to provision a server for does not get tried. Redis takes over when you actually need shared, durable state |
 | **Two-tier cache** | Exact-match tier answers literal repeats in about a millisecond without embedding; semantic tier handles rewording | 82% of hits came from the exact tier: free, fast and impossible to get semantically wrong |
 | **Per-model threshold calibration** | Ships measured safe thresholds for six embedding models and picks the right one automatically | Measured safe thresholds span 0.89 to 0.98. A threshold copied between models is a guess |
 | **Cacheability policy** | Classifies every prompt and decides cacheable, category, TTL | Creative writing, live data and personal questions must not be cached like a fact |
@@ -134,7 +137,8 @@ The same benchmark runs without any cloud credentials against the built-in fake
 | API | FastAPI + Uvicorn | Async, native streaming, OpenAPI docs for free |
 | Validation and config | Pydantic v2, pydantic-settings | Typed request shapes and typed configuration from the environment |
 | Embeddings | fastembed, `all-MiniLM-L6-v2` (ONNX, CPU) | In-process and about 6 ms. A hosted embedding API would put 100 ms in front of every cache hit and defeat the point |
-| Vector store | Redis 8 + RedisVL, HNSW over cosine | One process is the exact-match store, the vector index and the stampede lock. Sub-millisecond, and TTL expiry removes entries from the index for free |
+| Vector store, default | numpy matrix in the proxy's own memory | Nothing to install. Scanning 20,000 cached prompts takes 0.85 ms, where a Redis round trip alone costs 2 to 3 ms, so below roughly 100k entries this is not a compromise, it is faster |
+| Vector store, at scale | Redis 8 + RedisVL, HNSW over cosine | One process is the exact-match store, the vector index and the stampede lock. Shared across workers, survives restarts, and an approximate index starts paying above ~100k entries |
 | Providers | AWS Bedrock (Converse), any OpenAI-compatible endpoint, deterministic fake | Converse reaches Nova, Claude, Llama and Mistral with one request shape and no extra vendor keys |
 | Metrics | prometheus-client, Prometheus, Grafana | Dashboard ships provisioned, so a reviewer sees data on first boot |
 | Tracing | OpenTelemetry, optional, GenAI semantic conventions | Instrument once, export to Langfuse, Tempo or Jaeger |
@@ -270,32 +274,47 @@ MiniLM gives four times the safe recall of bge-small at a third of the download
 
 ## Quick start
 
-You need Python 3.11 or newer and a Redis 8 instance. Redis 8 is required because the vector index needs the query engine, and it must be database 0 because Redis Search only indexes that one.
+Python 3.11 or newer, and nothing else.
 
 ```bash
 pip install cachellm-proxy
+cachellm serve
 ```
 
-The distribution is `cachellm-proxy` because PyPI blocks `cachellm` as too close to an existing `cachelm`. The import name and the CLI are both still `cachellm`.
+That is the whole install. No Redis, no Docker, no config file. On startup it looks for a provider you already have: an API key in your environment, Ollama running locally, or AWS credentials. It logs which one it picked and how to override it.
 
-Or from source, which is what you want if you plan to change anything:
+To see what it found before starting anything:
 
 ```bash
-git clone https://github.com/adarshcod30/CacheLLM.git
-cd CacheLLM
-uv sync
+cachellm providers
 ```
 
-Start Redis if you do not have one running:
+To try it with no account at all, the built-in test double stands in for a model:
 
 ```bash
-docker run -d --name redis -p 6379:6379 redis:8-alpine
+CACHELLM_DEFAULT_PROVIDER=fake CACHELLM_FAKE_LATENCY_MS=600 cachellm serve
 ```
 
-Run the proxy. With no configuration it uses the built-in fake provider, so you can see it working before wiring up a real model or spending anything:
+You only install what you actually route to. The base package is the proxy plus the local embedding model; AWS and Redis are extras, and nothing here installs Grafana or a Prometheus server, which are separate programs rather than Python packages.
+
+| You want | Install |
+| --- | --- |
+| Any OpenAI-compatible endpoint: OpenAI, Groq, Gemini, Ollama, vLLM | `pip install cachellm-proxy` |
+| AWS Bedrock | `pip install "cachellm-proxy[aws]"` |
+| Redis instead of the in-process cache | `pip install "cachellm-proxy[redis]"` |
+| Tracing to Langfuse or Tempo | `pip install "cachellm-proxy[observability]"` |
+| Everything | `pip install "cachellm-proxy[all]"` |
+
+Ask for a provider or backend whose extra is missing and the proxy names the exact command to fix it rather than raising an import error.
+
+The distribution is `cachellm-proxy` because PyPI blocks `cachellm` as too close to an existing `cachelm`. The import name and the CLI are both still `cachellm`.
+
+From source, if you plan to change anything:
 
 ```bash
-CACHELLM_DEFAULT_PROVIDER=fake CACHELLM_FAKE_LATENCY_MS=600 uv run cachellm serve
+git clone https://github.com/adarshcod30/CacheLLM.git
+cd CacheLLM
+uv sync && uv run cachellm serve
 ```
 
 In another terminal, ask the same thing twice and watch the second one come back instantly:
@@ -342,28 +361,53 @@ make tune && make compare-models && make bench
 
 ### Using a real provider
 
-AWS Bedrock needs no extra vendor keys and reaches Nova, Claude, Llama and Mistral through one API:
+One environment variable per host. Everything except Bedrock speaks the OpenAI protocol, so a single adapter covers all of it and needs no extra.
+
+| Host | `CACHELLM_OPENAI_BASE_URL` | Example model |
+| --- | --- | --- |
+| OpenAI | `https://api.openai.com/v1` | `gpt-4o-mini` |
+| Groq | `https://api.groq.com/openai/v1` | `llama-3.3-70b-versatile` |
+| Google Gemini | `https://generativelanguage.googleapis.com/v1beta/openai/` | `gemini-2.5-flash` |
+| Anthropic | `https://api.anthropic.com/v1` | `claude-haiku-4-5` |
+| OpenRouter | `https://openrouter.ai/api/v1` | `anthropic/claude-3.5-sonnet` |
+| Together | `https://api.together.xyz/v1` | `meta-llama/Llama-3.3-70B-Instruct-Turbo` |
+| Ollama, local | `http://localhost:11434/v1` | `llama3.2` |
+| vLLM or LM Studio | `http://localhost:8000/v1` | whatever you served |
 
 ```bash
-CACHELLM_DEFAULT_PROVIDER=bedrock AWS_REGION=us-east-1 uv run cachellm serve
+CACHELLM_OPENAI_BASE_URL=https://api.groq.com/openai/v1 \
+CACHELLM_OPENAI_API_KEY=gsk_your_key \
+cachellm serve
 ```
 
-If your credentials come from `aws login` rather than static keys or an SSO profile, install the CRT extra once, because that credential provider needs it:
+**AWS Bedrock** is the one exception, because it does not speak the OpenAI protocol. It needs the `aws` extra, and then uses your existing AWS credentials with no vendor API key at all:
 
 ```bash
-uv sync --extra aws
+pip install "cachellm-proxy[aws]"
+CACHELLM_DEFAULT_PROVIDER=bedrock AWS_REGION=us-east-1 cachellm serve
 ```
 
-The proxy detects that case and says so in the error rather than passing along boto's version of the message.
-
 ```bash
 curl -s http://localhost:8080/v1/chat/completions -H 'Content-Type: application/json' -d '{"model":"bedrock/us.amazon.nova-micro-v1:0","temperature":0,"messages":[{"role":"user","content":"What is Redis used for?"}]}'
 ```
 
-Any OpenAI-compatible endpoint works too, which covers OpenAI, Groq, Together, OpenRouter and a local vLLM:
+If your credentials come from `aws login` rather than static keys or an SSO profile, that provider needs the CRT extra, which the `aws` extra already includes. The proxy detects that case and says so in the error rather than passing along boto's version of the message.
+
+### How a model name gets routed
+
+Three rules, in order. You never configure a model list.
+
+1. **An explicit prefix** this proxy owns: `bedrock/…`, `openai/…`, `fake/…`.
+2. **A recognisable vendor convention.** Bedrock ids are always `vendor.model`, so `amazon.nova-lite-v1:0` and `us.anthropic.claude-3-haiku-20240307-v1:0` are identified with no prefix. OpenAI's own families (`gpt-`, `o1`, `o3`, `text-embedding-`) are identified the same way, whatever the default is.
+3. **Otherwise the configured default**, which is the OpenAI-compatible adapter. Names like `llama3.2`, `mixtral-8x7b-32768` and `qwen2.5-coder:7b` are served by Groq, Ollama, Together and OpenRouter alike, so the endpoint you configured is the only sensible answer.
+
+Model ids that legitimately contain a slash, which OpenRouter and Together both use, are forwarded whole. Only this proxy's own prefixes are stripped. Note that `anthropic.claude-…` with a dot is a Bedrock id while `anthropic/claude-…` with a slash is an OpenRouter id, and the router tells them apart.
+
+Ask it directly if you are unsure:
 
 ```bash
-CACHELLM_DEFAULT_PROVIDER=openai CACHELLM_OPENAI_BASE_URL=https://api.groq.com/openai/v1 CACHELLM_OPENAI_API_KEY=... uv run cachellm serve
+curl -s localhost:8080/admin/route/meta-llama/Llama-3.3-70B-Instruct-Turbo
+curl -s localhost:8080/admin/providers
 ```
 
 ## API reference
@@ -400,6 +444,8 @@ Clients can steer per request with `X-Cache-Control`:
 | --- | --- |
 | `GET /admin/stats` | Hit rate, tier split, money saved, latency percentiles, entry count |
 | `GET /admin/config` | Effective thresholds, TTLs and rules |
+| `GET /admin/providers` | Every host, its base URL, its pip extra, example model ids |
+| `GET /admin/route/{model}` | Where one model name would go, and why |
 | `POST /admin/invalidate` | Drop by `namespace`, by `model`, or `all` |
 | `GET /admin/near-misses` | Recent lookups that landed just below threshold |
 | `GET /admin/near-miss-histogram` | Cumulative view of what a lower threshold would buy |
@@ -418,20 +464,50 @@ curl -s http://localhost:8080/admin/threshold-sweep -H 'Content-Type: applicatio
 ### Command line
 
 ```bash
-uv run cachellm serve          # run the proxy
-uv run cachellm stats          # cache statistics
-uv run cachellm config         # effective configuration
-uv run cachellm invalidate --all
-uv run cachellm tune pairs.jsonl
+cachellm serve                 # run the proxy, auto-detecting a provider
+cachellm providers             # every host, and what this machine can reach
+cachellm stats                 # hit rate, savings, latency, recent requests
+cachellm watch                 # follow requests live, like tail -f
+cachellm config                # effective configuration
+cachellm invalidate --all
+cachellm tune pairs.jsonl
 ```
 
+`cachellm stats` is the dashboard, in your terminal:
+
+```
+  CacheLLM · memory backend · all-MiniLM-L6-v2
+
+  HIT RATE    77.0%  ████████████████░░░░░░  1,540 of 2,000 requests
+             1,265 exact · 275 semantic · 460 missed · 0 bypassed
+
+  SAVED      $0.0135   spent $0.0038  78% lower
+             116,029 tokens never generated
+
+  LATENCY    cached       2.6 ms p50 ·      5.7 ms p95
+             uncached   797.0 ms p50 ·   1022.8 ms p95   179x faster
+
+  CACHE      451 entries · 9 coalesced · 17 near misses
+
+  time      result tier        latency  score  saved       prompt
+  02:30:55  HIT    exact         1.2ms  1.000  +$0.000010  What is Redis used for?
+  02:30:54  BYPASS             451.5ms    ·          ·     What is my order 1234…  (pii:long_digits)
+  02:30:53  HIT    semantic      7.4ms  0.952  +$0.000011  What is Redis typically used for?
+  02:30:53  MISS               454.8ms    ·          ·     How do I configure nginx for TLS?
+```
+
+Prometheus and Grafana are still there if you want history and alerting, but nothing needs them.
+
 ## Configuration
 
 Every setting is an environment variable prefixed `CACHELLM_`, or a line in `.env`. Copy `.env.example` to start. The ones that matter most:
 
 | Variable | Default | Notes |
 | --- | --- | --- |
+| `CACHELLM_BACKEND` | `auto` | `memory` needs nothing, `redis` shares one cache across workers, `auto` uses Redis when reachable and memory when not |
 | `CACHELLM_REDIS_URL` | `redis://localhost:6379/0` | Must be database 0. Redis Search cannot index any other |
+| `CACHELLM_MEMORY_MAX_ENTRIES` | `50000` | Cap for the in-memory store. 50k of 384-dim vectors is about 73 MB |
+| `CACHELLM_MEMORY_SNAPSHOT_PATH` | unset | Persist the in-memory cache to this file so a restart does not start cold |
 | `CACHELLM_API_KEYS` | empty | Comma-separated client keys. Empty disables auth, which is local development only |
 | `CACHELLM_EMBEDDING_MODEL` | `all-MiniLM-L6-v2` | Changing this changes the safe threshold. See the calibration table |
 | `CACHELLM_THRESHOLD_*` | `0` | Zero means use the calibrated value for your model |

{cachellm_proxy-0.1.0 → cachellm_proxy-0.2.0}/README.md

@@ -3,7 +3,7 @@
 [![CI](https://github.com/adarshcod30/CacheLLM/actions/workflows/ci.yml/badge.svg)](https://github.com/adarshcod30/CacheLLM/actions/workflows/ci.yml)
 [![Python 3.11+](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-blue)](https://www.python.org/)
 [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
-[![Tests](https://img.shields.io/badge/tests-128%20passing-brightgreen)](tests/)
+[![Tests](https://img.shields.io/badge/tests-261%20passing-brightgreen)](tests/)
 [![PyPI](https://img.shields.io/pypi/v/cachellm-proxy)](https://pypi.org/project/cachellm-proxy/)
 [![Hit rate](https://img.shields.io/badge/hit%20rate-77%25%20on%20Bedrock-orange)](docs/evaluation.md)
 
@@ -63,6 +63,7 @@ The same benchmark runs without any cloud credentials against the built-in fake
 | Feature | What it does | Why it exists |
 | --- | --- | --- |
 | **Drop-in OpenAI API** | Same request and response shape, streaming included, verified against the official `openai` Python SDK in CI | Adoption has to cost one line, or nobody adopts it |
+| **Runs with nothing installed** | Defaults to an in-process numpy store; uses Redis automatically when it can reach one | A cache you have to provision a server for does not get tried. Redis takes over when you actually need shared, durable state |
 | **Two-tier cache** | Exact-match tier answers literal repeats in about a millisecond without embedding; semantic tier handles rewording | 82% of hits came from the exact tier: free, fast and impossible to get semantically wrong |
 | **Per-model threshold calibration** | Ships measured safe thresholds for six embedding models and picks the right one automatically | Measured safe thresholds span 0.89 to 0.98. A threshold copied between models is a guess |
 | **Cacheability policy** | Classifies every prompt and decides cacheable, category, TTL | Creative writing, live data and personal questions must not be cached like a fact |
@@ -86,7 +87,8 @@ The same benchmark runs without any cloud credentials against the built-in fake
 | API | FastAPI + Uvicorn | Async, native streaming, OpenAPI docs for free |
 | Validation and config | Pydantic v2, pydantic-settings | Typed request shapes and typed configuration from the environment |
 | Embeddings | fastembed, `all-MiniLM-L6-v2` (ONNX, CPU) | In-process and about 6 ms. A hosted embedding API would put 100 ms in front of every cache hit and defeat the point |
-| Vector store | Redis 8 + RedisVL, HNSW over cosine | One process is the exact-match store, the vector index and the stampede lock. Sub-millisecond, and TTL expiry removes entries from the index for free |
+| Vector store, default | numpy matrix in the proxy's own memory | Nothing to install. Scanning 20,000 cached prompts takes 0.85 ms, where a Redis round trip alone costs 2 to 3 ms, so below roughly 100k entries this is not a compromise, it is faster |
+| Vector store, at scale | Redis 8 + RedisVL, HNSW over cosine | One process is the exact-match store, the vector index and the stampede lock. Shared across workers, survives restarts, and an approximate index starts paying above ~100k entries |
 | Providers | AWS Bedrock (Converse), any OpenAI-compatible endpoint, deterministic fake | Converse reaches Nova, Claude, Llama and Mistral with one request shape and no extra vendor keys |
 | Metrics | prometheus-client, Prometheus, Grafana | Dashboard ships provisioned, so a reviewer sees data on first boot |
 | Tracing | OpenTelemetry, optional, GenAI semantic conventions | Instrument once, export to Langfuse, Tempo or Jaeger |
@@ -222,32 +224,47 @@ MiniLM gives four times the safe recall of bge-small at a third of the download
 
 ## Quick start
 
-You need Python 3.11 or newer and a Redis 8 instance. Redis 8 is required because the vector index needs the query engine, and it must be database 0 because Redis Search only indexes that one.
+Python 3.11 or newer, and nothing else.
 
 ```bash
 pip install cachellm-proxy
+cachellm serve
 ```
 
-The distribution is `cachellm-proxy` because PyPI blocks `cachellm` as too close to an existing `cachelm`. The import name and the CLI are both still `cachellm`.
+That is the whole install. No Redis, no Docker, no config file. On startup it looks for a provider you already have: an API key in your environment, Ollama running locally, or AWS credentials. It logs which one it picked and how to override it.
 
-Or from source, which is what you want if you plan to change anything:
+To see what it found before starting anything:
 
 ```bash
-git clone https://github.com/adarshcod30/CacheLLM.git
-cd CacheLLM
-uv sync
+cachellm providers
 ```
 
-Start Redis if you do not have one running:
+To try it with no account at all, the built-in test double stands in for a model:
 
 ```bash
-docker run -d --name redis -p 6379:6379 redis:8-alpine
+CACHELLM_DEFAULT_PROVIDER=fake CACHELLM_FAKE_LATENCY_MS=600 cachellm serve
 ```
 
-Run the proxy. With no configuration it uses the built-in fake provider, so you can see it working before wiring up a real model or spending anything:
+You only install what you actually route to. The base package is the proxy plus the local embedding model; AWS and Redis are extras, and nothing here installs Grafana or a Prometheus server, which are separate programs rather than Python packages.
+
+| You want | Install |
+| --- | --- |
+| Any OpenAI-compatible endpoint: OpenAI, Groq, Gemini, Ollama, vLLM | `pip install cachellm-proxy` |
+| AWS Bedrock | `pip install "cachellm-proxy[aws]"` |
+| Redis instead of the in-process cache | `pip install "cachellm-proxy[redis]"` |
+| Tracing to Langfuse or Tempo | `pip install "cachellm-proxy[observability]"` |
+| Everything | `pip install "cachellm-proxy[all]"` |
+
+Ask for a provider or backend whose extra is missing and the proxy names the exact command to fix it rather than raising an import error.
+
+The distribution is `cachellm-proxy` because PyPI blocks `cachellm` as too close to an existing `cachelm`. The import name and the CLI are both still `cachellm`.
+
+From source, if you plan to change anything:
 
 ```bash
-CACHELLM_DEFAULT_PROVIDER=fake CACHELLM_FAKE_LATENCY_MS=600 uv run cachellm serve
+git clone https://github.com/adarshcod30/CacheLLM.git
+cd CacheLLM
+uv sync && uv run cachellm serve
 ```
 
 In another terminal, ask the same thing twice and watch the second one come back instantly:
@@ -294,28 +311,53 @@ make tune && make compare-models && make bench
 
 ### Using a real provider
 
-AWS Bedrock needs no extra vendor keys and reaches Nova, Claude, Llama and Mistral through one API:
+One environment variable per host. Everything except Bedrock speaks the OpenAI protocol, so a single adapter covers all of it and needs no extra.
+
+| Host | `CACHELLM_OPENAI_BASE_URL` | Example model |
+| --- | --- | --- |
+| OpenAI | `https://api.openai.com/v1` | `gpt-4o-mini` |
+| Groq | `https://api.groq.com/openai/v1` | `llama-3.3-70b-versatile` |
+| Google Gemini | `https://generativelanguage.googleapis.com/v1beta/openai/` | `gemini-2.5-flash` |
+| Anthropic | `https://api.anthropic.com/v1` | `claude-haiku-4-5` |
+| OpenRouter | `https://openrouter.ai/api/v1` | `anthropic/claude-3.5-sonnet` |
+| Together | `https://api.together.xyz/v1` | `meta-llama/Llama-3.3-70B-Instruct-Turbo` |
+| Ollama, local | `http://localhost:11434/v1` | `llama3.2` |
+| vLLM or LM Studio | `http://localhost:8000/v1` | whatever you served |
 
 ```bash
-CACHELLM_DEFAULT_PROVIDER=bedrock AWS_REGION=us-east-1 uv run cachellm serve
+CACHELLM_OPENAI_BASE_URL=https://api.groq.com/openai/v1 \
+CACHELLM_OPENAI_API_KEY=gsk_your_key \
+cachellm serve
 ```
 
-If your credentials come from `aws login` rather than static keys or an SSO profile, install the CRT extra once, because that credential provider needs it:
+**AWS Bedrock** is the one exception, because it does not speak the OpenAI protocol. It needs the `aws` extra, and then uses your existing AWS credentials with no vendor API key at all:
 
 ```bash
-uv sync --extra aws
+pip install "cachellm-proxy[aws]"
+CACHELLM_DEFAULT_PROVIDER=bedrock AWS_REGION=us-east-1 cachellm serve
 ```
 
-The proxy detects that case and says so in the error rather than passing along boto's version of the message.
-
 ```bash
 curl -s http://localhost:8080/v1/chat/completions -H 'Content-Type: application/json' -d '{"model":"bedrock/us.amazon.nova-micro-v1:0","temperature":0,"messages":[{"role":"user","content":"What is Redis used for?"}]}'
 ```
 
-Any OpenAI-compatible endpoint works too, which covers OpenAI, Groq, Together, OpenRouter and a local vLLM:
+If your credentials come from `aws login` rather than static keys or an SSO profile, that provider needs the CRT extra, which the `aws` extra already includes. The proxy detects that case and says so in the error rather than passing along boto's version of the message.
+
+### How a model name gets routed
+
+Three rules, in order. You never configure a model list.
+
+1. **An explicit prefix** this proxy owns: `bedrock/…`, `openai/…`, `fake/…`.
+2. **A recognisable vendor convention.** Bedrock ids are always `vendor.model`, so `amazon.nova-lite-v1:0` and `us.anthropic.claude-3-haiku-20240307-v1:0` are identified with no prefix. OpenAI's own families (`gpt-`, `o1`, `o3`, `text-embedding-`) are identified the same way, whatever the default is.
+3. **Otherwise the configured default**, which is the OpenAI-compatible adapter. Names like `llama3.2`, `mixtral-8x7b-32768` and `qwen2.5-coder:7b` are served by Groq, Ollama, Together and OpenRouter alike, so the endpoint you configured is the only sensible answer.
+
+Model ids that legitimately contain a slash, which OpenRouter and Together both use, are forwarded whole. Only this proxy's own prefixes are stripped. Note that `anthropic.claude-…` with a dot is a Bedrock id while `anthropic/claude-…` with a slash is an OpenRouter id, and the router tells them apart.
+
+Ask it directly if you are unsure:
 
 ```bash
-CACHELLM_DEFAULT_PROVIDER=openai CACHELLM_OPENAI_BASE_URL=https://api.groq.com/openai/v1 CACHELLM_OPENAI_API_KEY=... uv run cachellm serve
+curl -s localhost:8080/admin/route/meta-llama/Llama-3.3-70B-Instruct-Turbo
+curl -s localhost:8080/admin/providers
 ```
 
 ## API reference
@@ -352,6 +394,8 @@ Clients can steer per request with `X-Cache-Control`:
 | --- | --- |
 | `GET /admin/stats` | Hit rate, tier split, money saved, latency percentiles, entry count |
 | `GET /admin/config` | Effective thresholds, TTLs and rules |
+| `GET /admin/providers` | Every host, its base URL, its pip extra, example model ids |
+| `GET /admin/route/{model}` | Where one model name would go, and why |
 | `POST /admin/invalidate` | Drop by `namespace`, by `model`, or `all` |
 | `GET /admin/near-misses` | Recent lookups that landed just below threshold |
 | `GET /admin/near-miss-histogram` | Cumulative view of what a lower threshold would buy |
@@ -370,12 +414,39 @@ curl -s http://localhost:8080/admin/threshold-sweep -H 'Content-Type: applicatio
 ### Command line
 
 ```bash
-uv run cachellm serve          # run the proxy
-uv run cachellm stats          # cache statistics
-uv run cachellm config         # effective configuration
-uv run cachellm invalidate --all
-uv run cachellm tune pairs.jsonl
+cachellm serve                 # run the proxy, auto-detecting a provider
+cachellm providers             # every host, and what this machine can reach
+cachellm stats                 # hit rate, savings, latency, recent requests
+cachellm watch                 # follow requests live, like tail -f
+cachellm config                # effective configuration
+cachellm invalidate --all
+cachellm tune pairs.jsonl
+```
+
+`cachellm stats` is the dashboard, in your terminal:
+
 ```
+  CacheLLM · memory backend · all-MiniLM-L6-v2
+
+  HIT RATE    77.0%  ████████████████░░░░░░  1,540 of 2,000 requests
+             1,265 exact · 275 semantic · 460 missed · 0 bypassed
+
+  SAVED      $0.0135   spent $0.0038  78% lower
+             116,029 tokens never generated
+
+  LATENCY    cached       2.6 ms p50 ·      5.7 ms p95
+             uncached   797.0 ms p50 ·   1022.8 ms p95   179x faster
+
+  CACHE      451 entries · 9 coalesced · 17 near misses
+
+  time      result tier        latency  score  saved       prompt
+  02:30:55  HIT    exact         1.2ms  1.000  +$0.000010  What is Redis used for?
+  02:30:54  BYPASS             451.5ms    ·          ·     What is my order 1234…  (pii:long_digits)
+  02:30:53  HIT    semantic      7.4ms  0.952  +$0.000011  What is Redis typically used for?
+  02:30:53  MISS               454.8ms    ·          ·     How do I configure nginx for TLS?
+```
+
+Prometheus and Grafana are still there if you want history and alerting, but nothing needs them.
 
 ## Configuration
 
@@ -383,7 +454,10 @@ Every setting is an environment variable prefixed `CACHELLM_`, or a line in `.en
 
 | Variable | Default | Notes |
 | --- | --- | --- |
+| `CACHELLM_BACKEND` | `auto` | `memory` needs nothing, `redis` shares one cache across workers, `auto` uses Redis when reachable and memory when not |
 | `CACHELLM_REDIS_URL` | `redis://localhost:6379/0` | Must be database 0. Redis Search cannot index any other |
+| `CACHELLM_MEMORY_MAX_ENTRIES` | `50000` | Cap for the in-memory store. 50k of 384-dim vectors is about 73 MB |
+| `CACHELLM_MEMORY_SNAPSHOT_PATH` | unset | Persist the in-memory cache to this file so a restart does not start cold |
 | `CACHELLM_API_KEYS` | empty | Comma-separated client keys. Empty disables auth, which is local development only |
 | `CACHELLM_EMBEDDING_MODEL` | `all-MiniLM-L6-v2` | Changing this changes the safe threshold. See the calibration table |
 | `CACHELLM_THRESHOLD_*` | `0` | Zero means use the calibrated value for your model |

{cachellm_proxy-0.1.0 → cachellm_proxy-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cachellm-proxy"
-version = "0.1.0"
+version = "0.2.0"
 description = "A drop-in semantic cache for OpenAI-compatible LLM APIs: match requests by meaning, serve repeats instantly, cut spend."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -29,19 +29,15 @@ classifiers = [
     "Typing :: Typed",
 ]
 dependencies = [
-    "boto3>=1.43.91",
     "fastapi>=0.141.1",
+    "uvicorn[standard]>=0.52.4",
     "fastembed>=0.8.0",
-    "httpx>=0.28.1",
     "numpy>=2.4.6",
-    "openai>=3.11.0",
-    "prometheus-client>=0.26.0",
     "pydantic-settings>=2.15.0",
-    "redis>=8.1.0",
-    "redisvl>=0.27.1",
+    "prometheus-client>=0.26.0",
     "structlog>=26.1.0",
     "typer>=0.27.2",
-    "uvicorn[standard]>=0.52.4",
+    "httpx>=0.28.1",
 ]
 
 [[project.authors]]
@@ -57,6 +53,14 @@ Issues = "https://github.com/adarshcod30/CacheLLM/issues"
 cachellm = "cachellm:main"
 
 [project.optional-dependencies]
+aws = [
+    "boto3>=1.43.91",
+    "botocore[crt]>=1.43.91",
+]
+redis = [
+    "redis>=8.1.0",
+    "redisvl>=0.27.1",
+]
 observability = [
     "langfuse>=4.15.2",
     "opentelemetry-exporter-otlp-proto-http>=1.44.0",
@@ -70,7 +74,7 @@ datasets = [
     "datasets>=5.0.1",
     "pandas>=3.0.5",
 ]
-aws = ["botocore[crt]>=1.43.91"]
+all = ["cachellm-proxy[aws,redis,observability,datasets]"]
 
 [build-system]
 requires = ["uv_build>=0.11.23,<0.12.0"]
@@ -166,6 +170,7 @@ exclude_lines = [
 [dependency-groups]
 dev = [
     "mypy>=2.3.1",
+    "openai>=3.11.0",
     "pytest>=9.1.1",
     "pytest-asyncio>=1.4.0",
     "pytest-cov>=7.1.0",

{cachellm_proxy-0.1.0 → cachellm_proxy-0.2.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "cachellm-proxy"
-version = "0.1.0"
+version = "0.2.0"
 description = "A drop-in semantic cache for OpenAI-compatible LLM APIs: match requests by meaning, serve repeats instantly, cut spend."
 readme = "README.md"
 authors = [
@@ -23,19 +23,17 @@ classifiers = [
     "Typing :: Typed",
 ]
 dependencies = [
-    "boto3>=1.43.91",
+    # The proxy itself and the semantic matching. Nothing here is optional:
+    # this is the smallest set that can cache by meaning and serve HTTP.
     "fastapi>=0.141.1",
-    "fastembed>=0.8.0",
-    "httpx>=0.28.1",
-    "numpy>=2.4.6",
-    "openai>=3.11.0",
-    "prometheus-client>=0.26.0",
+    "uvicorn[standard]>=0.52.4",
+    "fastembed>=0.8.0",       # local embeddings, CPU, no API call
+    "numpy>=2.4.6",           # the default vector search is one matrix multiply
     "pydantic-settings>=2.15.0",
-    "redis>=8.1.0",
-    "redisvl>=0.27.1",
+    "prometheus-client>=0.26.0",  # ~50 KB: writes a text page, not a server
     "structlog>=26.1.0",
     "typer>=0.27.2",
-    "uvicorn[standard]>=0.52.4",
+    "httpx>=0.28.1",
 ]
 
 [project.urls]
@@ -47,6 +45,15 @@ Issues = "https://github.com/adarshcod30/CacheLLM/issues"
 cachellm = "cachellm:main"
 
 [project.optional-dependencies]
+# Only install what you actually route to.
+aws = [
+    "boto3>=1.43.91",
+    "botocore[crt]>=1.43.91",  # `aws login` credentials need the CRT extra
+]
+redis = [
+    "redis>=8.1.0",
+    "redisvl>=0.27.1",
+]
 observability = [
     "langfuse>=4.15.2",
     "opentelemetry-exporter-otlp-proto-http>=1.44.0",
@@ -60,8 +67,8 @@ datasets = [
     "datasets>=5.0.1",
     "pandas>=3.0.5",
 ]
-aws = [
-    "botocore[crt]>=1.43.91",
+all = [
+    "cachellm-proxy[aws,redis,observability,datasets]",
 ]
 
 [build-system]
@@ -77,6 +84,7 @@ module-name = "cachellm"
 [dependency-groups]
 dev = [
     "mypy>=2.3.1",
+    "openai>=3.11.0",   # only the SDK-compatibility test needs this
     "pytest>=9.1.1",
     "pytest-asyncio>=1.4.0",
     "pytest-cov>=7.1.0",

{cachellm_proxy-0.1.0 → cachellm_proxy-0.2.0}/src/cachellm/__init__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"
 
 __all__ = ["__version__", "main"]