llmtap-proxy 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llmtap_proxy-0.2.0/LICENSE +21 -0
- llmtap_proxy-0.2.0/PKG-INFO +609 -0
- llmtap_proxy-0.2.0/README.md +590 -0
- llmtap_proxy-0.2.0/llmtap/__init__.py +2 -0
- llmtap_proxy-0.2.0/llmtap/__main__.py +3 -0
- llmtap_proxy-0.2.0/llmtap/budget.py +166 -0
- llmtap_proxy-0.2.0/llmtap/cache.py +129 -0
- llmtap_proxy-0.2.0/llmtap/cassette.py +172 -0
- llmtap_proxy-0.2.0/llmtap/cli.py +1066 -0
- llmtap_proxy-0.2.0/llmtap/config.py +112 -0
- llmtap_proxy-0.2.0/llmtap/dashboard.py +445 -0
- llmtap_proxy-0.2.0/llmtap/demo.py +244 -0
- llmtap_proxy-0.2.0/llmtap/doctor.py +275 -0
- llmtap_proxy-0.2.0/llmtap/guard.py +391 -0
- llmtap_proxy-0.2.0/llmtap/mock.py +478 -0
- llmtap_proxy-0.2.0/llmtap/models.py +75 -0
- llmtap_proxy-0.2.0/llmtap/pricing.py +379 -0
- llmtap_proxy-0.2.0/llmtap/providers.py +635 -0
- llmtap_proxy-0.2.0/llmtap/proxy.py +1153 -0
- llmtap_proxy-0.2.0/llmtap/pytest_plugin.py +444 -0
- llmtap_proxy-0.2.0/llmtap/report.py +893 -0
- llmtap_proxy-0.2.0/llmtap/store.py +319 -0
- llmtap_proxy-0.2.0/llmtap/term.py +477 -0
- llmtap_proxy-0.2.0/llmtap/ui/index.html +893 -0
- llmtap_proxy-0.2.0/llmtap_proxy.egg-info/PKG-INFO +609 -0
- llmtap_proxy-0.2.0/llmtap_proxy.egg-info/SOURCES.txt +48 -0
- llmtap_proxy-0.2.0/llmtap_proxy.egg-info/dependency_links.txt +1 -0
- llmtap_proxy-0.2.0/llmtap_proxy.egg-info/entry_points.txt +5 -0
- llmtap_proxy-0.2.0/llmtap_proxy.egg-info/requires.txt +3 -0
- llmtap_proxy-0.2.0/llmtap_proxy.egg-info/top_level.txt +1 -0
- llmtap_proxy-0.2.0/pyproject.toml +42 -0
- llmtap_proxy-0.2.0/setup.cfg +4 -0
- llmtap_proxy-0.2.0/tests/test_budget.py +319 -0
- llmtap_proxy-0.2.0/tests/test_cache.py +178 -0
- llmtap_proxy-0.2.0/tests/test_cassette.py +235 -0
- llmtap_proxy-0.2.0/tests/test_cli.py +482 -0
- llmtap_proxy-0.2.0/tests/test_cli_extra.py +234 -0
- llmtap_proxy-0.2.0/tests/test_dashboard.py +405 -0
- llmtap_proxy-0.2.0/tests/test_e2e.py +475 -0
- llmtap_proxy-0.2.0/tests/test_gemini.py +356 -0
- llmtap_proxy-0.2.0/tests/test_guard.py +554 -0
- llmtap_proxy-0.2.0/tests/test_hardening.py +1046 -0
- llmtap_proxy-0.2.0/tests/test_mock.py +251 -0
- llmtap_proxy-0.2.0/tests/test_pricing.py +312 -0
- llmtap_proxy-0.2.0/tests/test_providers.py +627 -0
- llmtap_proxy-0.2.0/tests/test_proxy.py +575 -0
- llmtap_proxy-0.2.0/tests/test_pytest_plugin.py +516 -0
- llmtap_proxy-0.2.0/tests/test_report.py +285 -0
- llmtap_proxy-0.2.0/tests/test_store.py +352 -0
- llmtap_proxy-0.2.0/tests/test_term.py +201 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 llmtap contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,609 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: llmtap-proxy
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: The flight recorder & firewall for LLM apps. See, cache, replay, budget and secure every LLM call β zero code changes, zero dependencies.
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/AshishAgrawal101/llmtap
|
|
7
|
+
Project-URL: Issues, https://github.com/AshishAgrawal101/llmtap/issues
|
|
8
|
+
Keywords: llm,openai,anthropic,claude,proxy,observability,cache,testing,vcr,agents,cost
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Topic :: Software Development :: Testing
|
|
12
|
+
Classifier: Topic :: Software Development :: Debuggers
|
|
13
|
+
Requires-Python: >=3.11
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
|
|
20
|
+
<div align="center">
|
|
21
|
+
|
|
22
|
+
<img src="assets/logo.svg" alt="llmtap" width="360">
|
|
23
|
+
|
|
24
|
+
### The flight recorder & firewall for LLM apps.
|
|
25
|
+
|
|
26
|
+
**One command. Zero code changes. Zero dependencies.**<br>
|
|
27
|
+
See every LLM call live, know exactly what it cost, replay it in CI for free, and stop secrets and runaway agents before they hit the API.
|
|
28
|
+
|
|
29
|
+
[](https://www.python.org/)
|
|
30
|
+
[](pyproject.toml)
|
|
31
|
+
[](LICENSE)
|
|
32
|
+
[](CONTRIBUTING.md)
|
|
33
|
+
|
|
34
|
+
[30-second start](#-30-second-start) Β·
|
|
35
|
+
[Features](#-features) Β·
|
|
36
|
+
[Cassettes for CI](#-recordreplay-cassettes-vcr-for-llms) Β·
|
|
37
|
+
[Guard](#-secret-guard) Β·
|
|
38
|
+
[CLI](#-cli-cheatsheet) Β·
|
|
39
|
+
[How it works](#-how-it-works) Β·
|
|
40
|
+
[Comparison](#-how-does-it-compare) Β·
|
|
41
|
+
[FAQ](#-faq)
|
|
42
|
+
|
|
43
|
+
</div>
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
llmtap wrap -- python my_agent.py
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
That's the whole integration. llmtap starts a local proxy, points your app's LLM SDKs at it through
|
|
50
|
+
environment variables, runs your command, and prints what happened when it exits: every request, every token,
|
|
51
|
+
every cent.
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## β‘ 30-second start
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pipx install llmtap-proxy # stdlib only, nothing else gets installed; the command is `llmtap`
|
|
59
|
+
llmtap demo # fake LLM traffic + the live dashboard in your browser. No API key needed.
|
|
60
|
+
llmtap wrap -- python app.py # now do it for real with your own app
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
<div align="center">
|
|
64
|
+
<img src="assets/demo.gif" alt="llmtap demo: live traffic, cache hits, a leaked key caught, a runaway loop blocked" width="900">
|
|
65
|
+
<br><br>
|
|
66
|
+
<img src="assets/dashboard.png" alt="The llmtap dashboard: live request feed, spend, tokens, cache hits and guard findings" width="900">
|
|
67
|
+
<br><sub>The live dashboard at <code>http://127.0.0.1:4141/_llmtap/</code>. Works offline, no CDN, no account.</sub>
|
|
68
|
+
</div>
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
|
|
72
|
+
## π€ Why
|
|
73
|
+
|
|
74
|
+
If you build with LLMs you have probably hit at least one of these:
|
|
75
|
+
|
|
76
|
+
- **Surprise bills.** An eval loop or a forgotten `for` loop burned $80 overnight, and the provider console only tells you *tomorrow*, in aggregate.
|
|
77
|
+
- **Flaky, expensive LLM tests.** CI needs a real API key, every run costs money, and the model answers slightly differently each time.
|
|
78
|
+
- **Leaked keys in prompts.** Someone pastes a `.env` file, a stack trace or a config blob into a prompt, and your AWS key is now in a third party's logs.
|
|
79
|
+
- **Agents stuck in loops at 3am.** The agent retries the same tool call 400 times. Nothing stops it except your credit card limit.
|
|
80
|
+
- **No idea what your framework actually sends.** LangChain, LlamaIndex or an agent SDK builds a 9,000-token prompt behind your back, and you can't see it.
|
|
81
|
+
|
|
82
|
+
llmtap sits on the wire between your code and the LLM API, so it sees *exactly* what is sent and received,
|
|
83
|
+
no matter which language, SDK or framework produced it. It's just a base URL.
|
|
84
|
+
|
|
85
|
+
## β¨ Features
|
|
86
|
+
|
|
87
|
+
<table>
|
|
88
|
+
<tr>
|
|
89
|
+
<td width="33%" valign="top">
|
|
90
|
+
|
|
91
|
+
**π΄ Live dashboard**<br>
|
|
92
|
+
Every request and response, streaming included, in a local web UI. Search prompts, inspect raw bodies, watch spend tick up in real time.
|
|
93
|
+
|
|
94
|
+
</td>
|
|
95
|
+
<td width="33%" valign="top">
|
|
96
|
+
|
|
97
|
+
**πΈ Exact cost tracking**<br>
|
|
98
|
+
Per-call USD cost from reported token usage (including prompt-cache reads and writes) for Claude, GPT, o-series, Gemini-via-OpenRouter, Llama, DeepSeek and more.
|
|
99
|
+
|
|
100
|
+
</td>
|
|
101
|
+
<td width="33%" valign="top">
|
|
102
|
+
|
|
103
|
+
**πΌ Record & replay**<br>
|
|
104
|
+
VCR for LLMs. Record real traffic to a `.jsonl` cassette once, then replay it in CI with fake keys: deterministic, offline, $0.
|
|
105
|
+
|
|
106
|
+
</td>
|
|
107
|
+
</tr>
|
|
108
|
+
<tr>
|
|
109
|
+
<td valign="top">
|
|
110
|
+
|
|
111
|
+
**β»οΈ Dev response cache**<br>
|
|
112
|
+
Identical requests are served from a local cache, so re-running your script while you iterate on the code around the prompt costs nothing.
|
|
113
|
+
|
|
114
|
+
</td>
|
|
115
|
+
<td valign="top">
|
|
116
|
+
|
|
117
|
+
**π‘οΈ Secret guard**<br>
|
|
118
|
+
Detects API keys, tokens, private keys and JWTs (and optionally emails, phone numbers and cards) in prompts. Warn, redact or block.
|
|
119
|
+
|
|
120
|
+
</td>
|
|
121
|
+
<td valign="top">
|
|
122
|
+
|
|
123
|
+
**π§― Budgets & loop breaker**<br>
|
|
124
|
+
Hard daily and per-session spend caps, plus automatic blocking of identical requests repeated N times in W seconds.
|
|
125
|
+
|
|
126
|
+
</td>
|
|
127
|
+
</tr>
|
|
128
|
+
<tr>
|
|
129
|
+
<td valign="top">
|
|
130
|
+
|
|
131
|
+
**π Every provider**<br>
|
|
132
|
+
Anthropic, OpenAI, OpenRouter, Groq, Together, Mistral, DeepSeek, Ollama, and any OpenAI-compatible endpoint you add.
|
|
133
|
+
|
|
134
|
+
</td>
|
|
135
|
+
<td valign="top">
|
|
136
|
+
|
|
137
|
+
**π Every language**<br>
|
|
138
|
+
Python, Node, Go, Rust, curl: if it lets you set a base URL, llmtap sees it. Streaming (SSE) is relayed chunk by chunk.
|
|
139
|
+
|
|
140
|
+
</td>
|
|
141
|
+
<td valign="top">
|
|
142
|
+
|
|
143
|
+
**π¦ Zero dependencies**<br>
|
|
144
|
+
Pure Python 3.11+ standard library. SQLite for storage. No Docker, no signup, no telemetry. Nothing leaves your machine except the upstream call.
|
|
145
|
+
|
|
146
|
+
</td>
|
|
147
|
+
</tr>
|
|
148
|
+
</table>
|
|
149
|
+
|
|
150
|
+
---
|
|
151
|
+
|
|
152
|
+
## π Usage
|
|
153
|
+
|
|
154
|
+
### Option 1: `llmtap wrap` (recommended)
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
llmtap wrap -- python my_agent.py
|
|
158
|
+
llmtap wrap -- npm run dev
|
|
159
|
+
llmtap wrap --cache --guard redact -- pytest -x
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
`wrap` starts the proxy on a free port, runs your command with these variables set, propagates the exit code,
|
|
163
|
+
and prints a cost/usage summary when the command finishes:
|
|
164
|
+
|
|
165
|
+
| Variable | Value | Read by |
|
|
166
|
+
|---|---|---|
|
|
167
|
+
| `ANTHROPIC_BASE_URL` | `http://127.0.0.1:<port>` | Anthropic SDKs (Python, TypeScript, Goβ¦), Claude Agent SDK, Claude Code |
|
|
168
|
+
| `OPENAI_BASE_URL` | `http://127.0.0.1:<port>/v1` | OpenAI SDKs (Python, Nodeβ¦) and most OpenAI-compatible clients |
|
|
169
|
+
| `OPENAI_API_BASE` | `http://127.0.0.1:<port>/v1` | Older clients, LangChain, Aider |
|
|
170
|
+
| `LLMTAP_URL` | `http://127.0.0.1:<port>` | Your own code, if you want to build URLs for other upstreams |
|
|
171
|
+
|
|
172
|
+
### Option 2: run the proxy, point your app at it
|
|
173
|
+
|
|
174
|
+
```bash
|
|
175
|
+
llmtap serve # proxy + dashboard on http://127.0.0.1:4141
|
|
176
|
+
eval "$(llmtap env)" # in another shell: export the base-URL variables
|
|
177
|
+
python my_agent.py
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
Or set the base URL yourself, per SDK:
|
|
181
|
+
|
|
182
|
+
<details open>
|
|
183
|
+
<summary><b>Python: <code>anthropic</code></b></summary>
|
|
184
|
+
|
|
185
|
+
```python
|
|
186
|
+
# Zero code changes: ANTHROPIC_BASE_URL=http://127.0.0.1:4141 python app.py
|
|
187
|
+
# or explicitly:
|
|
188
|
+
import anthropic
|
|
189
|
+
|
|
190
|
+
client = anthropic.Anthropic(base_url="http://127.0.0.1:4141")
|
|
191
|
+
msg = client.messages.create(
|
|
192
|
+
model="claude-sonnet-5-5",
|
|
193
|
+
max_tokens=512,
|
|
194
|
+
messages=[{"role": "user", "content": "Explain SSE in one paragraph."}],
|
|
195
|
+
)
|
|
196
|
+
```
|
|
197
|
+
</details>
|
|
198
|
+
|
|
199
|
+
<details open>
|
|
200
|
+
<summary><b>Python: <code>openai</code></b></summary>
|
|
201
|
+
|
|
202
|
+
```python
|
|
203
|
+
# Zero code changes: OPENAI_BASE_URL=http://127.0.0.1:4141/v1 python app.py
|
|
204
|
+
# or explicitly:
|
|
205
|
+
from openai import OpenAI
|
|
206
|
+
|
|
207
|
+
client = OpenAI(base_url="http://127.0.0.1:4141/v1")
|
|
208
|
+
resp = client.chat.completions.create(
|
|
209
|
+
model="gpt-4o-mini",
|
|
210
|
+
messages=[{"role": "user", "content": "Write a haiku about proxies."}],
|
|
211
|
+
)
|
|
212
|
+
```
|
|
213
|
+
</details>
|
|
214
|
+
|
|
215
|
+
<details>
|
|
216
|
+
<summary><b>Node / TypeScript</b></summary>
|
|
217
|
+
|
|
218
|
+
```js
|
|
219
|
+
// Zero code changes: OPENAI_BASE_URL / ANTHROPIC_BASE_URL are read by the official SDKs.
|
|
220
|
+
import OpenAI from "openai";
|
|
221
|
+
import Anthropic from "@anthropic-ai/sdk";
|
|
222
|
+
|
|
223
|
+
const openai = new OpenAI({ baseURL: "http://127.0.0.1:4141/v1" });
|
|
224
|
+
const anthropic = new Anthropic({ baseURL: "http://127.0.0.1:4141" });
|
|
225
|
+
```
|
|
226
|
+
</details>
|
|
227
|
+
|
|
228
|
+
<details>
|
|
229
|
+
<summary><b>curl</b></summary>
|
|
230
|
+
|
|
231
|
+
```bash
|
|
232
|
+
# Anthropic
|
|
233
|
+
curl http://127.0.0.1:4141/v1/messages \
|
|
234
|
+
-H "x-api-key: $ANTHROPIC_API_KEY" -H "anthropic-version: 2023-06-01" \
|
|
235
|
+
-H "content-type: application/json" \
|
|
236
|
+
-d '{"model":"claude-haiku-4-5","max_tokens":128,"messages":[{"role":"user","content":"hi"}]}'
|
|
237
|
+
|
|
238
|
+
# OpenAI
|
|
239
|
+
curl http://127.0.0.1:4141/v1/chat/completions \
|
|
240
|
+
-H "authorization: Bearer $OPENAI_API_KEY" -H "content-type: application/json" \
|
|
241
|
+
-d '{"model":"gpt-4o-mini","messages":[{"role":"user","content":"hi"}]}'
|
|
242
|
+
```
|
|
243
|
+
</details>
|
|
244
|
+
|
|
245
|
+
More: Go, LangChain, LlamaIndex, Vercel AI SDK, Claude Agent SDK, LiteLLM, DSPy, Aider, Ollama⦠in
|
|
246
|
+
**[docs/integrations.md](docs/integrations.md)**.
|
|
247
|
+
|
|
248
|
+
### Other providers: the `/<name>/` prefix
|
|
249
|
+
|
|
250
|
+
Any upstream in the config gets a path prefix. Point an OpenAI-compatible client at `http://127.0.0.1:4141/<name>/v1`:
|
|
251
|
+
|
|
252
|
+
| Provider | Base URL to use | Forwards to |
|
|
253
|
+
|---|---|---|
|
|
254
|
+
| OpenRouter | `http://127.0.0.1:4141/openrouter/v1` | `https://openrouter.ai/api/v1` |
|
|
255
|
+
| Groq | `http://127.0.0.1:4141/groq/v1` | `https://api.groq.com/openai/v1` |
|
|
256
|
+
| Together | `http://127.0.0.1:4141/together/v1` | `https://api.together.xyz/v1` |
|
|
257
|
+
| Mistral | `http://127.0.0.1:4141/mistral/v1` | `https://api.mistral.ai/v1` |
|
|
258
|
+
| DeepSeek | `http://127.0.0.1:4141/deepseek/v1` | `https://api.deepseek.com/v1` |
|
|
259
|
+
| Ollama | `http://127.0.0.1:4141/ollama/v1` | `http://localhost:11434/v1` |
|
|
260
|
+
|
|
261
|
+
```python
|
|
262
|
+
from openai import OpenAI
|
|
263
|
+
client = OpenAI(base_url="http://127.0.0.1:4141/openrouter/v1", api_key=os.environ["OPENROUTER_API_KEY"])
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
Add your own (vLLM, LM Studio, an internal gatewayβ¦) in `llmtap.toml`:
|
|
267
|
+
|
|
268
|
+
```toml
|
|
269
|
+
[upstreams.vllm]
|
|
270
|
+
base = "http://gpu-box:8000"
|
|
271
|
+
style = "openai" # or "anthropic"
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
β¦then use `http://127.0.0.1:4141/vllm/v1`. Without a prefix, requests are routed by shape: `/v1/messages` or an
|
|
275
|
+
`anthropic-version` / `x-api-key` header goes to Anthropic, everything else to OpenAI.
|
|
276
|
+
|
|
277
|
+
### Tagging requests
|
|
278
|
+
|
|
279
|
+
Group traffic by run, user or feature with headers. llmtap strips every `X-LLMTap-*` header before forwarding, so the provider never sees them.
|
|
280
|
+
|
|
281
|
+
| Header | Becomes tag |
|
|
282
|
+
|---|---|
|
|
283
|
+
| `X-LLMTap-Session: nightly-eval-42` | `session=nightly-eval-42` |
|
|
284
|
+
| `X-LLMTap-Tag-Feature: summarizer` | `feature=summarizer` |
|
|
285
|
+
| `X-LLMTap-Tag-<Anything>: value` | `<anything>=value` |
|
|
286
|
+
|
|
287
|
+
```python
|
|
288
|
+
client = anthropic.Anthropic(default_headers={
|
|
289
|
+
"X-LLMTap-Session": "refactor-agent-run-7",
|
|
290
|
+
"X-LLMTap-Tag-Feature": "code-review",
|
|
291
|
+
})
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
```bash
|
|
295
|
+
llmtap stats --by tag:feature --since 7d # what does each feature cost?
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
---
|
|
299
|
+
|
|
300
|
+
## πΌ Record/replay cassettes (VCR for LLMs)
|
|
301
|
+
|
|
302
|
+
Record real LLM traffic once, commit it, and your test suite replays it forever: **no API keys in CI, no cost,
|
|
303
|
+
no flakiness**, and it works for any language because it happens at the HTTP layer.
|
|
304
|
+
|
|
305
|
+
```bash
|
|
306
|
+
# 1. Record locally, with real keys
|
|
307
|
+
llmtap wrap --cassette tests/cassettes/llm.jsonl --mode record -- pytest
|
|
308
|
+
|
|
309
|
+
# 2. Commit the cassette
|
|
310
|
+
git add tests/cassettes/llm.jsonl
|
|
311
|
+
|
|
312
|
+
# 3. Replay in CI, with fake keys. Any request that isn't on tape fails with HTTP 409.
|
|
313
|
+
ANTHROPIC_API_KEY=fake OPENAI_API_KEY=fake \
|
|
314
|
+
llmtap wrap --cassette tests/cassettes/llm.jsonl --mode replay -- pytest
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
| Mode | Behaviour |
|
|
318
|
+
|---|---|
|
|
319
|
+
| `record` | Start a fresh cassette; forward everything upstream and record every 2xx response. |
|
|
320
|
+
| `replay` | Serve only from the cassette. A miss returns HTTP **409** with a provider-shaped error (or goes upstream with `--allow-miss`). |
|
|
321
|
+
| `auto` *(default)* | Replay what's on tape, forward and record anything new. |
|
|
322
|
+
|
|
323
|
+
**Matching:** a request matches a recording when the provider, the path (query string ignored) and the JSON body are
|
|
324
|
+
identical, compared after canonicalising key order and dropping the volatile top-level fields `user`, `metadata`
|
|
325
|
+
and `request_id`. Headers (including API keys) are not part of the match. If the same request was recorded several times, the recordings are
|
|
326
|
+
replayed in order and the last one repeats. Streaming responses are replayed as streams.
|
|
327
|
+
|
|
328
|
+
**GitHub Actions** (there's a composite action in this repo):
|
|
329
|
+
|
|
330
|
+
```yaml
|
|
331
|
+
- uses: actions/checkout@v4
|
|
332
|
+
- uses: actions/setup-python@v5
|
|
333
|
+
with: { python-version: "3.12" }
|
|
334
|
+
- run: pip install -e .
|
|
335
|
+
- uses: AshishAgrawal101/llmtap@main
|
|
336
|
+
with:
|
|
337
|
+
cassette: tests/cassettes/llm.jsonl
|
|
338
|
+
command: pytest -q
|
|
339
|
+
```
|
|
340
|
+
|
|
341
|
+
Full guide with a pytest example: **[docs/cassettes.md](docs/cassettes.md)** Β· [examples/pytest_cassette](examples/pytest_cassette) Β· [examples/github-actions.yml](examples/github-actions.yml)
|
|
342
|
+
|
|
343
|
+
## π§ͺ pytest plugin: free, deterministic LLM tests
|
|
344
|
+
|
|
345
|
+
```python
|
|
346
|
+
def test_summary(llmtap): # `llmtap` fixture ships with the package
|
|
347
|
+
out = summarize("...") # your normal code, calling the real SDK
|
|
348
|
+
assert "refund" in out.lower()
|
|
349
|
+
```
|
|
350
|
+
|
|
351
|
+
The first run (with real keys) records to `cassettes/<test>.jsonl`. Commit it. After that the test replays offline:
|
|
352
|
+
no API key, no cost, no flakiness. In CI (`CI=1`) it defaults to replay, and a request with no recording fails the test
|
|
353
|
+
with a message telling you to re-record (`pytest --llmtap-record`). Details: [docs/pytest-plugin.md](docs/pytest-plugin.md).
|
|
354
|
+
|
|
355
|
+
## π Find where the money goes
|
|
356
|
+
|
|
357
|
+
```bash
|
|
358
|
+
llmtap report --since 7d -o report.html # shareable, self-contained HTML (or --format md / json)
|
|
359
|
+
llmtap stats # ... "Potential savings from caching: $X (N duplicate requests)"
|
|
360
|
+
llmtap doctor # checks your setup and tells you how to fix it
|
|
361
|
+
```
|
|
362
|
+
|
|
363
|
+
The report's **waste finder** groups identical requests and tells you what `--cache` would have saved.
|
|
364
|
+
See [docs/report.md](docs/report.md) and a [sample report](assets/sample-report.html).
|
|
365
|
+
|
|
366
|
+
## β»οΈ Dev cache
|
|
367
|
+
|
|
368
|
+
```bash
|
|
369
|
+
llmtap wrap --cache -- python app.py # identical requests are served locally
|
|
370
|
+
llmtap serve --cache --cache-ttl 3600 # expire entries after an hour
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
The cache uses the same matching as cassettes, stores only 2xx responses, and records what each hit
|
|
374
|
+
**saved** you. Cached responses show up as `cache` in the dashboard and in `llmtap stats --by source`.
|
|
375
|
+
|
|
376
|
+
## π‘οΈ Secret guard
|
|
377
|
+
|
|
378
|
+
llmtap scans every outgoing request body for credentials before it leaves your machine.
|
|
379
|
+
|
|
380
|
+
```bash
|
|
381
|
+
llmtap wrap --guard warn -- python app.py # default: log findings, send unchanged
|
|
382
|
+
llmtap wrap --guard redact -- python app.py # replace each match with [REDACTED:<kind>] and send
|
|
383
|
+
llmtap wrap --guard block -- python app.py # refuse the request with HTTP 400
|
|
384
|
+
llmtap wrap --guard block --pii -- python app.py # also emails, phone numbers, credit cards (Luhn-checked)
|
|
385
|
+
```
|
|
386
|
+
|
|
387
|
+
Detects AWS access & secret keys, Anthropic, OpenAI, GitHub, Slack, Stripe and Google API keys, PEM private keys,
|
|
388
|
+
JWTs and generic `password=` / `api_key:`-style assignments. Previews are masked (`sk-ant-β¦a1b2`), so the
|
|
389
|
+
dashboard never shows the full secret. It also works offline as a linter:
|
|
390
|
+
|
|
391
|
+
```bash
|
|
392
|
+
llmtap scan prompts/system.txt # exit code 1 if anything is found
|
|
393
|
+
git diff | llmtap scan -
|
|
394
|
+
```
|
|
395
|
+
|
|
396
|
+
Full table of detectors: **[docs/guard.md](docs/guard.md)**
|
|
397
|
+
|
|
398
|
+
## π§― Budgets & loop breaker
|
|
399
|
+
|
|
400
|
+
```bash
|
|
401
|
+
llmtap wrap --budget-daily 5 --budget-session 1 -- python agent.py
|
|
402
|
+
llmtap wrap --loop-threshold 5 --loop-window 60 -- python agent.py
|
|
403
|
+
```
|
|
404
|
+
|
|
405
|
+
- **`--budget-daily USD`**: once today's upstream spend (since local midnight, across all runs sharing the same database) reaches the cap, requests get HTTP **429**.
|
|
406
|
+
- **`--budget-session USD`**: the same, for spend through this proxy process only.
|
|
407
|
+
- **`--loop-threshold N --loop-window W`**: once the *identical* request has been sent N times within W seconds, further copies are refused with HTTP **429**. This is the classic stuck-agent pattern, and it gets stopped in seconds instead of at 3am.
|
|
408
|
+
|
|
409
|
+
Refusals are returned as provider-shaped error JSON, so your SDK raises its normal error type with a clear message
|
|
410
|
+
(e.g. `daily budget $5.00 reached ($5.02 spent)` or `loop detected: identical request sent 5 times in 60s`).
|
|
411
|
+
Cache and cassette hits cost nothing and don't count toward budgets.
|
|
412
|
+
|
|
413
|
+
## βοΈ Configuration
|
|
414
|
+
|
|
415
|
+
Precedence: defaults β `llmtap.toml` β `LLMTAP_*` env vars β CLI flags. llmtap looks for `./llmtap.toml`, then
|
|
416
|
+
`~/.config/llmtap/config.toml`, or use `--config path`.
|
|
417
|
+
|
|
418
|
+
```toml
|
|
419
|
+
# llmtap.toml
|
|
420
|
+
port = 4141
|
|
421
|
+
cache = true
|
|
422
|
+
guard = "redact"
|
|
423
|
+
guard_pii = false
|
|
424
|
+
budget_daily_usd = 10.0
|
|
425
|
+
loop_threshold = 5
|
|
426
|
+
loop_window_s = 60
|
|
427
|
+
|
|
428
|
+
[upstreams.vllm]
|
|
429
|
+
base = "http://gpu-box:8000"
|
|
430
|
+
style = "openai"
|
|
431
|
+
|
|
432
|
+
[pricing_overrides."my-finetune"] # USD per 1M tokens
|
|
433
|
+
input = 0.50
|
|
434
|
+
output = 1.50
|
|
435
|
+
```
|
|
436
|
+
|
|
437
|
+
Every field, env var and flag: **[docs/configuration.md](docs/configuration.md)** Β· full example: [llmtap.example.toml](llmtap.example.toml)
|
|
438
|
+
|
|
439
|
+
---
|
|
440
|
+
|
|
441
|
+
## π§° CLI cheatsheet
|
|
442
|
+
|
|
443
|
+
| Command | What it does |
|
|
444
|
+
|---|---|
|
|
445
|
+
| `llmtap demo` | Fake upstream + generated traffic + dashboard. Zero keys, zero cost. |
|
|
446
|
+
| `llmtap wrap [flags] -- <cmdβ¦>` | Run a command through a fresh proxy; print a cost summary at the end |
|
|
447
|
+
| `llmtap serve [flags]` | Run the proxy + dashboard on `127.0.0.1:4141` |
|
|
448
|
+
| `llmtap env [--port N]` | Print `export β¦` lines: `eval "$(llmtap env)"` |
|
|
449
|
+
| `llmtap tail` | Stream new requests to the terminal as they happen |
|
|
450
|
+
| `llmtap ls [-n 20] [-q text]` | List recent requests, optionally full-text filtered |
|
|
451
|
+
| `llmtap show <id> [--raw]` | Full request/response for one exchange (id prefixes work) |
|
|
452
|
+
| `llmtap stats [--since 24h] [--by model]` | Totals and breakdowns: `model`, `provider`, `source`, `day`, `hour`, `tag:<name>` |
|
|
453
|
+
| `llmtap export [--format jsonl\|csv] [-o file]` | Dump everything for your own analysis |
|
|
454
|
+
| `llmtap report [--since 7d] [-o file]` | Self-contained HTML / Markdown / JSON cost + safety report with a waste finder |
|
|
455
|
+
| `llmtap doctor [--online]` | Check your setup (port, db, env vars, SQLite) and get fix hints |
|
|
456
|
+
| `llmtap prices [filter]` | Show the built-in price table |
|
|
457
|
+
| `llmtap scan [file\|-]` | Scan text for secrets; exit 1 on findings |
|
|
458
|
+
| `llmtap clear [--yes]` | Delete recorded history |
|
|
459
|
+
| `llmtap version` | Print the version |
|
|
460
|
+
|
|
461
|
+
Common flags for `serve` / `wrap`: `--port --host --db --config --quiet --cache --cache-ttl --cassette --mode --allow-miss
|
|
462
|
+
--guard --pii --budget-daily --budget-session --loop-threshold --loop-window`.
|
|
463
|
+
|
|
464
|
+
---
|
|
465
|
+
|
|
466
|
+
## π How it works
|
|
467
|
+
|
|
468
|
+
```
|
|
469
|
+
your app / agent llmtap (127.0.0.1:4141) upstream APIs
|
|
470
|
+
any SDK, any language ββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
471
|
+
β 1. route /v1/messages, /groq/β¦, β¦ β
|
|
472
|
+
base URL ββββ request ββΆβ 2. guard secrets? warn/redact/block 400 β
|
|
473
|
+
β 3. budget over cap / loop? 429 β
|
|
474
|
+
β 4. cassette on tape? replay it β
|
|
475
|
+
β 5. cache seen before? serve it β
|
|
476
|
+
β 6. forward HTTPS, auth headers as-is ββββββΌβββΆ api.anthropic.com
|
|
477
|
+
ββββ response / SSE βββββ relay stream chunks as they arrive βββΌβββ api.openai.com
|
|
478
|
+
β 7. account usage βΆ cost βΆ record β openrouter, groq,
|
|
479
|
+
βββββββββββββββββ¬βββββββββββββββββββββββββββββββ ollama, vLLM, β¦
|
|
480
|
+
βΌ
|
|
481
|
+
SQLite ~/.llmtap/llmtap.db
|
|
482
|
+
βΌ
|
|
483
|
+
live dashboard /_llmtap/ Β· llmtap tail / stats / ls
|
|
484
|
+
```
|
|
485
|
+
|
|
486
|
+
1. Your SDK talks plain HTTP to llmtap on localhost, exactly as it would to the provider.
|
|
487
|
+
2. llmtap picks the upstream (`/v1/messages` β Anthropic, `/groq/β¦` β Groq, β¦), runs the guard, budget and loop checks, and tries the cassette and cache.
|
|
488
|
+
3. Otherwise it forwards the request over HTTPS with your original auth headers, relaying streamed chunks as they arrive.
|
|
489
|
+
4. It parses token usage from the response (or SSE stream), prices it, stores the exchange in SQLite, and pushes it to the dashboard.
|
|
490
|
+
|
|
491
|
+
Module map and design notes: **[docs/architecture.md](docs/architecture.md)**
|
|
492
|
+
|
|
493
|
+
---
|
|
494
|
+
|
|
495
|
+
## π₯ How does it compare?
|
|
496
|
+
|
|
497
|
+
llmtap is deliberately small: a local, zero-dependency tool for **seeing, testing and guarding** LLM traffic on your
|
|
498
|
+
machine and in CI. The hosted platforms do a lot we don't, so pick the right tool for the job.
|
|
499
|
+
|
|
500
|
+
| | **llmtap** | Helicone | LangSmith | LiteLLM proxy | vcrpy |
|
|
501
|
+
|---|:---:|:---:|:---:|:---:|:---:|
|
|
502
|
+
| Runs fully local, no signup | β
| β self-host option | β cloud-first | β
| β
|
|
|
503
|
+
| Zero code changes (base URL) | β
| β
| β SDK / env tracing | β
| β test decorators |
|
|
504
|
+
| Zero runtime dependencies | β
| β | β | β | β |
|
|
505
|
+
| Any language / SDK | β
| β
| β Python / JS SDKs | β
| β Python only |
|
|
506
|
+
| Live request dashboard | β
| β
| β
| β
| β |
|
|
507
|
+
| Cost tracking | β
| β
| β
| β
| β |
|
|
508
|
+
| Response cache | β
| β
| β | β
| β |
|
|
509
|
+
| Record/replay cassettes for CI | β
| β | β | β | β
|
|
|
510
|
+
| Secret guard on prompts | β
| β | β | β via guardrail plugins | β |
|
|
511
|
+
| Spend caps | β
| β rate limits | β | β
| β |
|
|
512
|
+
| Runaway-loop breaker | β
| β | β | β | β |
|
|
513
|
+
| Team workspaces, hosted history, alerts | β | β
| β
| β | β |
|
|
514
|
+
| Evals, datasets, prompt management | β | β | β
| β | β |
|
|
515
|
+
| Provider translation, fallbacks, load balancing | β | β | β | β
| β |
|
|
516
|
+
|
|
517
|
+
β
yes Β· β partial / depends on setup Β· β no. Based on public docs at the time of writing; if we got something wrong, please open a PR.
|
|
518
|
+
|
|
519
|
+
**They combine well:** run llmtap in front of a LiteLLM proxy (add it as an upstream), or use llmtap for local dev and
|
|
520
|
+
CI while production traces go to your hosted platform.
|
|
521
|
+
|
|
522
|
+
---
|
|
523
|
+
|
|
524
|
+
## β FAQ
|
|
525
|
+
|
|
526
|
+
<details>
|
|
527
|
+
<summary><b>Does my data leave my machine?</b></summary>
|
|
528
|
+
|
|
529
|
+
Only the upstream call you were going to make anyway. llmtap has no telemetry, no accounts and no cloud
|
|
530
|
+
component. History lives in a local SQLite file (`~/.llmtap/llmtap.db`, override with `--db` or `LLMTAP_HOME`), and
|
|
531
|
+
`llmtap clear` deletes it. Request headers, where your API keys live, are forwarded but not stored. Prompts and responses
|
|
532
|
+
*are* stored locally; use `--guard redact` if you don't want secrets in them, since redacted text is what gets stored.
|
|
533
|
+
</details>
|
|
534
|
+
|
|
535
|
+
<details>
|
|
536
|
+
<summary><b>How much overhead does it add?</b></summary>
|
|
537
|
+
|
|
538
|
+
One extra hop over loopback plus a regex scan and a SQLite insert per request, which is small next to LLM latency.
|
|
539
|
+
Streaming responses are relayed chunk by chunk as they arrive, never buffered, and llmtap records time-to-first-token for you.
|
|
540
|
+
</details>
|
|
541
|
+
|
|
542
|
+
<details>
|
|
543
|
+
<summary><b>Does streaming work?</b></summary>
|
|
544
|
+
|
|
545
|
+
Yes. Anthropic Messages streaming, OpenAI Chat Completions streaming and the OpenAI Responses API streaming events are all
|
|
546
|
+
relayed live and parsed for usage. For OpenAI chat streams, usage is only sent when you pass
|
|
547
|
+
`stream_options={"include_usage": True}`. Without it, llmtap estimates tokens and marks the numbers as *estimated*.
|
|
548
|
+
</details>
|
|
549
|
+
|
|
550
|
+
<details>
|
|
551
|
+
<summary><b>Is this an HTTPS-intercepting (MITM) proxy?</b></summary>
|
|
552
|
+
|
|
553
|
+
No. You don't install certificates and it doesn't use `HTTP_PROXY`. Your app talks plain HTTP to `127.0.0.1`
|
|
554
|
+
by base URL, and llmtap makes a normal, verified HTTPS request upstream. Tools that call the provider from their own cloud
|
|
555
|
+
(rather than from your machine) can't be pointed at a local proxy.
|
|
556
|
+
</details>
|
|
557
|
+
|
|
558
|
+
<details>
|
|
559
|
+
<summary><b>Which models are priced?</b></summary>
|
|
560
|
+
|
|
561
|
+
Claude (3.x, 4.x, 4.5, 5 families), GPT-4o / 4.1 / 5 families, o1 / o3 / o4-mini, OpenAI embeddings, Gemini via OpenRouter, and
|
|
562
|
+
common Llama / Mixtral / DeepSeek names. Local models (Ollama) are $0. Run `llmtap prices` to see the table, and
|
|
563
|
+
add or override prices under `[pricing_overrides]` in `llmtap.toml`. Unknown models are recorded with cost $0.
|
|
564
|
+
</details>
|
|
565
|
+
|
|
566
|
+
<details>
|
|
567
|
+
<summary><b>Can I expose it on my network?</b></summary>
|
|
568
|
+
|
|
569
|
+
It binds to `127.0.0.1` by default. The dashboard has no authentication and shows full prompts, so only use
|
|
570
|
+
`--host 0.0.0.0` on a network you trust. See [SECURITY.md](SECURITY.md).
|
|
571
|
+
</details>
|
|
572
|
+
|
|
573
|
+
<details>
|
|
574
|
+
<summary><b>Why does my SDK retry on a cassette miss?</b></summary>
|
|
575
|
+
|
|
576
|
+
The official Anthropic and OpenAI SDKs retry HTTP 409. Set `max_retries=0` on clients used in replayed tests so a
|
|
577
|
+
miss fails immediately. See [docs/cassettes.md](docs/cassettes.md).
|
|
578
|
+
</details>
|
|
579
|
+
|
|
580
|
+
---
|
|
581
|
+
|
|
582
|
+
## πΊοΈ Roadmap
|
|
583
|
+
|
|
584
|
+
Ideas we'd love help with. None of these exist yet:
|
|
585
|
+
|
|
586
|
+
- [ ] Gemini `countTokens` and embeddings endpoints (`generateContent` routing already works)
|
|
587
|
+
- [ ] Configurable cassette matching (ignore fields, fuzzy match on model)
|
|
588
|
+
- [ ] `llmtap diff` between two cassettes or two runs
|
|
589
|
+
- [ ] Per-tag budgets (e.g. cap a single feature or session)
|
|
590
|
+
- [ ] OpenTelemetry / OTLP export
|
|
591
|
+
- [ ] Homebrew formula and a single-file zipapp release
|
|
592
|
+
|
|
593
|
+
Have a better idea? [Open an issue](https://github.com/AshishAgrawal101/llmtap/issues/new/choose).
|
|
594
|
+
|
|
595
|
+
## π€ Contributing
|
|
596
|
+
|
|
597
|
+
PRs are very welcome, especially new price entries, guard detectors, upstream presets and integration snippets.
|
|
598
|
+
The one hard rule is **standard library only**. See [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
599
|
+
|
|
600
|
+
```bash
|
|
601
|
+
git clone https://github.com/AshishAgrawal101/llmtap && cd llmtap
|
|
602
|
+
pip install -e ".[dev]" && pytest -q
|
|
603
|
+
```
|
|
604
|
+
|
|
605
|
+
## π License
|
|
606
|
+
|
|
607
|
+
[MIT](LICENSE) Β© 2026 llmtap contributors
|
|
608
|
+
|
|
609
|
+
<div align="center"><sub>If llmtap saved you money or a 3am page, a β helps others find it.</sub></div>
|