llmtap-proxy 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/PKG-INFO +44 -12
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/README.md +621 -590
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/__init__.py +2 -2
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/budget.py +72 -19
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/cache.py +10 -2
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/cassette.py +214 -172
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/cli.py +188 -51
- llmtap_proxy-0.2.2/llmtap/config.py +187 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/dashboard.py +24 -3
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/demo.py +4 -1
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/doctor.py +28 -10
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/guard.py +44 -11
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/models.py +81 -75
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/pricing.py +85 -21
- llmtap_proxy-0.2.2/llmtap/providers.py +1162 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/proxy.py +207 -46
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/pytest_plugin.py +454 -444
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/report.py +176 -19
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/store.py +168 -17
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/term.py +27 -11
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/ui/index.html +42 -19
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap_proxy.egg-info/PKG-INFO +44 -12
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap_proxy.egg-info/SOURCES.txt +9 -1
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap_proxy.egg-info/requires.txt +1 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/pyproject.toml +42 -42
- llmtap_proxy-0.2.2/tests/test_bedrock.py +275 -0
- llmtap_proxy-0.2.2/tests/test_bedrock_accounting.py +534 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_cli_extra.py +37 -0
- llmtap_proxy-0.2.2/tests/test_cli_fixes.py +242 -0
- llmtap_proxy-0.2.2/tests/test_config.py +93 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_guard.py +95 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_hardening.py +9 -4
- llmtap_proxy-0.2.2/tests/test_http_robustness.py +233 -0
- llmtap_proxy-0.2.2/tests/test_module_regressions.py +245 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_pricing.py +94 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_providers.py +33 -0
- llmtap_proxy-0.2.2/tests/test_proxy_regressions.py +215 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_pytest_plugin.py +69 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_report.py +27 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_store.py +381 -352
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_term.py +226 -201
- llmtap_proxy-0.2.2/tests/test_waste_index.py +191 -0
- llmtap_proxy-0.2.0/llmtap/config.py +0 -112
- llmtap_proxy-0.2.0/llmtap/providers.py +0 -635
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/LICENSE +0 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/__main__.py +0 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap/mock.py +0 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap_proxy.egg-info/dependency_links.txt +0 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap_proxy.egg-info/entry_points.txt +0 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/llmtap_proxy.egg-info/top_level.txt +0 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/setup.cfg +0 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_budget.py +0 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_cache.py +0 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_cassette.py +0 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_cli.py +0 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_dashboard.py +0 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_e2e.py +0 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_gemini.py +0 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_mock.py +0 -0
- {llmtap_proxy-0.2.0 → llmtap_proxy-0.2.2}/tests/test_proxy.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: llmtap-proxy
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: The flight recorder & firewall for LLM apps. See, cache, replay, budget and secure every LLM call — zero code changes, zero dependencies.
|
|
5
5
|
License: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/AshishAgrawal101/llmtap
|
|
@@ -15,6 +15,7 @@ Description-Content-Type: text/markdown
|
|
|
15
15
|
License-File: LICENSE
|
|
16
16
|
Provides-Extra: dev
|
|
17
17
|
Requires-Dist: pytest>=8; extra == "dev"
|
|
18
|
+
Requires-Dist: botocore>=1.34; extra == "dev"
|
|
18
19
|
Dynamic: license-file
|
|
19
20
|
|
|
20
21
|
<div align="center">
|
|
@@ -55,7 +56,7 @@ every cent.
|
|
|
55
56
|
## ⚡ 30-second start
|
|
56
57
|
|
|
57
58
|
```bash
|
|
58
|
-
|
|
59
|
+
pip install llmtap-proxy # stdlib only, nothing else gets installed; the command is `llmtap` (pipx works too)
|
|
59
60
|
llmtap demo # fake LLM traffic + the live dashboard in your browser. No API key needed.
|
|
60
61
|
llmtap wrap -- python app.py # now do it for real with your own app
|
|
61
62
|
```
|
|
@@ -95,7 +96,7 @@ Every request and response, streaming included, in a local web UI. Search prompt
|
|
|
95
96
|
<td width="33%" valign="top">
|
|
96
97
|
|
|
97
98
|
**💸 Exact cost tracking**<br>
|
|
98
|
-
Per-call USD cost from reported token usage (including prompt-cache reads and writes) for Claude, GPT, o-series, Gemini
|
|
99
|
+
Per-call USD cost from reported token usage (including prompt-cache reads and writes) for Claude (direct or on AWS Bedrock), GPT, o-series, Gemini, Llama, DeepSeek and more.
|
|
99
100
|
|
|
100
101
|
</td>
|
|
101
102
|
<td width="33%" valign="top">
|
|
@@ -272,11 +273,36 @@ style = "openai" # or "anthropic"
|
|
|
272
273
|
```
|
|
273
274
|
|
|
274
275
|
…then use `http://127.0.0.1:4141/vllm/v1`. Without a prefix, requests are routed by shape: `/v1/messages` or an
|
|
275
|
-
`
|
|
276
|
+
`x-api-key` / `anthropic-*` header goes to Anthropic, Gemini-style requests (`:generateContent`, `x-goog-api-key`,
|
|
277
|
+
`?key=`) go to Gemini, Bedrock Runtime calls (SigV4-signed, or with a Bedrock API key) go to AWS Bedrock, and
|
|
278
|
+
everything else goes to OpenAI.
|
|
279
|
+
|
|
280
|
+
### AWS Bedrock
|
|
281
|
+
|
|
282
|
+
Point the AWS SDK's `endpoint_url` at llmtap's root URL. `InvokeModel`, `Converse` and their streaming versions are
|
|
283
|
+
recorded, priced, budget- and loop-checked, cached and written to cassettes like any other LLM call.
|
|
284
|
+
|
|
285
|
+
```python
|
|
286
|
+
import boto3
|
|
287
|
+
client = boto3.client("bedrock-runtime", region_name="us-east-1", endpoint_url="http://127.0.0.1:4141")
|
|
288
|
+
client.converse(modelId="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
|
289
|
+
messages=[{"role": "user", "content": [{"text": "hi"}]}])
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
- **SigV4-signed requests** are forwarded byte-for-byte, and llmtap reads the region from the signature. Use the root
|
|
293
|
+
URL, not `/bedrock/`: the signature covers the path.
|
|
294
|
+
- **Bedrock API keys** (`AWS_BEARER_TOKEN_BEDROCK`) work too. With a long-term key, start llmtap with `AWS_REGION` set.
|
|
295
|
+
- **`--guard redact`** can't change a signed body without breaking its signature, so a SigV4 request with a secret in it
|
|
296
|
+
is blocked instead. Requests made with an API key are redacted as usual.
|
|
297
|
+
- **Prices:** Claude models (including inference profiles, ARNs and GovCloud IDs) cost what Anthropic lists. Other
|
|
298
|
+
Bedrock models get the closest match in `llmtap prices` if there is one (Llama, Mistral), which may differ from
|
|
299
|
+
Bedrock's rates, or $0 (Nova, Titan). Set the right price under `[pricing_overrides]`.
|
|
300
|
+
- Not yet tried against the real AWS endpoint. With SigV4, llmtap forwards the `Host` value the SDK signed
|
|
301
|
+
(`127.0.0.1:4141`); if AWS rejects that, use a Bedrock API key and let us know in an issue.
|
|
276
302
|
|
|
277
303
|
### Tagging requests
|
|
278
304
|
|
|
279
|
-
Group traffic by run, user or feature with headers. llmtap strips every `X-LLMTap-*` header before forwarding, so the provider never sees them.
|
|
305
|
+
Group traffic by run, user or feature with headers. llmtap strips every `X-LLMTap-*` header before forwarding, so the provider never sees them (except one an AWS SigV4 signature covers, which has to reach AWS unchanged).
|
|
280
306
|
|
|
281
307
|
| Header | Becomes tag |
|
|
282
308
|
|---|---|
|
|
@@ -407,7 +433,8 @@ llmtap wrap --loop-threshold 5 --loop-window 60 -- python agent.py
|
|
|
407
433
|
- **`--loop-threshold N --loop-window W`**: once the *identical* request has been sent N times within W seconds, further copies are refused with HTTP **429**. This is the classic stuck-agent pattern, and it gets stopped in seconds instead of at 3am.
|
|
408
434
|
|
|
409
435
|
Refusals are returned as provider-shaped error JSON, so your SDK raises its normal error type with a clear message
|
|
410
|
-
(e.g. `daily budget $5.00 reached ($5.02 spent)` or
|
|
436
|
+
(e.g. `llmtap: daily budget of $5.00 reached ($5.02 spent today)` or
|
|
437
|
+
`llmtap: runaway loop detected — this exact request was sent 5 times in 60s`).
|
|
411
438
|
Cache and cassette hits cost nothing and don't count toward budgets.
|
|
412
439
|
|
|
413
440
|
## ⚙️ Configuration
|
|
@@ -445,7 +472,7 @@ Every field, env var and flag: **[docs/configuration.md](docs/configuration.md)*
|
|
|
445
472
|
| `llmtap demo` | Fake upstream + generated traffic + dashboard. Zero keys, zero cost. |
|
|
446
473
|
| `llmtap wrap [flags] -- <cmd…>` | Run a command through a fresh proxy; print a cost summary at the end |
|
|
447
474
|
| `llmtap serve [flags]` | Run the proxy + dashboard on `127.0.0.1:4141` |
|
|
448
|
-
| `llmtap env [--port N]` | Print `export …` lines: `eval "$(llmtap env)"` |
|
|
475
|
+
| `llmtap env [--port N] [--shell sh\|fish\|powershell]` | Print `export …` lines: `eval "$(llmtap env)"` |
|
|
449
476
|
| `llmtap tail` | Stream new requests to the terminal as they happen |
|
|
450
477
|
| `llmtap ls [-n 20] [-q text]` | List recent requests, optionally full-text filtered |
|
|
451
478
|
| `llmtap show <id> [--raw]` | Full request/response for one exchange (id prefixes work) |
|
|
@@ -458,8 +485,9 @@ Every field, env var and flag: **[docs/configuration.md](docs/configuration.md)*
|
|
|
458
485
|
| `llmtap clear [--yes]` | Delete recorded history |
|
|
459
486
|
| `llmtap version` | Print the version |
|
|
460
487
|
|
|
461
|
-
Common flags for `serve` / `wrap`: `--port --host --db --config --quiet --
|
|
462
|
-
--guard --pii --budget-daily --budget-session --loop-threshold
|
|
488
|
+
Common flags for `serve` / `wrap`: `--port --host --db --config --quiet --timeout --upstream NAME=URL[,STYLE] --cache
|
|
489
|
+
--no-cache --cache-ttl --cassette --mode --allow-miss --guard --pii --budget-daily --budget-session --loop-threshold
|
|
490
|
+
--loop-window`.
|
|
463
491
|
|
|
464
492
|
---
|
|
465
493
|
|
|
@@ -527,7 +555,7 @@ CI while production traces go to your hosted platform.
|
|
|
527
555
|
<summary><b>Does my data leave my machine?</b></summary>
|
|
528
556
|
|
|
529
557
|
Only the upstream call you were going to make anyway. llmtap has no telemetry, no accounts and no cloud
|
|
530
|
-
component. History lives in a local SQLite file (`~/.llmtap/llmtap.db`, override with `--db` or `LLMTAP_HOME`), and
|
|
558
|
+
component. History lives in a local SQLite file (`~/.llmtap/llmtap.db`, override with `--db`, `LLMTAP_DB_PATH` or `LLMTAP_HOME`), and
|
|
531
559
|
`llmtap clear` deletes it. Request headers, where your API keys live, are forwarded but not stored. Prompts and responses
|
|
532
560
|
*are* stored locally; use `--guard redact` if you don't want secrets in them, since redacted text is what gets stored.
|
|
533
561
|
</details>
|
|
@@ -558,9 +586,13 @@ by base URL, and llmtap makes a normal, verified HTTPS request upstream. Tools t
|
|
|
558
586
|
<details>
|
|
559
587
|
<summary><b>Which models are priced?</b></summary>
|
|
560
588
|
|
|
561
|
-
Claude (3.x, 4.x, 4.5, 5 families), GPT-4o / 4.1 / 5 families, o1 / o3 / o4-mini,
|
|
562
|
-
common Llama / Mixtral / DeepSeek names. Local models (Ollama) are $0. Run `llmtap prices` to see the table, and
|
|
589
|
+
Claude (3.x, 4.x, 4.5, 5 families, also as AWS Bedrock model IDs and inference profiles), GPT-4o / 4.1 / 5 families, o1 / o3 / o4-mini,
|
|
590
|
+
OpenAI embeddings, Gemini, and common Llama / Mixtral / DeepSeek names. Local models (Ollama) are $0. Run `llmtap prices` to see the table, and
|
|
563
591
|
add or override prices under `[pricing_overrides]` in `llmtap.toml`. Unknown models are recorded with cost $0.
|
|
592
|
+
|
|
593
|
+
Costs follow what the response reports: long-prompt rate cards (Claude Haiku 5.5 above 100K tokens, Gemini Pro above
|
|
594
|
+
200K, ...), Anthropic fast mode (2x), 1-hour cache writes (2x input), web searches ($10 per 1,000) and OpenAI's flex
|
|
595
|
+
tier (half price).
|
|
564
596
|
</details>
|
|
565
597
|
|
|
566
598
|
<details>
|