llmtap-proxy 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llmtap/__init__.py +2 -0
- llmtap/__main__.py +3 -0
- llmtap/budget.py +166 -0
- llmtap/cache.py +129 -0
- llmtap/cassette.py +172 -0
- llmtap/cli.py +1066 -0
- llmtap/config.py +112 -0
- llmtap/dashboard.py +445 -0
- llmtap/demo.py +244 -0
- llmtap/doctor.py +275 -0
- llmtap/guard.py +391 -0
- llmtap/mock.py +478 -0
- llmtap/models.py +75 -0
- llmtap/pricing.py +379 -0
- llmtap/providers.py +635 -0
- llmtap/proxy.py +1153 -0
- llmtap/pytest_plugin.py +444 -0
- llmtap/report.py +893 -0
- llmtap/store.py +319 -0
- llmtap/term.py +477 -0
- llmtap/ui/index.html +893 -0
- llmtap_proxy-0.2.0.dist-info/METADATA +609 -0
- llmtap_proxy-0.2.0.dist-info/RECORD +27 -0
- llmtap_proxy-0.2.0.dist-info/WHEEL +5 -0
- llmtap_proxy-0.2.0.dist-info/entry_points.txt +5 -0
- llmtap_proxy-0.2.0.dist-info/licenses/LICENSE +21 -0
- llmtap_proxy-0.2.0.dist-info/top_level.txt +1 -0
llmtap/cli.py
ADDED
|
@@ -0,0 +1,1066 @@
|
|
|
1
|
+
"""The ``llmtap`` command line.
|
|
2
|
+
|
|
3
|
+
Sibling modules are imported lazily inside each command so ``llmtap --help`` keeps working
|
|
4
|
+
even when one of them is broken or missing.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import csv
|
|
10
|
+
import io
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
import signal
|
|
15
|
+
import subprocess
|
|
16
|
+
import sys
|
|
17
|
+
import tempfile
|
|
18
|
+
import threading
|
|
19
|
+
import time
|
|
20
|
+
from typing import Any, Callable, Iterable
|
|
21
|
+
|
|
22
|
+
from llmtap import __version__
|
|
23
|
+
|
|
24
|
+
# --------------------------------------------------------------------------------------------
|
|
25
|
+
# helpers
|
|
26
|
+
|
|
27
|
+
_UNITS = {"s": 1, "sec": 1, "m": 60, "min": 60, "h": 3600, "hr": 3600, "d": 86400, "w": 7 * 86400}
|
|
28
|
+
_DUR_RE = re.compile(r"(\d+(?:\.\d+)?)\s*(w|d|hr|h|min|m|sec|s)")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def parse_duration(s: str) -> float:
|
|
32
|
+
"""'24h' -> 86400.0; '1h30m' -> 5400.0; bare number = seconds. Raises ValueError."""
|
|
33
|
+
t = str(s).strip().lower()
|
|
34
|
+
if not t:
|
|
35
|
+
raise ValueError("empty duration")
|
|
36
|
+
try:
|
|
37
|
+
return float(t)
|
|
38
|
+
except ValueError:
|
|
39
|
+
pass
|
|
40
|
+
pos, total = 0, 0.0
|
|
41
|
+
for m in _DUR_RE.finditer(t):
|
|
42
|
+
if t[pos:m.start()].strip():
|
|
43
|
+
raise ValueError(f"invalid duration: {s!r}")
|
|
44
|
+
total += float(m.group(1)) * _UNITS[m.group(2)]
|
|
45
|
+
pos = m.end()
|
|
46
|
+
if pos == 0 or t[pos:].strip():
|
|
47
|
+
raise ValueError(f"invalid duration: {s!r} (try 30m, 24h, 7d or all)")
|
|
48
|
+
return total
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def parse_since(s: str | None, now: float | None = None) -> float | None:
|
|
52
|
+
"""Turn '24h' / '7d' / '30m' / 'today' / '2026-01-31' / 'all' into an epoch cutoff (None = all time)."""
|
|
53
|
+
if s is None:
|
|
54
|
+
return None
|
|
55
|
+
t = str(s).strip().lower()
|
|
56
|
+
if t in ("", "all", "ever", "forever", "none", "0"):
|
|
57
|
+
return None
|
|
58
|
+
now = time.time() if now is None else now
|
|
59
|
+
if t == "today":
|
|
60
|
+
lt = time.localtime(now)
|
|
61
|
+
return time.mktime((lt.tm_year, lt.tm_mon, lt.tm_mday, 0, 0, 0, 0, 0, -1))
|
|
62
|
+
if t == "yesterday":
|
|
63
|
+
lt = time.localtime(now)
|
|
64
|
+
return time.mktime((lt.tm_year, lt.tm_mon, lt.tm_mday - 1, 0, 0, 0, 0, 0, -1))
|
|
65
|
+
if re.fullmatch(r"\d{4}-\d{2}-\d{2}([t ]\d{2}:\d{2}(:\d{2})?)?", t):
|
|
66
|
+
from datetime import datetime
|
|
67
|
+
return datetime.fromisoformat(t.upper().replace(" ", "T")).timestamp()
|
|
68
|
+
return now - parse_duration(t)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _since_arg(s: str) -> float | None:
|
|
72
|
+
try:
|
|
73
|
+
return parse_since(s)
|
|
74
|
+
except ValueError as e:
|
|
75
|
+
raise argparse.ArgumentTypeError(str(e)) from None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _err(msg: str) -> None:
|
|
79
|
+
from llmtap import term
|
|
80
|
+
print(term.paint("llmtap: ", "bold", "red", color=term.color_enabled(sys.stderr)) + msg, file=sys.stderr)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _out(text: str = "") -> None:
|
|
84
|
+
print(text, flush=True)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _color_out() -> bool:
|
|
88
|
+
from llmtap import term
|
|
89
|
+
return term.color_enabled(sys.stdout)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
# --------------------------------------------------------------------------------------------
|
|
93
|
+
# argument parsing
|
|
94
|
+
|
|
95
|
+
S = argparse.SUPPRESS
|
|
96
|
+
|
|
97
|
+
# CLI dest -> Config field
|
|
98
|
+
_OVERRIDE_MAP: dict[str, str] = {
|
|
99
|
+
"host": "host", "port": "port", "db": "db_path", "quiet": "quiet",
|
|
100
|
+
"cache": "cache", "cache_ttl": "cache_ttl_s",
|
|
101
|
+
"cassette": "cassette", "mode": "mode", "allow_miss": "allow_miss",
|
|
102
|
+
"guard": "guard", "pii": "guard_pii",
|
|
103
|
+
"budget_daily": "budget_daily_usd", "budget_session": "budget_session_usd",
|
|
104
|
+
"loop_threshold": "loop_threshold", "loop_window": "loop_window_s",
|
|
105
|
+
"timeout": "upstream_timeout_s",
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _db_parent() -> argparse.ArgumentParser:
|
|
110
|
+
p = argparse.ArgumentParser(add_help=False)
|
|
111
|
+
g = p.add_argument_group("storage")
|
|
112
|
+
g.add_argument("--db", metavar="PATH", default=S, help="SQLite database path (default ~/.llmtap/llmtap.db)")
|
|
113
|
+
g.add_argument("--config", metavar="FILE", default=S, help="config file (default ./llmtap.toml)")
|
|
114
|
+
return p
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _proxy_parent() -> argparse.ArgumentParser:
|
|
118
|
+
p = argparse.ArgumentParser(add_help=False)
|
|
119
|
+
g = p.add_argument_group("proxy")
|
|
120
|
+
g.add_argument("--host", default=S, help="bind address (default 127.0.0.1)")
|
|
121
|
+
g.add_argument("--port", type=int, default=S, help="listen port (default 4141; 0 = any free port)")
|
|
122
|
+
g.add_argument("--quiet", "-q", action="store_const", const=True, default=S, help="no per-request log lines")
|
|
123
|
+
g.add_argument("--timeout", type=float, default=S, metavar="SECS", help="upstream timeout (default 600)")
|
|
124
|
+
g.add_argument("--upstream", action="append", default=S, metavar="NAME=URL[,STYLE]",
|
|
125
|
+
help="add/override an upstream, e.g. ollama=http://localhost:11434 (route via /<name>/...)")
|
|
126
|
+
g = p.add_argument_group("cache & cassettes")
|
|
127
|
+
g.add_argument("--cache", action="store_const", const=True, default=S, help="serve identical requests from a local cache")
|
|
128
|
+
g.add_argument("--no-cache", dest="cache", action="store_const", const=False, default=S, help="disable the cache")
|
|
129
|
+
g.add_argument("--cache-ttl", type=int, default=S, metavar="SECS", help="cache entry lifetime (0 = forever)")
|
|
130
|
+
g.add_argument("--cassette", default=S, metavar="FILE", help="record/replay a .jsonl cassette (VCR for LLMs)")
|
|
131
|
+
g.add_argument("--mode", choices=["record", "replay", "auto"], default=S, help="cassette mode (default auto)")
|
|
132
|
+
g.add_argument("--allow-miss", action="store_const", const=True, default=S,
|
|
133
|
+
help="replay mode: forward unrecorded requests upstream instead of failing")
|
|
134
|
+
g = p.add_argument_group("guard & budgets")
|
|
135
|
+
g.add_argument("--guard", choices=["off", "warn", "redact", "block"], default=S, help="secret firewall mode (default warn)")
|
|
136
|
+
g.add_argument("--pii", action="store_const", const=True, default=S, help="also detect emails, phones, credit cards")
|
|
137
|
+
g.add_argument("--budget-daily", type=float, default=S, metavar="USD", help="refuse calls once today's spend reaches this")
|
|
138
|
+
g.add_argument("--budget-session", type=float, default=S, metavar="USD", help="refuse calls once this run's spend reaches this")
|
|
139
|
+
g.add_argument("--loop-threshold", type=int, default=S, metavar="N", help="block an identical request seen N times...")
|
|
140
|
+
g.add_argument("--loop-window", type=int, default=S, metavar="SECS", help="...within this window (default 60)")
|
|
141
|
+
return p
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
145
|
+
"""Build the full argparse tree."""
|
|
146
|
+
db = _db_parent()
|
|
147
|
+
px = _proxy_parent()
|
|
148
|
+
parser = argparse.ArgumentParser(
|
|
149
|
+
prog="llmtap",
|
|
150
|
+
description="The flight recorder & firewall for LLM apps: record, price, cache, replay, guard and budget every LLM call.",
|
|
151
|
+
epilog="Quick start: llmtap wrap -- python my_agent.py | llmtap serve then eval \"$(llmtap env)\"",
|
|
152
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
153
|
+
)
|
|
154
|
+
parser.add_argument("-V", "--version", action="version", version=f"llmtap {__version__}")
|
|
155
|
+
parser.add_argument("--db", metavar="PATH", default=S, help=argparse.SUPPRESS)
|
|
156
|
+
parser.add_argument("--config", metavar="FILE", default=S, help=argparse.SUPPRESS)
|
|
157
|
+
sub = parser.add_subparsers(dest="command", metavar="<command>")
|
|
158
|
+
|
|
159
|
+
def add(name: str, help: str, parents: list[argparse.ArgumentParser] | None = None, **kw: Any) -> argparse.ArgumentParser:
|
|
160
|
+
sp = sub.add_parser(name, help=help, description=help, parents=parents or [], **kw)
|
|
161
|
+
return sp
|
|
162
|
+
|
|
163
|
+
add("serve", "run the proxy + live dashboard", [db, px])
|
|
164
|
+
|
|
165
|
+
sp = add("wrap", "run a command with its LLM traffic routed through llmtap", [db, px],
|
|
166
|
+
usage="llmtap wrap [options] -- <command> [args...]")
|
|
167
|
+
sp.add_argument("cmd", nargs=argparse.REMAINDER, help="command to run (after --)")
|
|
168
|
+
|
|
169
|
+
sp = add("env", "print shell exports pointing SDKs at llmtap: eval \"$(llmtap env)\"")
|
|
170
|
+
sp.add_argument("--host", default=S)
|
|
171
|
+
sp.add_argument("--port", type=int, default=S)
|
|
172
|
+
sp.add_argument("--config", default=S, help=argparse.SUPPRESS)
|
|
173
|
+
sp.add_argument("--shell", choices=["sh", "fish", "powershell"], default="sh", help="output syntax (default sh)")
|
|
174
|
+
|
|
175
|
+
sp = add("stats", "spend / token / latency report", [db])
|
|
176
|
+
sp.add_argument("--since", type=_since_arg, default=None, metavar="WHEN", help="30m, 24h, 7d, today, 2026-01-31 or all (default all)")
|
|
177
|
+
sp.add_argument("--by", default=None, metavar="KEY", help="group by model|provider|source|day|hour|tag:<name> (default model)")
|
|
178
|
+
sp.add_argument("--json", action="store_true", help="machine-readable output")
|
|
179
|
+
|
|
180
|
+
sp = add("ls", "list recent exchanges", [db])
|
|
181
|
+
sp.add_argument("-n", "--limit", type=int, default=20)
|
|
182
|
+
sp.add_argument("-q", "--query", default=None, help="substring search over bodies/model/path")
|
|
183
|
+
sp.add_argument("--since", type=_since_arg, default=None, metavar="WHEN")
|
|
184
|
+
sp.add_argument("--provider", default=None)
|
|
185
|
+
sp.add_argument("--model", default=None)
|
|
186
|
+
sp.add_argument("--source", default=None, help="upstream|cache|cassette|blocked|budget|loop|error")
|
|
187
|
+
sp.add_argument("--json", action="store_true", help="JSON lines of summaries")
|
|
188
|
+
|
|
189
|
+
sp = add("show", "show one exchange (id or unique prefix)", [db])
|
|
190
|
+
sp.add_argument("id")
|
|
191
|
+
sp.add_argument("--raw", action="store_true", help="dump the full exchange as JSON")
|
|
192
|
+
|
|
193
|
+
sp = add("tail", "follow new exchanges as they are recorded", [db])
|
|
194
|
+
sp.add_argument("-n", "--lines", type=int, default=10, help="show this many recent exchanges first")
|
|
195
|
+
sp.add_argument("--interval", type=float, default=0.5, help=argparse.SUPPRESS)
|
|
196
|
+
sp.add_argument("--once", action="store_true", help=argparse.SUPPRESS)
|
|
197
|
+
|
|
198
|
+
sp = add("export", "export exchanges as JSONL or CSV", [db])
|
|
199
|
+
sp.add_argument("--format", "-f", choices=["jsonl", "csv"], default="jsonl")
|
|
200
|
+
sp.add_argument("-o", "--output", default="-", help="output file (default stdout)")
|
|
201
|
+
sp.add_argument("--since", type=_since_arg, default=None, metavar="WHEN")
|
|
202
|
+
sp.add_argument("-n", "--limit", type=int, default=0, help="max rows (default all)")
|
|
203
|
+
|
|
204
|
+
sp = add("prices", "show the built-in price table (USD per 1M tokens)")
|
|
205
|
+
sp.add_argument("filter", nargs="?", default=None)
|
|
206
|
+
sp.add_argument("--json", action="store_true")
|
|
207
|
+
|
|
208
|
+
sp = add("scan", "scan files/stdin for secrets (exit 1 if any found)")
|
|
209
|
+
sp.add_argument("files", nargs="*", default=[], metavar="FILE", help="files to scan, or - for stdin (default stdin)")
|
|
210
|
+
sp.add_argument("--pii", action="store_true", help="also detect emails, phones, credit cards")
|
|
211
|
+
sp.add_argument("--json", action="store_true")
|
|
212
|
+
|
|
213
|
+
sp = add("clear", "delete all recorded exchanges", [db])
|
|
214
|
+
sp.add_argument("--yes", "-y", action="store_true", help="don't ask for confirmation")
|
|
215
|
+
|
|
216
|
+
add("version", "print version")
|
|
217
|
+
|
|
218
|
+
sp = add("demo", "spin up a fake LLM + traffic and open the dashboard (no API keys needed)")
|
|
219
|
+
sp.add_argument("--port", type=int, default=4141)
|
|
220
|
+
sp.add_argument("--host", default="127.0.0.1")
|
|
221
|
+
sp.add_argument("--no-open", action="store_true", help="don't open a browser")
|
|
222
|
+
sp.add_argument("--count", type=int, default=40, help="number of demo requests (default 40)")
|
|
223
|
+
sp.add_argument("--delay", type=float, default=0.3, help="seconds between demo requests")
|
|
224
|
+
sp.add_argument("--exit", action="store_true", help="exit (with a summary) once the demo traffic is done")
|
|
225
|
+
sp.add_argument("--quiet", "-q", action="store_true", help="no per-request log lines")
|
|
226
|
+
|
|
227
|
+
sp = add("report", "shareable cost & safety report: spend, waste finder, guard findings (HTML / md / json)", [db])
|
|
228
|
+
sp.add_argument("--since", type=_since_arg, default="7d", metavar="WHEN",
|
|
229
|
+
help="30m, 24h, 7d, today, 2026-01-31 or all (default 7d)")
|
|
230
|
+
sp.add_argument("-o", "--output", default=None, metavar="FILE",
|
|
231
|
+
help="output file (default llmtap-report.html; '-' = stdout)")
|
|
232
|
+
sp.add_argument("--format", "-f", choices=["html", "md", "json"], default=None,
|
|
233
|
+
help="html (default; or inferred from -o extension), md or json")
|
|
234
|
+
sp.add_argument("--open", action="store_true", help="open the HTML report in your browser")
|
|
235
|
+
|
|
236
|
+
sp = add("doctor", "check your setup (port, db, env vars, SDK base URLs) and print fix hints", [db])
|
|
237
|
+
sp.add_argument("--online", action="store_true", help="also check that the upstream API hosts are reachable (5s timeout)")
|
|
238
|
+
sp.add_argument("--host", default=S)
|
|
239
|
+
sp.add_argument("--port", type=int, default=S)
|
|
240
|
+
return parser
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def config_overrides(args: argparse.Namespace) -> dict[str, Any]:
|
|
244
|
+
"""Map parsed CLI flags onto Config field overrides (unset flags are omitted)."""
|
|
245
|
+
ov: dict[str, Any] = {}
|
|
246
|
+
for dest, field_name in _OVERRIDE_MAP.items():
|
|
247
|
+
if hasattr(args, dest):
|
|
248
|
+
v = getattr(args, dest)
|
|
249
|
+
if v is not None:
|
|
250
|
+
ov[field_name] = v
|
|
251
|
+
ups = getattr(args, "upstream", None)
|
|
252
|
+
if ups:
|
|
253
|
+
table: dict[str, dict[str, str]] = {}
|
|
254
|
+
for spec in ups:
|
|
255
|
+
name, sep, rest = spec.partition("=")
|
|
256
|
+
if not sep or not name or not rest:
|
|
257
|
+
raise ValueError(f"--upstream expects NAME=URL[,STYLE], got {spec!r}")
|
|
258
|
+
url, _, style = rest.partition(",")
|
|
259
|
+
table[name.strip()] = {"base": url.strip().rstrip("/"), "style": (style.strip() or "openai")}
|
|
260
|
+
ov["upstreams"] = table
|
|
261
|
+
return ov
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def make_config(args: argparse.Namespace, **extra: Any) -> Any:
|
|
265
|
+
"""load_config(--config, overrides-from-flags + extra)."""
|
|
266
|
+
from llmtap.config import load_config
|
|
267
|
+
ov = config_overrides(args)
|
|
268
|
+
for k, v in extra.items():
|
|
269
|
+
ov.setdefault(k, v)
|
|
270
|
+
return load_config(getattr(args, "config", None), ov)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _open_store(args: argparse.Namespace) -> Any:
|
|
274
|
+
from llmtap.store import Store
|
|
275
|
+
cfg = make_config(args)
|
|
276
|
+
return Store(cfg.resolved_db_path())
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
# --------------------------------------------------------------------------------------------
|
|
280
|
+
# request-body helpers for show / ls
|
|
281
|
+
|
|
282
|
+
def _content_text(c: Any) -> str:
|
|
283
|
+
if c is None:
|
|
284
|
+
return ""
|
|
285
|
+
if isinstance(c, str):
|
|
286
|
+
return c
|
|
287
|
+
if isinstance(c, dict):
|
|
288
|
+
t = c.get("type")
|
|
289
|
+
if "text" in c and isinstance(c["text"], str):
|
|
290
|
+
return c["text"]
|
|
291
|
+
if t == "tool_use" or t == "function_call":
|
|
292
|
+
args = c.get("input", c.get("arguments", ""))
|
|
293
|
+
return f"[tool call {c.get('name', '?')}: {json.dumps(args)[:200] if not isinstance(args, str) else args[:200]}]"
|
|
294
|
+
if t == "tool_result":
|
|
295
|
+
return f"[tool result: {_content_text(c.get('content'))}]"
|
|
296
|
+
if t in ("image", "image_url", "input_image"):
|
|
297
|
+
return "[image]"
|
|
298
|
+
if "content" in c:
|
|
299
|
+
return _content_text(c["content"])
|
|
300
|
+
return f"[{t or 'object'}]"
|
|
301
|
+
if isinstance(c, list):
|
|
302
|
+
return "\n".join(x for x in (_content_text(i) for i in c) if x)
|
|
303
|
+
return str(c)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def conversation(req: Any) -> list[tuple[str, str]]:
|
|
307
|
+
"""Extract [(role, text)] from an anthropic/openai/responses request body."""
|
|
308
|
+
out: list[tuple[str, str]] = []
|
|
309
|
+
if not isinstance(req, dict):
|
|
310
|
+
return out
|
|
311
|
+
if req.get("system"):
|
|
312
|
+
out.append(("system", _content_text(req["system"])))
|
|
313
|
+
if req.get("instructions"):
|
|
314
|
+
out.append(("system", _content_text(req["instructions"])))
|
|
315
|
+
for m in req.get("messages") or []:
|
|
316
|
+
if isinstance(m, dict):
|
|
317
|
+
text = _content_text(m.get("content"))
|
|
318
|
+
if m.get("tool_calls"):
|
|
319
|
+
calls = ", ".join(str((tc.get("function") or {}).get("name", "?")) for tc in m["tool_calls"] if isinstance(tc, dict))
|
|
320
|
+
text = (text + "\n" if text else "") + f"[tool calls: {calls}]"
|
|
321
|
+
out.append((str(m.get("role", "?")), text))
|
|
322
|
+
inp = req.get("input")
|
|
323
|
+
if isinstance(inp, str):
|
|
324
|
+
out.append(("user", inp))
|
|
325
|
+
elif isinstance(inp, list):
|
|
326
|
+
for m in inp:
|
|
327
|
+
if isinstance(m, dict) and "role" in m:
|
|
328
|
+
out.append((str(m["role"]), _content_text(m.get("content"))))
|
|
329
|
+
elif isinstance(m, str):
|
|
330
|
+
out.append(("user", m))
|
|
331
|
+
else:
|
|
332
|
+
out.append(("input", _content_text(m)))
|
|
333
|
+
if not out and isinstance(req.get("prompt"), str):
|
|
334
|
+
out.append(("user", req["prompt"]))
|
|
335
|
+
return out
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _last_user(req_body: str) -> str:
|
|
339
|
+
try:
|
|
340
|
+
req = json.loads(req_body) if req_body else None
|
|
341
|
+
except ValueError:
|
|
342
|
+
return req_body[:200]
|
|
343
|
+
conv = conversation(req)
|
|
344
|
+
for role, text in reversed(conv):
|
|
345
|
+
if role == "user":
|
|
346
|
+
return text
|
|
347
|
+
return conv[-1][1] if conv else ""
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def _one_line(s: str) -> str:
|
|
351
|
+
return " ".join(str(s or "").split())
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _when(ts: float) -> str:
|
|
355
|
+
now = time.localtime()
|
|
356
|
+
lt = time.localtime(ts)
|
|
357
|
+
if (lt.tm_year, lt.tm_yday) == (now.tm_year, now.tm_yday):
|
|
358
|
+
return time.strftime("%H:%M:%S", lt)
|
|
359
|
+
return time.strftime("%m-%d %H:%M", lt)
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
# --------------------------------------------------------------------------------------------
|
|
363
|
+
# commands
|
|
364
|
+
|
|
365
|
+
def cmd_version(args: argparse.Namespace) -> int:
|
|
366
|
+
_out(f"llmtap {__version__}")
|
|
367
|
+
return 0
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def cmd_env(args: argparse.Namespace) -> int:
|
|
371
|
+
cfg = make_config(args)
|
|
372
|
+
host = cfg.host if cfg.host not in ("0.0.0.0", "::", "") else "127.0.0.1"
|
|
373
|
+
base = f"http://{host}:{cfg.port}"
|
|
374
|
+
pairs = env_vars(base)
|
|
375
|
+
for k, v in pairs.items():
|
|
376
|
+
if args.shell == "fish":
|
|
377
|
+
_out(f"set -gx {k} {v}")
|
|
378
|
+
elif args.shell == "powershell":
|
|
379
|
+
_out(f'$env:{k} = "{v}"')
|
|
380
|
+
else:
|
|
381
|
+
_out(f"export {k}={v}")
|
|
382
|
+
return 0
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def env_vars(base: str) -> dict[str, str]:
|
|
386
|
+
"""The environment variables that point SDKs at a llmtap proxy at ``base``."""
|
|
387
|
+
base = base.rstrip("/")
|
|
388
|
+
return {
|
|
389
|
+
"ANTHROPIC_BASE_URL": base,
|
|
390
|
+
"OPENAI_BASE_URL": f"{base}/v1",
|
|
391
|
+
"OPENAI_API_BASE": f"{base}/v1",
|
|
392
|
+
"LLMTAP_URL": base,
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _wait_forever(stop: threading.Event | None = None) -> None:
|
|
397
|
+
while not (stop and stop.is_set()):
|
|
398
|
+
time.sleep(0.25)
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def cmd_serve(args: argparse.Namespace) -> int:
|
|
402
|
+
from llmtap import term
|
|
403
|
+
from llmtap.proxy import LLMTap
|
|
404
|
+
cfg = make_config(args)
|
|
405
|
+
try:
|
|
406
|
+
tap = LLMTap(cfg)
|
|
407
|
+
tap.start_background()
|
|
408
|
+
except OSError as e:
|
|
409
|
+
_err(f"cannot listen on {cfg.host}:{cfg.port}: {e.strerror or e}")
|
|
410
|
+
return 1
|
|
411
|
+
base = getattr(tap, "base_url", None) or cfg.base_url()
|
|
412
|
+
print(term.banner(cfg, base, color=term.color_enabled(sys.stdout)), flush=True)
|
|
413
|
+
try:
|
|
414
|
+
_wait_forever()
|
|
415
|
+
except KeyboardInterrupt:
|
|
416
|
+
pass
|
|
417
|
+
finally:
|
|
418
|
+
print(term.paint("\nshutting down…", "dim", color=term.color_enabled(sys.stderr)), file=sys.stderr, flush=True)
|
|
419
|
+
_shutdown(tap)
|
|
420
|
+
return 0
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def _shutdown(tap: Any) -> None:
|
|
424
|
+
try:
|
|
425
|
+
tap.shutdown()
|
|
426
|
+
except Exception:
|
|
427
|
+
pass
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def cmd_wrap(args: argparse.Namespace) -> int:
|
|
431
|
+
from llmtap import term
|
|
432
|
+
cmd = list(getattr(args, "cmd", None) or [])
|
|
433
|
+
if cmd and cmd[0] == "--":
|
|
434
|
+
cmd = cmd[1:]
|
|
435
|
+
if not cmd:
|
|
436
|
+
_err("nothing to run. usage: llmtap wrap [options] -- <command> [args...]")
|
|
437
|
+
return 2
|
|
438
|
+
from llmtap.proxy import LLMTap
|
|
439
|
+
cfg = make_config(args, port=0)
|
|
440
|
+
try:
|
|
441
|
+
tap = LLMTap(cfg)
|
|
442
|
+
tap.start_background()
|
|
443
|
+
except OSError as e:
|
|
444
|
+
_err(f"cannot start proxy: {e}")
|
|
445
|
+
return 1
|
|
446
|
+
base = getattr(tap, "base_url", None) or f"http://{cfg.host}:{getattr(tap, 'port', cfg.port)}"
|
|
447
|
+
env = dict(os.environ)
|
|
448
|
+
env.update(env_vars(base))
|
|
449
|
+
# make sure HTTP(S)_PROXY settings don't hijack traffic to the local proxy
|
|
450
|
+
for k in ("NO_PROXY", "no_proxy"):
|
|
451
|
+
cur = env.get(k, "")
|
|
452
|
+
extra = "127.0.0.1,localhost"
|
|
453
|
+
env[k] = f"{cur},{extra}" if cur else extra
|
|
454
|
+
color = term.color_enabled(sys.stderr)
|
|
455
|
+
if not cfg.quiet:
|
|
456
|
+
print(term.paint("◉ llmtap", "bold", "bright_magenta", color=color)
|
|
457
|
+
+ term.paint(f" recording → {base} · dashboard {base}/_llmtap/", "dim", color=color),
|
|
458
|
+
file=sys.stderr, flush=True)
|
|
459
|
+
start = time.time() - 0.001
|
|
460
|
+
try:
|
|
461
|
+
proc = subprocess.Popen(cmd, env=env)
|
|
462
|
+
except FileNotFoundError:
|
|
463
|
+
_shutdown(tap)
|
|
464
|
+
_err(f"command not found: {cmd[0]}")
|
|
465
|
+
return 127
|
|
466
|
+
except PermissionError:
|
|
467
|
+
_shutdown(tap)
|
|
468
|
+
_err(f"permission denied: {cmd[0]}")
|
|
469
|
+
return 126
|
|
470
|
+
|
|
471
|
+
def forward(signum: int, _frame: Any) -> None:
|
|
472
|
+
# A terminal Ctrl-C already reaches the child (same process group). Forward only if
|
|
473
|
+
# it is still running shortly after, which covers `kill -INT <llmtap pid>`.
|
|
474
|
+
def later() -> None:
|
|
475
|
+
time.sleep(0.3)
|
|
476
|
+
if proc.poll() is None:
|
|
477
|
+
try:
|
|
478
|
+
proc.send_signal(signum)
|
|
479
|
+
except OSError:
|
|
480
|
+
pass
|
|
481
|
+
threading.Thread(target=later, daemon=True).start()
|
|
482
|
+
|
|
483
|
+
old: dict[int, Any] = {}
|
|
484
|
+
in_main = threading.current_thread() is threading.main_thread()
|
|
485
|
+
if in_main:
|
|
486
|
+
for sig in (signal.SIGINT, signal.SIGTERM):
|
|
487
|
+
try:
|
|
488
|
+
old[sig] = signal.signal(sig, forward)
|
|
489
|
+
except (ValueError, OSError):
|
|
490
|
+
pass
|
|
491
|
+
try:
|
|
492
|
+
while True:
|
|
493
|
+
try:
|
|
494
|
+
rc = proc.wait()
|
|
495
|
+
break
|
|
496
|
+
except KeyboardInterrupt: # pragma: no cover - only if handler couldn't be installed
|
|
497
|
+
continue
|
|
498
|
+
finally:
|
|
499
|
+
for sig, h in old.items():
|
|
500
|
+
try:
|
|
501
|
+
signal.signal(sig, h)
|
|
502
|
+
except (ValueError, OSError):
|
|
503
|
+
pass
|
|
504
|
+
time.sleep(0.15) # let in-flight records land
|
|
505
|
+
try:
|
|
506
|
+
stats = tap.store.stats(since=start, group_by="model")
|
|
507
|
+
try:
|
|
508
|
+
spent = float(tap.budget.session_spent())
|
|
509
|
+
except Exception:
|
|
510
|
+
spent = float(stats.get("cost_usd", 0) or 0)
|
|
511
|
+
print(term.summary(stats, spent, color=color), file=sys.stderr, flush=True)
|
|
512
|
+
except Exception as e:
|
|
513
|
+
_err(f"could not compute summary: {e}")
|
|
514
|
+
finally:
|
|
515
|
+
_shutdown(tap)
|
|
516
|
+
if rc < 0:
|
|
517
|
+
return 128 + (-rc)
|
|
518
|
+
return rc
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def cmd_stats(args: argparse.Namespace) -> int:
|
|
522
|
+
from llmtap import term
|
|
523
|
+
store = _open_store(args)
|
|
524
|
+
group = args.by or "model"
|
|
525
|
+
try:
|
|
526
|
+
st = store.stats(since=args.since, group_by=group)
|
|
527
|
+
waste_usd, waste_n = _potential_savings(store, args.since)
|
|
528
|
+
except ValueError as e:
|
|
529
|
+
_err(str(e))
|
|
530
|
+
return 2
|
|
531
|
+
finally:
|
|
532
|
+
store.close()
|
|
533
|
+
if args.json:
|
|
534
|
+
st = dict(st, potential_savings_usd=waste_usd, duplicate_requests=waste_n)
|
|
535
|
+
_out(json.dumps(st, indent=2, default=str))
|
|
536
|
+
return 0
|
|
537
|
+
color = _color_out()
|
|
538
|
+
label = "all time" if args.since is None else "since " + time.strftime("%Y-%m-%d %H:%M", time.localtime(args.since))
|
|
539
|
+
totals = {k: v for k, v in st.items() if k != "groups"}
|
|
540
|
+
_out(term.summary(totals, float(st.get("cost_usd", 0) or 0), color=color, title=f"llmtap stats · {label}"))
|
|
541
|
+
if int(st.get("requests", 0) or 0) > 0:
|
|
542
|
+
_out(_savings_line(waste_usd, waste_n, color))
|
|
543
|
+
groups = st.get("groups") or []
|
|
544
|
+
if groups:
|
|
545
|
+
total_cost = float(st.get("cost_usd", 0) or 0)
|
|
546
|
+
rows = []
|
|
547
|
+
for g in groups:
|
|
548
|
+
c = float(g.get("cost_usd", 0) or 0)
|
|
549
|
+
share = f"{100 * c / total_cost:.0f}%" if total_cost > 0 else "-"
|
|
550
|
+
rows.append([
|
|
551
|
+
str(g.get("key") if g.get("key") not in (None, "") else "-"),
|
|
552
|
+
fmt_int(g.get("requests")),
|
|
553
|
+
term.fmt_tokens(g.get("input_tokens", 0)),
|
|
554
|
+
term.fmt_tokens(g.get("output_tokens", 0)),
|
|
555
|
+
term.fmt_usd(c),
|
|
556
|
+
share,
|
|
557
|
+
])
|
|
558
|
+
_out(term.table(rows, [group.upper(), "REQS", "IN", "OUT", "COST", "SHARE"], color=color))
|
|
559
|
+
return 0
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def _potential_savings(store: Any, since: float | None) -> tuple[float, int]:
|
|
563
|
+
"""Waste-finder numbers for ``stats``; (0, 0) if the report module is unavailable."""
|
|
564
|
+
try:
|
|
565
|
+
from llmtap.report import potential_savings
|
|
566
|
+
return potential_savings(store, since)
|
|
567
|
+
except ImportError: # pragma: no cover
|
|
568
|
+
return 0.0, 0
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def _savings_line(usd: float, n: int, color: bool) -> str:
|
|
572
|
+
from llmtap import term
|
|
573
|
+
amount = term.paint(term.fmt_usd(usd), "bold", "green" if usd > 0 else "dim", color=color)
|
|
574
|
+
return f"Potential savings from caching: {amount} ({n} duplicate request{'s' if n != 1 else ''})"
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def fmt_int(n: Any) -> str:
|
|
578
|
+
return f"{int(n or 0):,}"
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def cmd_ls(args: argparse.Namespace) -> int:
|
|
582
|
+
from llmtap import term
|
|
583
|
+
store = _open_store(args)
|
|
584
|
+
try:
|
|
585
|
+
items = store.list(limit=args.limit, q=args.query, since=args.since, provider=args.provider,
|
|
586
|
+
model=args.model, source=args.source)
|
|
587
|
+
finally:
|
|
588
|
+
store.close()
|
|
589
|
+
if args.json:
|
|
590
|
+
for ex in items:
|
|
591
|
+
d = ex.to_dict()
|
|
592
|
+
d.pop("request_body", None)
|
|
593
|
+
d.pop("response_body", None)
|
|
594
|
+
_out(json.dumps(d, default=str))
|
|
595
|
+
return 0
|
|
596
|
+
if not items:
|
|
597
|
+
_out("no exchanges recorded yet — try: llmtap wrap -- <your command> or llmtap demo")
|
|
598
|
+
return 0
|
|
599
|
+
color = _color_out()
|
|
600
|
+
rows = []
|
|
601
|
+
for ex in items:
|
|
602
|
+
u = ex.usage
|
|
603
|
+
tin = u.input_tokens + u.cache_read_tokens + u.cache_write_tokens
|
|
604
|
+
st = str(ex.status or "---")
|
|
605
|
+
rows.append([
|
|
606
|
+
term.paint(ex.id[:8], "gray", color=color),
|
|
607
|
+
_when(ex.ts),
|
|
608
|
+
term.source_badge(ex.source, color=color).strip(),
|
|
609
|
+
f"{ex.provider}/{ex.model or '-'}",
|
|
610
|
+
term.paint(st, term.status_color(ex.status), color=color),
|
|
611
|
+
f"{term.fmt_tokens(tin)}→{term.fmt_tokens(u.output_tokens)}",
|
|
612
|
+
term.fmt_usd(ex.cost_usd),
|
|
613
|
+
term.fmt_ms(ex.latency_ms),
|
|
614
|
+
term.truncate(_one_line(_last_user(ex.request_body)), 40),
|
|
615
|
+
])
|
|
616
|
+
_out(term.table(rows, ["ID", "TIME", "SRC", "MODEL", "ST", "TOKENS", "COST", "LATENCY", "PROMPT"],
|
|
617
|
+
align=["<", "<", "<", "<", "<", ">", ">", ">", "<"], color=color))
|
|
618
|
+
return 0
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
def cmd_show(args: argparse.Namespace) -> int:
|
|
622
|
+
from llmtap import term
|
|
623
|
+
store = _open_store(args)
|
|
624
|
+
try:
|
|
625
|
+
ex = store.get(args.id)
|
|
626
|
+
finally:
|
|
627
|
+
store.close()
|
|
628
|
+
if ex is None:
|
|
629
|
+
_err(f"no exchange matching {args.id!r}")
|
|
630
|
+
return 1
|
|
631
|
+
if args.raw:
|
|
632
|
+
_out(json.dumps(ex.to_dict(), indent=2, default=str))
|
|
633
|
+
return 0
|
|
634
|
+
color = _color_out()
|
|
635
|
+
p = lambda s, *st: term.paint(s, *st, color=color) # noqa: E731
|
|
636
|
+
u = ex.usage
|
|
637
|
+
meta = [
|
|
638
|
+
("id", ex.id),
|
|
639
|
+
("time", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ex.ts))),
|
|
640
|
+
("route", f"{ex.method} {ex.path} ({ex.provider}, {ex.style} style)"),
|
|
641
|
+
("model", ex.model or "-"),
|
|
642
|
+
("status", p(str(ex.status), "bold", term.status_color(ex.status))),
|
|
643
|
+
("source", term.source_badge(ex.source, color=color).strip()),
|
|
644
|
+
("tokens", f"{fmt_int(u.input_tokens)} in · {fmt_int(u.output_tokens)} out"
|
|
645
|
+
+ (f" · {fmt_int(u.cache_read_tokens)} cache-read" if u.cache_read_tokens else "")
|
|
646
|
+
+ (f" · {fmt_int(u.cache_write_tokens)} cache-write" if u.cache_write_tokens else "")
|
|
647
|
+
+ (" (estimated)" if u.estimated else "")),
|
|
648
|
+
("cost", term.fmt_usd(ex.cost_usd) + (f" (saved {term.fmt_usd(ex.saved_usd)})" if ex.saved_usd else "")),
|
|
649
|
+
("latency", term.fmt_ms(ex.latency_ms) + (f" · ttft {term.fmt_ms(ex.ttft_ms)}" if ex.ttft_ms is not None else "")
|
|
650
|
+
+ (" · streamed" if ex.streamed else "")),
|
|
651
|
+
]
|
|
652
|
+
if ex.tags:
|
|
653
|
+
meta.append(("tags", ", ".join(f"{k}={v}" for k, v in ex.tags.items())))
|
|
654
|
+
if ex.error:
|
|
655
|
+
meta.append(("error", p(ex.error, "red")))
|
|
656
|
+
lines = [f"{p(k.ljust(8), 'dim')} {v}" for k, v in meta]
|
|
657
|
+
_out(term.box(lines, title=p(f"exchange {ex.id[:8]}", "bold"), color=color))
|
|
658
|
+
if ex.guard_findings:
|
|
659
|
+
_out(p(f"\n⚠ {len(ex.guard_findings)} guard finding(s)", "bold", "red"))
|
|
660
|
+
for f in ex.guard_findings:
|
|
661
|
+
_out(f" {p(str(f.get('kind', '?')), 'red')} {f.get('path', '')} {f.get('preview', '')}")
|
|
662
|
+
try:
|
|
663
|
+
req = json.loads(ex.request_body) if ex.request_body else None
|
|
664
|
+
except ValueError:
|
|
665
|
+
req = None
|
|
666
|
+
conv = conversation(req)
|
|
667
|
+
width = term.term_width()
|
|
668
|
+
role_style = {"system": ("magenta",), "user": ("cyan",), "assistant": ("green",), "tool": ("yellow",)}
|
|
669
|
+
if conv or ex.output_text:
|
|
670
|
+
_out("")
|
|
671
|
+
for role, text in conv:
|
|
672
|
+
_out(p(f"▍{role}", "bold", *role_style.get(role, ("white",))))
|
|
673
|
+
_out(_indent(_clip(text), width))
|
|
674
|
+
if ex.output_text:
|
|
675
|
+
_out(p("▍assistant (response)", "bold", "green"))
|
|
676
|
+
_out(_indent(_clip(ex.output_text), width))
|
|
677
|
+
elif not conv and ex.request_body:
|
|
678
|
+
_out(p("▍request body", "bold"))
|
|
679
|
+
_out(_indent(_clip(ex.request_body), width))
|
|
680
|
+
if ex.response_body and not ex.output_text:
|
|
681
|
+
_out(p("▍response body", "bold"))
|
|
682
|
+
_out(_indent(_clip(ex.response_body), width))
|
|
683
|
+
return 0
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
def _clip(text: str, limit: int = 4000) -> str:
|
|
687
|
+
text = str(text or "")
|
|
688
|
+
return text if len(text) <= limit else text[:limit] + f"\n… ({len(text) - limit} more chars; use --raw)"
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
def _indent(text: str, width: int) -> str:
|
|
692
|
+
return "\n".join(" " + line for line in text.splitlines() or [""])
|
|
693
|
+
|
|
694
|
+
|
|
695
|
+
def cmd_tail(args: argparse.Namespace) -> int:
|
|
696
|
+
from llmtap import term
|
|
697
|
+
store = _open_store(args)
|
|
698
|
+
seen: set[str] = set()
|
|
699
|
+
try:
|
|
700
|
+
recent = store.list(limit=max(args.lines, 0)) if args.lines > 0 else []
|
|
701
|
+
for ex in reversed(recent):
|
|
702
|
+
seen.add(ex.id)
|
|
703
|
+
term.log_exchange(ex, stream=sys.stdout)
|
|
704
|
+
last_ts = max([ex.ts for ex in recent], default=time.time() - 1)
|
|
705
|
+
if args.once:
|
|
706
|
+
return 0
|
|
707
|
+
print(term.paint("… waiting for new exchanges (Ctrl-C to stop)", "dim", color=_color_out()), flush=True)
|
|
708
|
+
# remember everything already in the window so only new rows print
|
|
709
|
+
for ex in store.list(limit=500, since=last_ts - 5):
|
|
710
|
+
seen.add(ex.id)
|
|
711
|
+
while True:
|
|
712
|
+
time.sleep(args.interval)
|
|
713
|
+
new = [ex for ex in store.list(limit=200, since=last_ts - 5) if ex.id not in seen]
|
|
714
|
+
for ex in sorted(new, key=lambda e: e.ts):
|
|
715
|
+
seen.add(ex.id)
|
|
716
|
+
last_ts = max(last_ts, ex.ts)
|
|
717
|
+
term.log_exchange(ex, stream=sys.stdout)
|
|
718
|
+
except KeyboardInterrupt:
|
|
719
|
+
return 0
|
|
720
|
+
finally:
|
|
721
|
+
store.close()
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
CSV_COLUMNS = [
|
|
725
|
+
"id", "ts", "time", "provider", "style", "method", "path", "model", "status", "source", "streamed",
|
|
726
|
+
"latency_ms", "ttft_ms", "input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens",
|
|
727
|
+
"usage_estimated", "cost_usd", "saved_usd", "tags", "guard_findings", "error", "output_text", "request_body",
|
|
728
|
+
]
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
def flatten(ex: Any) -> dict[str, Any]:
|
|
732
|
+
"""One flat CSV-friendly row for an Exchange."""
|
|
733
|
+
u = ex.usage
|
|
734
|
+
return {
|
|
735
|
+
"id": ex.id, "ts": f"{ex.ts:.3f}",
|
|
736
|
+
"time": time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(ex.ts)),
|
|
737
|
+
"provider": ex.provider, "style": ex.style, "method": ex.method, "path": ex.path,
|
|
738
|
+
"model": ex.model or "", "status": ex.status, "source": ex.source, "streamed": int(bool(ex.streamed)),
|
|
739
|
+
"latency_ms": round(ex.latency_ms or 0, 1), "ttft_ms": "" if ex.ttft_ms is None else round(ex.ttft_ms, 1),
|
|
740
|
+
"input_tokens": u.input_tokens, "output_tokens": u.output_tokens,
|
|
741
|
+
"cache_read_tokens": u.cache_read_tokens, "cache_write_tokens": u.cache_write_tokens,
|
|
742
|
+
"usage_estimated": int(bool(u.estimated)),
|
|
743
|
+
"cost_usd": f"{ex.cost_usd:.6f}", "saved_usd": f"{ex.saved_usd:.6f}",
|
|
744
|
+
"tags": json.dumps(ex.tags) if ex.tags else "",
|
|
745
|
+
"guard_findings": len(ex.guard_findings or []),
|
|
746
|
+
"error": ex.error or "", "output_text": ex.output_text, "request_body": ex.request_body,
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
def _iter_all(store: Any, since: float | None, limit: int) -> Iterable[Any]:
|
|
751
|
+
page, offset, n = 500, 0, 0
|
|
752
|
+
while True:
|
|
753
|
+
want = page if not limit else min(page, limit - n)
|
|
754
|
+
if want <= 0:
|
|
755
|
+
return
|
|
756
|
+
batch = store.list(limit=want, offset=offset, since=since)
|
|
757
|
+
for ex in batch:
|
|
758
|
+
yield ex
|
|
759
|
+
n += len(batch)
|
|
760
|
+
offset += len(batch)
|
|
761
|
+
if len(batch) < want:
|
|
762
|
+
return
|
|
763
|
+
|
|
764
|
+
|
|
765
|
+
def cmd_export(args: argparse.Namespace) -> int:
|
|
766
|
+
store = _open_store(args)
|
|
767
|
+
try:
|
|
768
|
+
items = list(_iter_all(store, args.since, args.limit))
|
|
769
|
+
finally:
|
|
770
|
+
store.close()
|
|
771
|
+
items.reverse() # chronological
|
|
772
|
+
if args.output in ("-", ""):
|
|
773
|
+
fh: Any = sys.stdout
|
|
774
|
+
close = False
|
|
775
|
+
else:
|
|
776
|
+
fh = open(args.output, "w", newline="", encoding="utf-8")
|
|
777
|
+
close = True
|
|
778
|
+
try:
|
|
779
|
+
if args.format == "csv":
|
|
780
|
+
w = csv.DictWriter(fh, fieldnames=CSV_COLUMNS, extrasaction="ignore")
|
|
781
|
+
w.writeheader()
|
|
782
|
+
for ex in items:
|
|
783
|
+
w.writerow(flatten(ex))
|
|
784
|
+
else:
|
|
785
|
+
for ex in items:
|
|
786
|
+
fh.write(json.dumps(ex.to_dict(), default=str) + "\n")
|
|
787
|
+
fh.flush()
|
|
788
|
+
finally:
|
|
789
|
+
if close:
|
|
790
|
+
fh.close()
|
|
791
|
+
if close:
|
|
792
|
+
print(f"exported {len(items)} exchange(s) to {args.output}", file=sys.stderr)
|
|
793
|
+
return 0
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def cmd_prices(args: argparse.Namespace) -> int:
|
|
797
|
+
from llmtap import term
|
|
798
|
+
from llmtap.pricing import PRICES
|
|
799
|
+
flt = (args.filter or "").lower()
|
|
800
|
+
items = sorted((k, v) for k, v in PRICES.items() if flt in k.lower())
|
|
801
|
+
if args.json:
|
|
802
|
+
_out(json.dumps(dict(items), indent=2))
|
|
803
|
+
return 0
|
|
804
|
+
if not items:
|
|
805
|
+
_out(f"no models match {args.filter!r}")
|
|
806
|
+
return 1
|
|
807
|
+
|
|
808
|
+
def money(v: Any) -> str:
|
|
809
|
+
if v is None:
|
|
810
|
+
return "-"
|
|
811
|
+
v = float(v)
|
|
812
|
+
return "free" if v == 0 else (f"${v:,.2f}" if v >= 0.1 else f"${v:,.3f}")
|
|
813
|
+
|
|
814
|
+
rows = [[k, money(v.get("input")), money(v.get("output")), money(v.get("cache_read")), money(v.get("cache_write"))]
|
|
815
|
+
for k, v in items]
|
|
816
|
+
color = _color_out()
|
|
817
|
+
_out(term.table(rows, ["MODEL", "INPUT", "OUTPUT", "CACHE READ", "CACHE WRITE"],
|
|
818
|
+
align=["<", ">", ">", ">", ">"], color=color))
|
|
819
|
+
_out(term.paint(f"{len(rows)} model(s) · USD per 1M tokens · override in llmtap.toml [pricing]", "dim", color=color))
|
|
820
|
+
return 0
|
|
821
|
+
|
|
822
|
+
|
|
823
|
+
def _line_col(text: str, offset: int) -> tuple[int, int]:
|
|
824
|
+
line = text.count("\n", 0, offset) + 1
|
|
825
|
+
col = offset - (text.rfind("\n", 0, offset) + 1) + 1
|
|
826
|
+
return line, col
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
def cmd_scan(args: argparse.Namespace) -> int:
|
|
830
|
+
from llmtap import guard, term
|
|
831
|
+
kinds: dict[str, str] = dict(getattr(guard, "KINDS", {}) or {})
|
|
832
|
+
files = args.files or ["-"]
|
|
833
|
+
color = _color_out()
|
|
834
|
+
p = lambda s, *st: term.paint(s, *st, color=color) # noqa: E731
|
|
835
|
+
total = 0
|
|
836
|
+
report: list[dict[str, Any]] = []
|
|
837
|
+
for name in files:
|
|
838
|
+
try:
|
|
839
|
+
if name == "-":
|
|
840
|
+
text = sys.stdin.read()
|
|
841
|
+
label = "<stdin>"
|
|
842
|
+
else:
|
|
843
|
+
with open(name, encoding="utf-8", errors="replace") as fh:
|
|
844
|
+
text = fh.read()
|
|
845
|
+
label = name
|
|
846
|
+
except OSError as e:
|
|
847
|
+
_err(f"{name}: {e.strerror or e}")
|
|
848
|
+
return 2
|
|
849
|
+
findings = guard.scan_text(text, pii=args.pii)
|
|
850
|
+
total += len(findings)
|
|
851
|
+
for f in findings:
|
|
852
|
+
ln, col = _line_col(text, f.start)
|
|
853
|
+
d = f.to_dict() if hasattr(f, "to_dict") else dict(vars(f))
|
|
854
|
+
d.update({"file": label, "line": ln, "col": col})
|
|
855
|
+
report.append(d)
|
|
856
|
+
if not args.json:
|
|
857
|
+
desc = kinds.get(f.kind, "")
|
|
858
|
+
_out(f"{p(f'{label}:{ln}:{col}', 'bold')} {p(f.kind, 'bold', 'red')} {f.preview}"
|
|
859
|
+
+ (p(f" — {desc}", "dim") if desc else ""))
|
|
860
|
+
if args.json:
|
|
861
|
+
_out(json.dumps(report, indent=2))
|
|
862
|
+
elif total:
|
|
863
|
+
_out(p(f"\n⚠ {total} potential secret{'s' if total != 1 else ''} found", "bold", "red"))
|
|
864
|
+
else:
|
|
865
|
+
_out(p("✓ no secrets found", "green"))
|
|
866
|
+
return 1 if total else 0
|
|
867
|
+
|
|
868
|
+
|
|
869
|
+
def cmd_clear(args: argparse.Namespace) -> int:
|
|
870
|
+
store = _open_store(args)
|
|
871
|
+
try:
|
|
872
|
+
n = store.count()
|
|
873
|
+
if not args.yes:
|
|
874
|
+
if not sys.stdin.isatty():
|
|
875
|
+
_err(f"refusing to delete {n} exchange(s) without --yes")
|
|
876
|
+
return 1
|
|
877
|
+
ans = input(f"Delete all {n} recorded exchange(s)? [y/N] ").strip().lower()
|
|
878
|
+
if ans not in ("y", "yes"):
|
|
879
|
+
_out("aborted")
|
|
880
|
+
return 1
|
|
881
|
+
store.clear()
|
|
882
|
+
finally:
|
|
883
|
+
store.close()
|
|
884
|
+
_out(f"cleared {n} exchange(s)")
|
|
885
|
+
return 0
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
def cmd_demo(args: argparse.Namespace) -> int:
|
|
889
|
+
from llmtap import term
|
|
890
|
+
from llmtap.config import load_config
|
|
891
|
+
from llmtap.demo import generate_traffic
|
|
892
|
+
from llmtap.mock import MockUpstream
|
|
893
|
+
from llmtap.proxy import LLMTap
|
|
894
|
+
|
|
895
|
+
mock = MockUpstream(token_delay_s=0.01).start()
|
|
896
|
+
tmp = tempfile.mkdtemp(prefix="llmtap-demo-")
|
|
897
|
+
cfg = load_config(getattr(args, "config", None), {
|
|
898
|
+
"host": args.host, "port": args.port, "quiet": args.quiet,
|
|
899
|
+
"db_path": os.path.join(tmp, "demo.db"),
|
|
900
|
+
"cache": True, "guard": "redact", "loop_threshold": 4, "loop_window_s": 60,
|
|
901
|
+
"upstreams": {"anthropic": {"base": mock.base_url, "style": "anthropic"},
|
|
902
|
+
"openai": {"base": mock.base_url, "style": "openai"}},
|
|
903
|
+
})
|
|
904
|
+
try:
|
|
905
|
+
tap = LLMTap(cfg)
|
|
906
|
+
tap.start_background()
|
|
907
|
+
except OSError as e:
|
|
908
|
+
mock.stop()
|
|
909
|
+
_err(f"cannot listen on {cfg.host}:{cfg.port}: {e.strerror or e} (try --port 0)")
|
|
910
|
+
return 1
|
|
911
|
+
base = getattr(tap, "base_url", None) or cfg.base_url()
|
|
912
|
+
print(term.banner(cfg, base, color=term.color_enabled(sys.stdout)), flush=True)
|
|
913
|
+
print(term.paint(f"demo: fake LLM upstream at {mock.base_url} · no API keys, no network, no cost", "dim",
|
|
914
|
+
color=term.color_enabled(sys.stdout)), flush=True)
|
|
915
|
+
stop = threading.Event()
|
|
916
|
+
done = threading.Event()
|
|
917
|
+
|
|
918
|
+
def run() -> None:
|
|
919
|
+
try:
|
|
920
|
+
generate_traffic(base, count=args.count, delay_s=args.delay, stop=stop)
|
|
921
|
+
except Exception as e: # pragma: no cover
|
|
922
|
+
_err(f"demo traffic failed: {e}")
|
|
923
|
+
finally:
|
|
924
|
+
done.set()
|
|
925
|
+
|
|
926
|
+
start = time.time() - 0.001
|
|
927
|
+
threading.Thread(target=run, daemon=True, name="llmtap-demo").start()
|
|
928
|
+
if not args.no_open:
|
|
929
|
+
try:
|
|
930
|
+
import webbrowser
|
|
931
|
+
webbrowser.open(base + "/_llmtap/")
|
|
932
|
+
except Exception:
|
|
933
|
+
pass
|
|
934
|
+
try:
|
|
935
|
+
if args.exit:
|
|
936
|
+
done.wait()
|
|
937
|
+
else:
|
|
938
|
+
_wait_forever()
|
|
939
|
+
except KeyboardInterrupt:
|
|
940
|
+
pass
|
|
941
|
+
finally:
|
|
942
|
+
stop.set()
|
|
943
|
+
time.sleep(0.2) # let in-flight records land
|
|
944
|
+
try:
|
|
945
|
+
st = tap.store.stats(since=start, group_by="model")
|
|
946
|
+
print(term.summary(st, float(tap.budget.session_spent()), color=term.color_enabled(sys.stderr),
|
|
947
|
+
title="llmtap demo summary"), file=sys.stderr, flush=True)
|
|
948
|
+
except Exception:
|
|
949
|
+
pass
|
|
950
|
+
_shutdown(tap)
|
|
951
|
+
mock.stop()
|
|
952
|
+
return 0
|
|
953
|
+
|
|
954
|
+
|
|
955
|
+
_REPORT_EXT = {".md": "md", ".markdown": "md", ".json": "json", ".html": "html", ".htm": "html"}
|
|
956
|
+
|
|
957
|
+
|
|
958
|
+
def cmd_report(args: argparse.Namespace) -> int:
|
|
959
|
+
"""`llmtap report`: build the cost & safety report and write/print it."""
|
|
960
|
+
from llmtap import report, term
|
|
961
|
+
out = args.output
|
|
962
|
+
fmt = args.format
|
|
963
|
+
if fmt is None:
|
|
964
|
+
ext = os.path.splitext(out)[1].lower() if out and out != "-" else ""
|
|
965
|
+
fmt = _REPORT_EXT.get(ext, "html")
|
|
966
|
+
store = _open_store(args)
|
|
967
|
+
try:
|
|
968
|
+
data = report.build_report(store, since=args.since)
|
|
969
|
+
finally:
|
|
970
|
+
store.close()
|
|
971
|
+
text = {"html": report.render_html, "md": report.render_markdown, "json": report.render_json}[fmt](data)
|
|
972
|
+
if out is None:
|
|
973
|
+
out = "llmtap-report.html" if fmt == "html" else "-"
|
|
974
|
+
if out == "-":
|
|
975
|
+
sys.stdout.write(text if text.endswith("\n") else text + "\n")
|
|
976
|
+
sys.stdout.flush()
|
|
977
|
+
return 0
|
|
978
|
+
with open(out, "w", encoding="utf-8") as f:
|
|
979
|
+
f.write(text)
|
|
980
|
+
t, w = data["totals"], data["waste"]
|
|
981
|
+
color = _color_out()
|
|
982
|
+
path = os.path.abspath(out)
|
|
983
|
+
_out(f"{term.paint('✓', 'bold', 'green', color=color)} wrote {out} "
|
|
984
|
+
+ term.paint(f"({fmt}, {t['requests']:,} requests, {term.fmt_usd(t['cost_usd'])} spend", "dim", color=color)
|
|
985
|
+
+ (term.paint(f", {term.fmt_usd(w['savings_usd'])} avoidable waste", "yellow", color=color) if w["savings_usd"] > 0 else "")
|
|
986
|
+
+ term.paint(")", "dim", color=color))
|
|
987
|
+
if fmt == "html":
|
|
988
|
+
_out(term.paint(" file://" + path, "dim", color=color))
|
|
989
|
+
if args.open:
|
|
990
|
+
try:
|
|
991
|
+
import webbrowser
|
|
992
|
+
webbrowser.open("file://" + path)
|
|
993
|
+
except Exception:
|
|
994
|
+
pass
|
|
995
|
+
return 0
|
|
996
|
+
|
|
997
|
+
|
|
998
|
+
def cmd_doctor(args: argparse.Namespace) -> int:
|
|
999
|
+
"""`llmtap doctor`: print checks with fix hints; exit 1 on a hard failure."""
|
|
1000
|
+
from llmtap import doctor
|
|
1001
|
+
cfg = make_config(args)
|
|
1002
|
+
checks = doctor.run_checks(cfg, getattr(args, "config", None), online=args.online)
|
|
1003
|
+
_out(doctor.render(checks, color=_color_out()))
|
|
1004
|
+
return doctor.exit_code(checks)
|
|
1005
|
+
|
|
1006
|
+
|
|
1007
|
+
COMMANDS: dict[str, Callable[[argparse.Namespace], int]] = {
|
|
1008
|
+
"serve": cmd_serve, "wrap": cmd_wrap, "env": cmd_env, "stats": cmd_stats, "ls": cmd_ls,
|
|
1009
|
+
"show": cmd_show, "tail": cmd_tail, "export": cmd_export, "prices": cmd_prices, "scan": cmd_scan,
|
|
1010
|
+
"clear": cmd_clear, "version": cmd_version, "demo": cmd_demo,
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
# Added later; kept apart so the core table (and tests that pin it) stay stable.
|
|
1014
|
+
EXTRA_COMMANDS: dict[str, Callable[[argparse.Namespace], int]] = {
|
|
1015
|
+
"report": cmd_report, "doctor": cmd_doctor,
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
|
|
1019
|
+
def _split_wrap(argv: list[str]) -> tuple[list[str], list[str] | None]:
|
|
1020
|
+
"""For ``wrap``, everything after the first ``--`` is the command (kept verbatim)."""
|
|
1021
|
+
if "wrap" in argv and "--" in argv and argv.index("wrap") < argv.index("--"):
|
|
1022
|
+
i = argv.index("--")
|
|
1023
|
+
return argv[:i], argv[i + 1:]
|
|
1024
|
+
return argv, None
|
|
1025
|
+
|
|
1026
|
+
|
|
1027
|
+
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
1028
|
+
"""Parse argv (default sys.argv[1:]) into a Namespace; ``wrap`` gets ``.cmd``."""
|
|
1029
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
1030
|
+
head, tail = _split_wrap(argv)
|
|
1031
|
+
args = build_parser().parse_args(head)
|
|
1032
|
+
if tail is not None:
|
|
1033
|
+
args.cmd = tail
|
|
1034
|
+
return args
|
|
1035
|
+
|
|
1036
|
+
|
|
1037
|
+
def main(argv: list[str] | None = None) -> int:
|
|
1038
|
+
"""Entry point for the ``llmtap`` console script."""
|
|
1039
|
+
try:
|
|
1040
|
+
args = parse_args(argv)
|
|
1041
|
+
except SystemExit as e: # argparse --help / errors
|
|
1042
|
+
return int(e.code or 0) if isinstance(e.code, int) or e.code is None else 2
|
|
1043
|
+
if not getattr(args, "command", None):
|
|
1044
|
+
build_parser().print_help()
|
|
1045
|
+
return 0
|
|
1046
|
+
fn = COMMANDS.get(args.command) or EXTRA_COMMANDS[args.command]
|
|
1047
|
+
try:
|
|
1048
|
+
return int(fn(args) or 0)
|
|
1049
|
+
except KeyboardInterrupt:
|
|
1050
|
+
return 130
|
|
1051
|
+
except BrokenPipeError:
|
|
1052
|
+
try:
|
|
1053
|
+
sys.stdout = open(os.devnull, "w")
|
|
1054
|
+
except Exception:
|
|
1055
|
+
pass
|
|
1056
|
+
return 0
|
|
1057
|
+
except (ImportError, ModuleNotFoundError) as e:
|
|
1058
|
+
_err(f"`{args.command}` is unavailable: {e}")
|
|
1059
|
+
return 1
|
|
1060
|
+
except ValueError as e:
|
|
1061
|
+
_err(str(e))
|
|
1062
|
+
return 2
|
|
1063
|
+
|
|
1064
|
+
|
|
1065
|
+
if __name__ == "__main__": # pragma: no cover
|
|
1066
|
+
raise SystemExit(main())
|