llmtap-proxy 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
llmtap/config.py ADDED
@@ -0,0 +1,112 @@
1
+ """Configuration: defaults <- llmtap.toml <- LLMTAP_* env vars <- CLI flags."""
2
+ from __future__ import annotations
3
+
4
+ import os
5
+ import tomllib
6
+ from dataclasses import dataclass, field, fields
7
+ from pathlib import Path
8
+ from typing import Any
9
+
10
+
11
+ def llmtap_home() -> Path:
12
+ return Path(os.environ.get("LLMTAP_HOME", Path.home() / ".llmtap"))
13
+
14
+
15
+ DEFAULT_UPSTREAMS: dict[str, dict[str, str]] = {
16
+ # name -> {"base": url, "style": "anthropic"|"openai"}
17
+ "anthropic": {"base": "https://api.anthropic.com", "style": "anthropic"},
18
+ "openai": {"base": "https://api.openai.com", "style": "openai"},
19
+ "openrouter": {"base": "https://openrouter.ai/api", "style": "openai"},
20
+ "groq": {"base": "https://api.groq.com/openai", "style": "openai"},
21
+ "together": {"base": "https://api.together.xyz", "style": "openai"},
22
+ "mistral": {"base": "https://api.mistral.ai", "style": "openai"},
23
+ "deepseek": {"base": "https://api.deepseek.com", "style": "openai"},
24
+ "ollama": {"base": "http://localhost:11434", "style": "openai"},
25
+ "gemini": {"base":"https://generativelanguage.googleapis.com","style":"gemini"}
26
+ }
27
+
28
+
29
+ @dataclass
30
+ class Config:
31
+ host: str = "127.0.0.1"
32
+ port: int = 4141
33
+ db_path: str = "" # "" -> llmtap_home()/llmtap.db ; ":memory:" allowed
34
+ quiet: bool = False # suppress per-request console lines
35
+ # caching
36
+ cache: bool = False
37
+ cache_ttl_s: int = 0 # 0 = forever
38
+ # cassettes (record/replay)
39
+ cassette: str = "" # path to .jsonl cassette; "" = disabled
40
+ mode: str = "auto" # "record" | "replay" | "auto"
41
+ allow_miss: bool = False # in replay mode, forward misses upstream instead of erroring
42
+ # secret guard
43
+ guard: str = "warn" # "off" | "warn" | "redact" | "block"
44
+ guard_pii: bool = False # also detect emails / phone numbers / credit cards
45
+ # budgets & loop detection
46
+ budget_daily_usd: float = 0.0 # 0 = unlimited
47
+ budget_session_usd: float = 0.0 # 0 = unlimited (per proxy process)
48
+ loop_threshold: int = 0 # 0 = off; block when identical request seen N times...
49
+ loop_window_s: int = 60 # ...within this many seconds
50
+ # upstreams & pricing
51
+ upstreams: dict[str, dict[str, str]] = field(default_factory=lambda: {k: dict(v) for k, v in DEFAULT_UPSTREAMS.items()})
52
+ pricing_overrides: dict[str, dict[str, float]] = field(default_factory=dict) # model -> {"input":..,"output":..,"cache_read":..,"cache_write":..} USD per 1M tokens
53
+ upstream_timeout_s: float = 600.0
54
+
55
+ def resolved_db_path(self) -> str:
56
+ if self.db_path:
57
+ return self.db_path
58
+ home = llmtap_home()
59
+ home.mkdir(parents=True, exist_ok=True)
60
+ return str(home / "llmtap.db")
61
+
62
+ def base_url(self) -> str:
63
+ return f"http://{self.host}:{self.port}"
64
+
65
+
66
+ _ENV_PREFIX = "LLMTAP_"
67
+
68
+
69
+ def _coerce(value: str, typ: Any) -> Any:
70
+ t = typ if isinstance(typ, type) else type(typ)
71
+ if t is bool:
72
+ return value.strip().lower() in ("1", "true", "yes", "on")
73
+ if t is int:
74
+ return int(value)
75
+ if t is float:
76
+ return float(value)
77
+ return value
78
+
79
+
80
+ def load_config(path: str | None = None, overrides: dict[str, Any] | None = None) -> Config:
81
+ """Build a Config. `overrides` (e.g. from CLI) win; keys with value None are ignored."""
82
+ cfg = Config()
83
+ candidates = [Path(path)] if path else [Path("llmtap.toml"), Path.home() / ".config" / "llmtap" / "config.toml"]
84
+ for p in candidates:
85
+ if p.is_file():
86
+ data = tomllib.loads(p.read_text())
87
+ _apply(cfg, data.get("llmtap", data))
88
+ break
89
+ for f in fields(Config):
90
+ env = os.environ.get(_ENV_PREFIX + f.name.upper())
91
+ if env is not None and f.name not in ("upstreams", "pricing_overrides"):
92
+ setattr(cfg, f.name, _coerce(env, getattr(cfg, f.name)))
93
+ if overrides:
94
+ _apply(cfg, {k: v for k, v in overrides.items() if v is not None})
95
+ return cfg
96
+
97
+
98
+ def _apply(cfg: Config, data: dict[str, Any]) -> None:
99
+ names = {f.name for f in fields(Config)}
100
+ for k, v in data.items():
101
+ k = k.replace("-", "_")
102
+ if k == "pricing":
103
+ k = "pricing_overrides"
104
+ if k not in names:
105
+ continue
106
+ if k == "upstreams":
107
+ for name, spec in v.items():
108
+ cfg.upstreams[name] = {"base": spec["base"].rstrip("/"), "style": spec.get("style", "openai")}
109
+ elif k == "pricing_overrides":
110
+ cfg.pricing_overrides.update(v)
111
+ else:
112
+ setattr(cfg, k, v)
llmtap/dashboard.py ADDED
@@ -0,0 +1,445 @@
1
+ """Live web dashboard + JSON API, served by the proxy under ``/_llmtap/``.
2
+
3
+ Everything here talks to the proxy through a tiny surface: ``tap.store``, ``tap.cache``,
4
+ ``tap.cassette``, ``tap.cfg``, ``tap.budget`` and ``tap.bus`` — plus the ``Responder`` protocol
5
+ (``start``/``write``/``end``), so it is independent from ``http.server``.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import ipaddress
10
+ import json
11
+ import queue
12
+ import time
13
+ from pathlib import Path
14
+ from typing import TYPE_CHECKING, Any, Mapping, Protocol
15
+ from urllib.parse import parse_qs, unquote, urlsplit
16
+
17
+ from . import __version__
18
+ from .models import Exchange
19
+
20
+ if TYPE_CHECKING: # pragma: no cover
21
+ from .proxy import LLMTap
22
+
23
+ PREFIX = "/_llmtap"
24
+ UI_PATH = Path(__file__).parent / "ui" / "index.html"
25
+ HEARTBEAT_S: float = 15.0 # SSE keep-alive interval (": ping")
26
+ PREVIEW_CHARS = 200
27
+ MAX_LIMIT = 1000
28
+
29
+
30
+ class Responder(Protocol):
31
+ def start(self, status: int, headers: dict[str, str]) -> None: ...
32
+ def write(self, chunk: bytes) -> None: ...
33
+ def end(self) -> None: ...
34
+
35
+
36
+ # --------------------------------------------------------------------------------------------
37
+ # summaries / previews
38
+ # --------------------------------------------------------------------------------------------
39
+
40
+ def _clip(text: str, n: int = PREVIEW_CHARS) -> str:
41
+ text = " ".join(str(text).split())
42
+ return text if len(text) <= n else text[: n - 1] + "…"
43
+
44
+
45
+ def _content_text(content: Any) -> str:
46
+ """Text of a message ``content``: a string, or a list of blocks (anthropic / openai / responses)."""
47
+ if content is None:
48
+ return ""
49
+ if isinstance(content, str):
50
+ return content
51
+ if isinstance(content, dict):
52
+ content = [content]
53
+ if not isinstance(content, list):
54
+ return ""
55
+ texts: list[str] = []
56
+ fallback: list[str] = []
57
+ for block in content:
58
+ if isinstance(block, str):
59
+ texts.append(block)
60
+ elif isinstance(block, dict):
61
+ btype = block.get("type")
62
+ if isinstance(block.get("text"), str) and btype in (None, "text", "input_text", "output_text"):
63
+ texts.append(block["text"])
64
+ elif btype == "tool_result":
65
+ fallback.append(_content_text(block.get("content")))
66
+ elif btype in ("image", "image_url", "input_image"):
67
+ fallback.append("[image]")
68
+ elif isinstance(block.get("text"), str):
69
+ texts.append(block["text"])
70
+ return "\n".join(t for t in texts if t) or "\n".join(t for t in fallback if t)
71
+
72
+
73
+ def _last_user_text(req: Any) -> str:
74
+ """Last user message text from an anthropic / openai chat / responses / embeddings / completions body."""
75
+ if not isinstance(req, dict):
76
+ return ""
77
+ msgs = req.get("messages")
78
+ if isinstance(msgs, list):
79
+ for m in reversed(msgs):
80
+ if isinstance(m, dict) and m.get("role") == "user":
81
+ text = _content_text(m.get("content"))
82
+ if text:
83
+ return text
84
+ for m in reversed(msgs): # no user text: fall back to any message text
85
+ if isinstance(m, dict):
86
+ text = _content_text(m.get("content"))
87
+ if text:
88
+ return text
89
+ inp = req.get("input")
90
+ if isinstance(inp, str):
91
+ return inp
92
+ if isinstance(inp, list):
93
+ for item in reversed(inp):
94
+ if isinstance(item, str):
95
+ return item
96
+ if isinstance(item, dict) and item.get("role", "user") == "user":
97
+ text = _content_text(item.get("content"))
98
+ if text:
99
+ return text
100
+ prompt = req.get("prompt")
101
+ if isinstance(prompt, str):
102
+ return prompt
103
+ if isinstance(prompt, list) and prompt and isinstance(prompt[-1], str):
104
+ return prompt[-1]
105
+ return ""
106
+
107
+
108
+ def request_preview(request_body: str) -> str:
109
+ """Preview of the last user message in a raw request body (<= 200 chars)."""
110
+ if not request_body:
111
+ return ""
112
+ try:
113
+ req = json.loads(request_body)
114
+ except (ValueError, TypeError):
115
+ return _clip(request_body)
116
+ if not isinstance(req, dict):
117
+ return _clip(request_body)
118
+ return _clip(_last_user_text(req))
119
+
120
+
121
+ def summary(ex: Exchange | Mapping[str, Any]) -> dict[str, Any]:
122
+ """Exchange dict minus request/response bodies, plus ``preview`` and ``output_preview``."""
123
+ d = ex.to_dict() if isinstance(ex, Exchange) else dict(ex)
124
+ req = d.pop("request_body", "") or ""
125
+ d.pop("response_body", None)
126
+ if "preview" not in d:
127
+ d["preview"] = request_preview(req)
128
+ if "output_preview" not in d:
129
+ out = d.get("output_text") or ""
130
+ d["output_preview"] = _clip(out or (d.get("error") or ""))
131
+ d.pop("output_text", None)
132
+ return d
133
+
134
+
135
+ exchange_summary = summary # alias
136
+
137
+
138
+ # --------------------------------------------------------------------------------------------
139
+ # response helpers
140
+ # --------------------------------------------------------------------------------------------
141
+
142
+ def _send(respond: Responder, status: int, body: bytes, ctype: str, extra: dict[str, str] | None = None) -> None:
143
+ headers = {
144
+ "content-type": ctype,
145
+ "content-length": str(len(body)),
146
+ "cache-control": "no-store",
147
+ "x-content-type-options": "nosniff",
148
+ }
149
+ if extra:
150
+ headers.update(extra)
151
+ respond.start(status, headers)
152
+ if body:
153
+ respond.write(body)
154
+ respond.end()
155
+
156
+
157
+ def _json(respond: Responder, obj: Any, status: int = 200) -> None:
158
+ body = json.dumps(obj, default=str, separators=(",", ":")).encode("utf-8")
159
+ _send(respond, status, body, "application/json; charset=utf-8")
160
+
161
+
162
+ def _error(respond: Responder, status: int, message: str) -> None:
163
+ _json(respond, {"error": message}, status)
164
+
165
+
166
+ def _first(qs: dict[str, list[str]], name: str) -> str | None:
167
+ vals = qs.get(name)
168
+ if not vals:
169
+ return None
170
+ v = vals[0].strip()
171
+ return v or None
172
+
173
+
174
+ def _int(qs: dict[str, list[str]], name: str, default: int, lo: int, hi: int) -> int:
175
+ raw = _first(qs, name)
176
+ if raw is None:
177
+ return default
178
+ try:
179
+ return max(lo, min(hi, int(float(raw))))
180
+ except ValueError:
181
+ return default
182
+
183
+
184
+ def _since(qs: dict[str, list[str]], default: float | None = None) -> float | None:
185
+ """``since`` is an epoch timestamp, or (if < 1e9) a number of seconds ago. ``all``/``0`` = no cutoff."""
186
+ raw = _first(qs, "since")
187
+ if raw is None:
188
+ return default
189
+ if raw.lower() in ("all", "0"):
190
+ return None
191
+ try:
192
+ v = float(raw)
193
+ except ValueError:
194
+ return default
195
+ if v <= 0:
196
+ return None
197
+ return v if v >= 1e9 else time.time() - v
198
+
199
+
200
+ def _local_midnight() -> float:
201
+ t = time.localtime()
202
+ return time.mktime((t.tm_year, t.tm_mon, t.tm_mday, 0, 0, 0, 0, 0, -1))
203
+
204
+
205
+ def _header(headers: Any, name: str) -> str | None:
206
+ if headers is None:
207
+ return None
208
+ try:
209
+ v = headers.get(name)
210
+ if v is None:
211
+ v = headers.get(name.title())
212
+ if v is not None:
213
+ return v
214
+ for k in headers.keys():
215
+ if k.lower() == name:
216
+ return headers[k]
217
+ except Exception:
218
+ return None
219
+ return None
220
+
221
+
222
+ # --------------------------------------------------------------------------------------------
223
+ # routes
224
+ # --------------------------------------------------------------------------------------------
225
+
226
+ def _stats(tap: "LLMTap", qs: dict[str, list[str]]) -> dict[str, Any]:
227
+ group_by = _first(qs, "group_by")
228
+ since = _since(qs)
229
+ out: dict[str, Any] = dict(tap.store.stats(since=since, group_by=group_by))
230
+ out["since"] = since
231
+ try:
232
+ out["session_spent"] = float(tap.budget.session_spent())
233
+ except Exception:
234
+ out["session_spent"] = 0.0
235
+ try:
236
+ out["spent_today"] = float(tap.store.spent_since(_local_midnight()))
237
+ except Exception:
238
+ pass
239
+ cache = getattr(tap, "cache", None)
240
+ try:
241
+ out["cache"] = cache.stats() if cache is not None else None
242
+ except Exception:
243
+ out["cache"] = {"entries": 0, "hits": 0}
244
+ cas = getattr(tap, "cassette", None)
245
+ if cas is not None:
246
+ try:
247
+ entries = len(cas)
248
+ except Exception:
249
+ entries = 0
250
+ out["cassette"] = {"path": str(getattr(cas, "path", "") or getattr(tap.cfg, "cassette", "")),
251
+ "mode": getattr(cas, "mode", None) or getattr(tap.cfg, "mode", "auto"),
252
+ "entries": entries}
253
+ else:
254
+ out["cassette"] = None
255
+ cfg = tap.cfg
256
+ out["config"] = {
257
+ "guard": cfg.guard,
258
+ "guard_pii": cfg.guard_pii,
259
+ "cache": bool(cfg.cache),
260
+ "cache_ttl_s": cfg.cache_ttl_s,
261
+ "budget_daily_usd": cfg.budget_daily_usd,
262
+ "budget_session_usd": cfg.budget_session_usd,
263
+ "loop_threshold": cfg.loop_threshold,
264
+ "loop_window_s": cfg.loop_window_s,
265
+ "mode": cfg.mode,
266
+ "port": getattr(tap, "port", cfg.port),
267
+ }
268
+ out["version"] = __version__
269
+ return out
270
+
271
+
272
+ def _exchanges(tap: "LLMTap", qs: dict[str, list[str]]) -> dict[str, Any]:
273
+ limit = _int(qs, "limit", 50, 1, MAX_LIMIT)
274
+ offset = _int(qs, "offset", 0, 0, 10**9)
275
+ items = tap.store.list(limit=limit, offset=offset, q=_first(qs, "q"), since=_since(qs),
276
+ provider=_first(qs, "provider"), model=_first(qs, "model"),
277
+ source=_first(qs, "source"))
278
+ return {"items": [summary(ex) for ex in items], "limit": limit, "offset": offset}
279
+
280
+
281
+ def _timeseries(tap: "LLMTap", qs: dict[str, list[str]]) -> dict[str, Any]:
282
+ since = _since(qs, default=time.time() - 86400) or 0.0
283
+ bucket = _int(qs, "bucket", 3600, 1, 86400 * 31)
284
+ return {"since": since, "bucket": bucket, "points": tap.store.timeseries(since, bucket)}
285
+
286
+
287
+ def _event_payload(event: Any) -> dict[str, Any]:
288
+ """Normalize a bus event: make sure any embedded exchange is a lightweight summary dict."""
289
+ if isinstance(event, Exchange):
290
+ return {"type": "exchange", "exchange": summary(event)}
291
+ if not isinstance(event, dict):
292
+ return {"type": "message", "data": event}
293
+ ex = event.get("exchange")
294
+ if isinstance(ex, Exchange) or (isinstance(ex, dict) and ("request_body" in ex or "response_body" in ex)):
295
+ event = dict(event)
296
+ event["exchange"] = summary(ex)
297
+ return event
298
+
299
+
300
+ def _events(tap: "LLMTap", respond: Responder) -> None:
301
+ """Server-sent events: one ``data:`` frame per bus event, ``: ping`` heartbeats."""
302
+ bus = tap.bus
303
+ q = bus.subscribe()
304
+ try:
305
+ respond.start(200, {
306
+ "content-type": "text/event-stream; charset=utf-8",
307
+ "cache-control": "no-cache, no-store",
308
+ "transfer-encoding": "chunked",
309
+ "x-accel-buffering": "no",
310
+ })
311
+ respond.write(b"retry: 2000\n: connected\n\n")
312
+ while True:
313
+ try:
314
+ event = q.get(timeout=HEARTBEAT_S)
315
+ except queue.Empty:
316
+ respond.write(b": ping\n\n")
317
+ continue
318
+ if event is None: # sentinel: shut down this stream
319
+ break
320
+ data = json.dumps(_event_payload(event), default=str, separators=(",", ":"))
321
+ respond.write(f"data: {data}\n\n".encode("utf-8"))
322
+ except Exception: # client went away (BrokenPipe / ConnectionReset / ...)
323
+ pass
324
+ finally:
325
+ try:
326
+ bus.unsubscribe(q)
327
+ except Exception:
328
+ pass
329
+ try:
330
+ respond.end()
331
+ except Exception:
332
+ pass
333
+
334
+
335
+ def _hostname(host: str) -> str:
336
+ """Hostname part of a Host header value: ``[::1]:4141`` -> ``::1``, ``LocalHost:80`` -> ``localhost``."""
337
+ host = host.strip().lower()
338
+ if host.startswith("["):
339
+ return host[1:].split("]", 1)[0]
340
+ if host.count(":") == 1:
341
+ host = host.split(":", 1)[0]
342
+ return host.rstrip(".")
343
+
344
+
345
+ def _host_allowed(tap: "LLMTap", headers: Any) -> bool:
346
+ """DNS-rebinding defense: the dashboard only answers to Host headers naming this machine.
347
+
348
+ Allowed: localhost, 127.0.0.1, ::1 (and any loopback IP literal), the configured host, and -- only when
349
+ llmtap is deliberately bound to a non-loopback address -- any IP literal (a rebinding page can only ever
350
+ send a *hostname*). A missing Host header (HTTP/1.0 tools) is allowed.
351
+ """
352
+ raw = _header(headers, "host")
353
+ if raw is None or not raw.strip():
354
+ return True
355
+ name = _hostname(raw)
356
+ cfg_host = _hostname(str(getattr(tap.cfg, "host", "") or ""))
357
+ if name in ("localhost", "127.0.0.1", "::1") or (cfg_host and name == cfg_host):
358
+ return True
359
+ try:
360
+ ip = ipaddress.ip_address(name)
361
+ except ValueError:
362
+ return False
363
+ return ip.is_loopback or bool(getattr(tap, "is_exposed", False))
364
+
365
+
366
+ def _same_origin(headers: Any) -> bool:
367
+ """Reject cross-site POSTs (a random web page must not be able to wipe your history)."""
368
+ fetch_site = _header(headers, "sec-fetch-site")
369
+ if fetch_site is not None and fetch_site.strip().lower() not in ("same-origin", "none"):
370
+ return False
371
+ origin = _header(headers, "origin")
372
+ if not origin or origin == "null":
373
+ return origin is None
374
+ host = _header(headers, "host") or ""
375
+ return urlsplit(origin).netloc.lower() == host.lower()
376
+
377
+
378
+ def handle_internal(tap: "LLMTap", method: str, path: str, headers: Any, body: bytes, respond: Responder) -> bool:
379
+ """Serve ``/_llmtap/*``. Returns True if the request was handled (always, for that prefix)."""
380
+ parts = urlsplit(path)
381
+ route = unquote(parts.path)
382
+ if not (route == PREFIX or route.startswith(PREFIX + "/")):
383
+ return False
384
+ qs = parse_qs(parts.query, keep_blank_values=False)
385
+ sub = route[len(PREFIX):] or ""
386
+ method = method.upper()
387
+ if not _host_allowed(tap, headers):
388
+ _error(respond, 403, "llmtap: request rejected (Host header is not this machine; "
389
+ "possible DNS-rebinding attack). Use http://localhost or http://127.0.0.1.")
390
+ return True
391
+
392
+ try:
393
+ if sub == "":
394
+ _send(respond, 302, b"", "text/plain", {"location": PREFIX + "/"})
395
+ return True
396
+
397
+ if method in ("GET", "HEAD"):
398
+ if sub in ("/", "/index.html"):
399
+ try:
400
+ html = UI_PATH.read_bytes()
401
+ except OSError:
402
+ html = b"<!doctype html><title>llmtap</title><p>llmtap UI missing (llmtap/ui/index.html)."
403
+ _send(respond, 200, b"" if method == "HEAD" else html, "text/html; charset=utf-8")
404
+ return True
405
+ if sub == "/health":
406
+ _json(respond, {"ok": True, "version": __version__})
407
+ return True
408
+ if sub == "/api/stats":
409
+ _json(respond, _stats(tap, qs))
410
+ return True
411
+ if sub == "/api/exchanges":
412
+ _json(respond, _exchanges(tap, qs))
413
+ return True
414
+ if sub.startswith("/api/exchanges/"):
415
+ ex_id = sub[len("/api/exchanges/"):].strip("/")
416
+ ex = tap.store.get(ex_id) if ex_id else None
417
+ if ex is None:
418
+ _error(respond, 404, f"exchange not found: {ex_id}")
419
+ return True
420
+ d = ex.to_dict()
421
+ d["preview"] = request_preview(ex.request_body)
422
+ _json(respond, d)
423
+ return True
424
+ if sub == "/api/timeseries":
425
+ _json(respond, _timeseries(tap, qs))
426
+ return True
427
+ if sub == "/api/events":
428
+ _events(tap, respond)
429
+ return True
430
+ elif method == "POST":
431
+ if sub == "/api/clear":
432
+ if not _same_origin(headers):
433
+ _error(respond, 403, "cross-origin request rejected")
434
+ return True
435
+ tap.store.clear()
436
+ _json(respond, {"ok": True})
437
+ return True
438
+ _error(respond, 404, f"no route for {method} {route}")
439
+ return True
440
+ except Exception as e: # never let the dashboard take down the proxy thread
441
+ try:
442
+ _error(respond, 500, f"dashboard error: {type(e).__name__}: {e}")
443
+ except Exception:
444
+ pass
445
+ return True