llmtap-proxy 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llmtap/__init__.py +2 -0
- llmtap/__main__.py +3 -0
- llmtap/budget.py +166 -0
- llmtap/cache.py +129 -0
- llmtap/cassette.py +172 -0
- llmtap/cli.py +1066 -0
- llmtap/config.py +112 -0
- llmtap/dashboard.py +445 -0
- llmtap/demo.py +244 -0
- llmtap/doctor.py +275 -0
- llmtap/guard.py +391 -0
- llmtap/mock.py +478 -0
- llmtap/models.py +75 -0
- llmtap/pricing.py +379 -0
- llmtap/providers.py +635 -0
- llmtap/proxy.py +1153 -0
- llmtap/pytest_plugin.py +444 -0
- llmtap/report.py +893 -0
- llmtap/store.py +319 -0
- llmtap/term.py +477 -0
- llmtap/ui/index.html +893 -0
- llmtap_proxy-0.2.0.dist-info/METADATA +609 -0
- llmtap_proxy-0.2.0.dist-info/RECORD +27 -0
- llmtap_proxy-0.2.0.dist-info/WHEEL +5 -0
- llmtap_proxy-0.2.0.dist-info/entry_points.txt +5 -0
- llmtap_proxy-0.2.0.dist-info/licenses/LICENSE +21 -0
- llmtap_proxy-0.2.0.dist-info/top_level.txt +1 -0
llmtap/config.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Configuration: defaults <- llmtap.toml <- LLMTAP_* env vars <- CLI flags."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import os
|
|
5
|
+
import tomllib
|
|
6
|
+
from dataclasses import dataclass, field, fields
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def llmtap_home() -> Path:
|
|
12
|
+
return Path(os.environ.get("LLMTAP_HOME", Path.home() / ".llmtap"))
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
DEFAULT_UPSTREAMS: dict[str, dict[str, str]] = {
|
|
16
|
+
# name -> {"base": url, "style": "anthropic"|"openai"}
|
|
17
|
+
"anthropic": {"base": "https://api.anthropic.com", "style": "anthropic"},
|
|
18
|
+
"openai": {"base": "https://api.openai.com", "style": "openai"},
|
|
19
|
+
"openrouter": {"base": "https://openrouter.ai/api", "style": "openai"},
|
|
20
|
+
"groq": {"base": "https://api.groq.com/openai", "style": "openai"},
|
|
21
|
+
"together": {"base": "https://api.together.xyz", "style": "openai"},
|
|
22
|
+
"mistral": {"base": "https://api.mistral.ai", "style": "openai"},
|
|
23
|
+
"deepseek": {"base": "https://api.deepseek.com", "style": "openai"},
|
|
24
|
+
"ollama": {"base": "http://localhost:11434", "style": "openai"},
|
|
25
|
+
"gemini": {"base":"https://generativelanguage.googleapis.com","style":"gemini"}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class Config:
|
|
31
|
+
host: str = "127.0.0.1"
|
|
32
|
+
port: int = 4141
|
|
33
|
+
db_path: str = "" # "" -> llmtap_home()/llmtap.db ; ":memory:" allowed
|
|
34
|
+
quiet: bool = False # suppress per-request console lines
|
|
35
|
+
# caching
|
|
36
|
+
cache: bool = False
|
|
37
|
+
cache_ttl_s: int = 0 # 0 = forever
|
|
38
|
+
# cassettes (record/replay)
|
|
39
|
+
cassette: str = "" # path to .jsonl cassette; "" = disabled
|
|
40
|
+
mode: str = "auto" # "record" | "replay" | "auto"
|
|
41
|
+
allow_miss: bool = False # in replay mode, forward misses upstream instead of erroring
|
|
42
|
+
# secret guard
|
|
43
|
+
guard: str = "warn" # "off" | "warn" | "redact" | "block"
|
|
44
|
+
guard_pii: bool = False # also detect emails / phone numbers / credit cards
|
|
45
|
+
# budgets & loop detection
|
|
46
|
+
budget_daily_usd: float = 0.0 # 0 = unlimited
|
|
47
|
+
budget_session_usd: float = 0.0 # 0 = unlimited (per proxy process)
|
|
48
|
+
loop_threshold: int = 0 # 0 = off; block when identical request seen N times...
|
|
49
|
+
loop_window_s: int = 60 # ...within this many seconds
|
|
50
|
+
# upstreams & pricing
|
|
51
|
+
upstreams: dict[str, dict[str, str]] = field(default_factory=lambda: {k: dict(v) for k, v in DEFAULT_UPSTREAMS.items()})
|
|
52
|
+
pricing_overrides: dict[str, dict[str, float]] = field(default_factory=dict) # model -> {"input":..,"output":..,"cache_read":..,"cache_write":..} USD per 1M tokens
|
|
53
|
+
upstream_timeout_s: float = 600.0
|
|
54
|
+
|
|
55
|
+
def resolved_db_path(self) -> str:
|
|
56
|
+
if self.db_path:
|
|
57
|
+
return self.db_path
|
|
58
|
+
home = llmtap_home()
|
|
59
|
+
home.mkdir(parents=True, exist_ok=True)
|
|
60
|
+
return str(home / "llmtap.db")
|
|
61
|
+
|
|
62
|
+
def base_url(self) -> str:
|
|
63
|
+
return f"http://{self.host}:{self.port}"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
_ENV_PREFIX = "LLMTAP_"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _coerce(value: str, typ: Any) -> Any:
|
|
70
|
+
t = typ if isinstance(typ, type) else type(typ)
|
|
71
|
+
if t is bool:
|
|
72
|
+
return value.strip().lower() in ("1", "true", "yes", "on")
|
|
73
|
+
if t is int:
|
|
74
|
+
return int(value)
|
|
75
|
+
if t is float:
|
|
76
|
+
return float(value)
|
|
77
|
+
return value
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def load_config(path: str | None = None, overrides: dict[str, Any] | None = None) -> Config:
|
|
81
|
+
"""Build a Config. `overrides` (e.g. from CLI) win; keys with value None are ignored."""
|
|
82
|
+
cfg = Config()
|
|
83
|
+
candidates = [Path(path)] if path else [Path("llmtap.toml"), Path.home() / ".config" / "llmtap" / "config.toml"]
|
|
84
|
+
for p in candidates:
|
|
85
|
+
if p.is_file():
|
|
86
|
+
data = tomllib.loads(p.read_text())
|
|
87
|
+
_apply(cfg, data.get("llmtap", data))
|
|
88
|
+
break
|
|
89
|
+
for f in fields(Config):
|
|
90
|
+
env = os.environ.get(_ENV_PREFIX + f.name.upper())
|
|
91
|
+
if env is not None and f.name not in ("upstreams", "pricing_overrides"):
|
|
92
|
+
setattr(cfg, f.name, _coerce(env, getattr(cfg, f.name)))
|
|
93
|
+
if overrides:
|
|
94
|
+
_apply(cfg, {k: v for k, v in overrides.items() if v is not None})
|
|
95
|
+
return cfg
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _apply(cfg: Config, data: dict[str, Any]) -> None:
|
|
99
|
+
names = {f.name for f in fields(Config)}
|
|
100
|
+
for k, v in data.items():
|
|
101
|
+
k = k.replace("-", "_")
|
|
102
|
+
if k == "pricing":
|
|
103
|
+
k = "pricing_overrides"
|
|
104
|
+
if k not in names:
|
|
105
|
+
continue
|
|
106
|
+
if k == "upstreams":
|
|
107
|
+
for name, spec in v.items():
|
|
108
|
+
cfg.upstreams[name] = {"base": spec["base"].rstrip("/"), "style": spec.get("style", "openai")}
|
|
109
|
+
elif k == "pricing_overrides":
|
|
110
|
+
cfg.pricing_overrides.update(v)
|
|
111
|
+
else:
|
|
112
|
+
setattr(cfg, k, v)
|
llmtap/dashboard.py
ADDED
|
@@ -0,0 +1,445 @@
|
|
|
1
|
+
"""Live web dashboard + JSON API, served by the proxy under ``/_llmtap/``.
|
|
2
|
+
|
|
3
|
+
Everything here talks to the proxy through a tiny surface: ``tap.store``, ``tap.cache``,
|
|
4
|
+
``tap.cassette``, ``tap.cfg``, ``tap.budget`` and ``tap.bus`` — plus the ``Responder`` protocol
|
|
5
|
+
(``start``/``write``/``end``), so it is independent from ``http.server``.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import ipaddress
|
|
10
|
+
import json
|
|
11
|
+
import queue
|
|
12
|
+
import time
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import TYPE_CHECKING, Any, Mapping, Protocol
|
|
15
|
+
from urllib.parse import parse_qs, unquote, urlsplit
|
|
16
|
+
|
|
17
|
+
from . import __version__
|
|
18
|
+
from .models import Exchange
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
21
|
+
from .proxy import LLMTap
|
|
22
|
+
|
|
23
|
+
PREFIX = "/_llmtap"
|
|
24
|
+
UI_PATH = Path(__file__).parent / "ui" / "index.html"
|
|
25
|
+
HEARTBEAT_S: float = 15.0 # SSE keep-alive interval (": ping")
|
|
26
|
+
PREVIEW_CHARS = 200
|
|
27
|
+
MAX_LIMIT = 1000
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class Responder(Protocol):
|
|
31
|
+
def start(self, status: int, headers: dict[str, str]) -> None: ...
|
|
32
|
+
def write(self, chunk: bytes) -> None: ...
|
|
33
|
+
def end(self) -> None: ...
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# --------------------------------------------------------------------------------------------
|
|
37
|
+
# summaries / previews
|
|
38
|
+
# --------------------------------------------------------------------------------------------
|
|
39
|
+
|
|
40
|
+
def _clip(text: str, n: int = PREVIEW_CHARS) -> str:
|
|
41
|
+
text = " ".join(str(text).split())
|
|
42
|
+
return text if len(text) <= n else text[: n - 1] + "…"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _content_text(content: Any) -> str:
|
|
46
|
+
"""Text of a message ``content``: a string, or a list of blocks (anthropic / openai / responses)."""
|
|
47
|
+
if content is None:
|
|
48
|
+
return ""
|
|
49
|
+
if isinstance(content, str):
|
|
50
|
+
return content
|
|
51
|
+
if isinstance(content, dict):
|
|
52
|
+
content = [content]
|
|
53
|
+
if not isinstance(content, list):
|
|
54
|
+
return ""
|
|
55
|
+
texts: list[str] = []
|
|
56
|
+
fallback: list[str] = []
|
|
57
|
+
for block in content:
|
|
58
|
+
if isinstance(block, str):
|
|
59
|
+
texts.append(block)
|
|
60
|
+
elif isinstance(block, dict):
|
|
61
|
+
btype = block.get("type")
|
|
62
|
+
if isinstance(block.get("text"), str) and btype in (None, "text", "input_text", "output_text"):
|
|
63
|
+
texts.append(block["text"])
|
|
64
|
+
elif btype == "tool_result":
|
|
65
|
+
fallback.append(_content_text(block.get("content")))
|
|
66
|
+
elif btype in ("image", "image_url", "input_image"):
|
|
67
|
+
fallback.append("[image]")
|
|
68
|
+
elif isinstance(block.get("text"), str):
|
|
69
|
+
texts.append(block["text"])
|
|
70
|
+
return "\n".join(t for t in texts if t) or "\n".join(t for t in fallback if t)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _last_user_text(req: Any) -> str:
|
|
74
|
+
"""Last user message text from an anthropic / openai chat / responses / embeddings / completions body."""
|
|
75
|
+
if not isinstance(req, dict):
|
|
76
|
+
return ""
|
|
77
|
+
msgs = req.get("messages")
|
|
78
|
+
if isinstance(msgs, list):
|
|
79
|
+
for m in reversed(msgs):
|
|
80
|
+
if isinstance(m, dict) and m.get("role") == "user":
|
|
81
|
+
text = _content_text(m.get("content"))
|
|
82
|
+
if text:
|
|
83
|
+
return text
|
|
84
|
+
for m in reversed(msgs): # no user text: fall back to any message text
|
|
85
|
+
if isinstance(m, dict):
|
|
86
|
+
text = _content_text(m.get("content"))
|
|
87
|
+
if text:
|
|
88
|
+
return text
|
|
89
|
+
inp = req.get("input")
|
|
90
|
+
if isinstance(inp, str):
|
|
91
|
+
return inp
|
|
92
|
+
if isinstance(inp, list):
|
|
93
|
+
for item in reversed(inp):
|
|
94
|
+
if isinstance(item, str):
|
|
95
|
+
return item
|
|
96
|
+
if isinstance(item, dict) and item.get("role", "user") == "user":
|
|
97
|
+
text = _content_text(item.get("content"))
|
|
98
|
+
if text:
|
|
99
|
+
return text
|
|
100
|
+
prompt = req.get("prompt")
|
|
101
|
+
if isinstance(prompt, str):
|
|
102
|
+
return prompt
|
|
103
|
+
if isinstance(prompt, list) and prompt and isinstance(prompt[-1], str):
|
|
104
|
+
return prompt[-1]
|
|
105
|
+
return ""
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def request_preview(request_body: str) -> str:
|
|
109
|
+
"""Preview of the last user message in a raw request body (<= 200 chars)."""
|
|
110
|
+
if not request_body:
|
|
111
|
+
return ""
|
|
112
|
+
try:
|
|
113
|
+
req = json.loads(request_body)
|
|
114
|
+
except (ValueError, TypeError):
|
|
115
|
+
return _clip(request_body)
|
|
116
|
+
if not isinstance(req, dict):
|
|
117
|
+
return _clip(request_body)
|
|
118
|
+
return _clip(_last_user_text(req))
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def summary(ex: Exchange | Mapping[str, Any]) -> dict[str, Any]:
|
|
122
|
+
"""Exchange dict minus request/response bodies, plus ``preview`` and ``output_preview``."""
|
|
123
|
+
d = ex.to_dict() if isinstance(ex, Exchange) else dict(ex)
|
|
124
|
+
req = d.pop("request_body", "") or ""
|
|
125
|
+
d.pop("response_body", None)
|
|
126
|
+
if "preview" not in d:
|
|
127
|
+
d["preview"] = request_preview(req)
|
|
128
|
+
if "output_preview" not in d:
|
|
129
|
+
out = d.get("output_text") or ""
|
|
130
|
+
d["output_preview"] = _clip(out or (d.get("error") or ""))
|
|
131
|
+
d.pop("output_text", None)
|
|
132
|
+
return d
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
exchange_summary = summary # alias
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
# --------------------------------------------------------------------------------------------
|
|
139
|
+
# response helpers
|
|
140
|
+
# --------------------------------------------------------------------------------------------
|
|
141
|
+
|
|
142
|
+
def _send(respond: Responder, status: int, body: bytes, ctype: str, extra: dict[str, str] | None = None) -> None:
|
|
143
|
+
headers = {
|
|
144
|
+
"content-type": ctype,
|
|
145
|
+
"content-length": str(len(body)),
|
|
146
|
+
"cache-control": "no-store",
|
|
147
|
+
"x-content-type-options": "nosniff",
|
|
148
|
+
}
|
|
149
|
+
if extra:
|
|
150
|
+
headers.update(extra)
|
|
151
|
+
respond.start(status, headers)
|
|
152
|
+
if body:
|
|
153
|
+
respond.write(body)
|
|
154
|
+
respond.end()
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _json(respond: Responder, obj: Any, status: int = 200) -> None:
|
|
158
|
+
body = json.dumps(obj, default=str, separators=(",", ":")).encode("utf-8")
|
|
159
|
+
_send(respond, status, body, "application/json; charset=utf-8")
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _error(respond: Responder, status: int, message: str) -> None:
|
|
163
|
+
_json(respond, {"error": message}, status)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _first(qs: dict[str, list[str]], name: str) -> str | None:
|
|
167
|
+
vals = qs.get(name)
|
|
168
|
+
if not vals:
|
|
169
|
+
return None
|
|
170
|
+
v = vals[0].strip()
|
|
171
|
+
return v or None
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _int(qs: dict[str, list[str]], name: str, default: int, lo: int, hi: int) -> int:
|
|
175
|
+
raw = _first(qs, name)
|
|
176
|
+
if raw is None:
|
|
177
|
+
return default
|
|
178
|
+
try:
|
|
179
|
+
return max(lo, min(hi, int(float(raw))))
|
|
180
|
+
except ValueError:
|
|
181
|
+
return default
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _since(qs: dict[str, list[str]], default: float | None = None) -> float | None:
|
|
185
|
+
"""``since`` is an epoch timestamp, or (if < 1e9) a number of seconds ago. ``all``/``0`` = no cutoff."""
|
|
186
|
+
raw = _first(qs, "since")
|
|
187
|
+
if raw is None:
|
|
188
|
+
return default
|
|
189
|
+
if raw.lower() in ("all", "0"):
|
|
190
|
+
return None
|
|
191
|
+
try:
|
|
192
|
+
v = float(raw)
|
|
193
|
+
except ValueError:
|
|
194
|
+
return default
|
|
195
|
+
if v <= 0:
|
|
196
|
+
return None
|
|
197
|
+
return v if v >= 1e9 else time.time() - v
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _local_midnight() -> float:
|
|
201
|
+
t = time.localtime()
|
|
202
|
+
return time.mktime((t.tm_year, t.tm_mon, t.tm_mday, 0, 0, 0, 0, 0, -1))
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _header(headers: Any, name: str) -> str | None:
|
|
206
|
+
if headers is None:
|
|
207
|
+
return None
|
|
208
|
+
try:
|
|
209
|
+
v = headers.get(name)
|
|
210
|
+
if v is None:
|
|
211
|
+
v = headers.get(name.title())
|
|
212
|
+
if v is not None:
|
|
213
|
+
return v
|
|
214
|
+
for k in headers.keys():
|
|
215
|
+
if k.lower() == name:
|
|
216
|
+
return headers[k]
|
|
217
|
+
except Exception:
|
|
218
|
+
return None
|
|
219
|
+
return None
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
# --------------------------------------------------------------------------------------------
|
|
223
|
+
# routes
|
|
224
|
+
# --------------------------------------------------------------------------------------------
|
|
225
|
+
|
|
226
|
+
def _stats(tap: "LLMTap", qs: dict[str, list[str]]) -> dict[str, Any]:
|
|
227
|
+
group_by = _first(qs, "group_by")
|
|
228
|
+
since = _since(qs)
|
|
229
|
+
out: dict[str, Any] = dict(tap.store.stats(since=since, group_by=group_by))
|
|
230
|
+
out["since"] = since
|
|
231
|
+
try:
|
|
232
|
+
out["session_spent"] = float(tap.budget.session_spent())
|
|
233
|
+
except Exception:
|
|
234
|
+
out["session_spent"] = 0.0
|
|
235
|
+
try:
|
|
236
|
+
out["spent_today"] = float(tap.store.spent_since(_local_midnight()))
|
|
237
|
+
except Exception:
|
|
238
|
+
pass
|
|
239
|
+
cache = getattr(tap, "cache", None)
|
|
240
|
+
try:
|
|
241
|
+
out["cache"] = cache.stats() if cache is not None else None
|
|
242
|
+
except Exception:
|
|
243
|
+
out["cache"] = {"entries": 0, "hits": 0}
|
|
244
|
+
cas = getattr(tap, "cassette", None)
|
|
245
|
+
if cas is not None:
|
|
246
|
+
try:
|
|
247
|
+
entries = len(cas)
|
|
248
|
+
except Exception:
|
|
249
|
+
entries = 0
|
|
250
|
+
out["cassette"] = {"path": str(getattr(cas, "path", "") or getattr(tap.cfg, "cassette", "")),
|
|
251
|
+
"mode": getattr(cas, "mode", None) or getattr(tap.cfg, "mode", "auto"),
|
|
252
|
+
"entries": entries}
|
|
253
|
+
else:
|
|
254
|
+
out["cassette"] = None
|
|
255
|
+
cfg = tap.cfg
|
|
256
|
+
out["config"] = {
|
|
257
|
+
"guard": cfg.guard,
|
|
258
|
+
"guard_pii": cfg.guard_pii,
|
|
259
|
+
"cache": bool(cfg.cache),
|
|
260
|
+
"cache_ttl_s": cfg.cache_ttl_s,
|
|
261
|
+
"budget_daily_usd": cfg.budget_daily_usd,
|
|
262
|
+
"budget_session_usd": cfg.budget_session_usd,
|
|
263
|
+
"loop_threshold": cfg.loop_threshold,
|
|
264
|
+
"loop_window_s": cfg.loop_window_s,
|
|
265
|
+
"mode": cfg.mode,
|
|
266
|
+
"port": getattr(tap, "port", cfg.port),
|
|
267
|
+
}
|
|
268
|
+
out["version"] = __version__
|
|
269
|
+
return out
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _exchanges(tap: "LLMTap", qs: dict[str, list[str]]) -> dict[str, Any]:
|
|
273
|
+
limit = _int(qs, "limit", 50, 1, MAX_LIMIT)
|
|
274
|
+
offset = _int(qs, "offset", 0, 0, 10**9)
|
|
275
|
+
items = tap.store.list(limit=limit, offset=offset, q=_first(qs, "q"), since=_since(qs),
|
|
276
|
+
provider=_first(qs, "provider"), model=_first(qs, "model"),
|
|
277
|
+
source=_first(qs, "source"))
|
|
278
|
+
return {"items": [summary(ex) for ex in items], "limit": limit, "offset": offset}
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _timeseries(tap: "LLMTap", qs: dict[str, list[str]]) -> dict[str, Any]:
|
|
282
|
+
since = _since(qs, default=time.time() - 86400) or 0.0
|
|
283
|
+
bucket = _int(qs, "bucket", 3600, 1, 86400 * 31)
|
|
284
|
+
return {"since": since, "bucket": bucket, "points": tap.store.timeseries(since, bucket)}
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _event_payload(event: Any) -> dict[str, Any]:
|
|
288
|
+
"""Normalize a bus event: make sure any embedded exchange is a lightweight summary dict."""
|
|
289
|
+
if isinstance(event, Exchange):
|
|
290
|
+
return {"type": "exchange", "exchange": summary(event)}
|
|
291
|
+
if not isinstance(event, dict):
|
|
292
|
+
return {"type": "message", "data": event}
|
|
293
|
+
ex = event.get("exchange")
|
|
294
|
+
if isinstance(ex, Exchange) or (isinstance(ex, dict) and ("request_body" in ex or "response_body" in ex)):
|
|
295
|
+
event = dict(event)
|
|
296
|
+
event["exchange"] = summary(ex)
|
|
297
|
+
return event
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _events(tap: "LLMTap", respond: Responder) -> None:
|
|
301
|
+
"""Server-sent events: one ``data:`` frame per bus event, ``: ping`` heartbeats."""
|
|
302
|
+
bus = tap.bus
|
|
303
|
+
q = bus.subscribe()
|
|
304
|
+
try:
|
|
305
|
+
respond.start(200, {
|
|
306
|
+
"content-type": "text/event-stream; charset=utf-8",
|
|
307
|
+
"cache-control": "no-cache, no-store",
|
|
308
|
+
"transfer-encoding": "chunked",
|
|
309
|
+
"x-accel-buffering": "no",
|
|
310
|
+
})
|
|
311
|
+
respond.write(b"retry: 2000\n: connected\n\n")
|
|
312
|
+
while True:
|
|
313
|
+
try:
|
|
314
|
+
event = q.get(timeout=HEARTBEAT_S)
|
|
315
|
+
except queue.Empty:
|
|
316
|
+
respond.write(b": ping\n\n")
|
|
317
|
+
continue
|
|
318
|
+
if event is None: # sentinel: shut down this stream
|
|
319
|
+
break
|
|
320
|
+
data = json.dumps(_event_payload(event), default=str, separators=(",", ":"))
|
|
321
|
+
respond.write(f"data: {data}\n\n".encode("utf-8"))
|
|
322
|
+
except Exception: # client went away (BrokenPipe / ConnectionReset / ...)
|
|
323
|
+
pass
|
|
324
|
+
finally:
|
|
325
|
+
try:
|
|
326
|
+
bus.unsubscribe(q)
|
|
327
|
+
except Exception:
|
|
328
|
+
pass
|
|
329
|
+
try:
|
|
330
|
+
respond.end()
|
|
331
|
+
except Exception:
|
|
332
|
+
pass
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _hostname(host: str) -> str:
|
|
336
|
+
"""Hostname part of a Host header value: ``[::1]:4141`` -> ``::1``, ``LocalHost:80`` -> ``localhost``."""
|
|
337
|
+
host = host.strip().lower()
|
|
338
|
+
if host.startswith("["):
|
|
339
|
+
return host[1:].split("]", 1)[0]
|
|
340
|
+
if host.count(":") == 1:
|
|
341
|
+
host = host.split(":", 1)[0]
|
|
342
|
+
return host.rstrip(".")
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _host_allowed(tap: "LLMTap", headers: Any) -> bool:
|
|
346
|
+
"""DNS-rebinding defense: the dashboard only answers to Host headers naming this machine.
|
|
347
|
+
|
|
348
|
+
Allowed: localhost, 127.0.0.1, ::1 (and any loopback IP literal), the configured host, and -- only when
|
|
349
|
+
llmtap is deliberately bound to a non-loopback address -- any IP literal (a rebinding page can only ever
|
|
350
|
+
send a *hostname*). A missing Host header (HTTP/1.0 tools) is allowed.
|
|
351
|
+
"""
|
|
352
|
+
raw = _header(headers, "host")
|
|
353
|
+
if raw is None or not raw.strip():
|
|
354
|
+
return True
|
|
355
|
+
name = _hostname(raw)
|
|
356
|
+
cfg_host = _hostname(str(getattr(tap.cfg, "host", "") or ""))
|
|
357
|
+
if name in ("localhost", "127.0.0.1", "::1") or (cfg_host and name == cfg_host):
|
|
358
|
+
return True
|
|
359
|
+
try:
|
|
360
|
+
ip = ipaddress.ip_address(name)
|
|
361
|
+
except ValueError:
|
|
362
|
+
return False
|
|
363
|
+
return ip.is_loopback or bool(getattr(tap, "is_exposed", False))
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _same_origin(headers: Any) -> bool:
|
|
367
|
+
"""Reject cross-site POSTs (a random web page must not be able to wipe your history)."""
|
|
368
|
+
fetch_site = _header(headers, "sec-fetch-site")
|
|
369
|
+
if fetch_site is not None and fetch_site.strip().lower() not in ("same-origin", "none"):
|
|
370
|
+
return False
|
|
371
|
+
origin = _header(headers, "origin")
|
|
372
|
+
if not origin or origin == "null":
|
|
373
|
+
return origin is None
|
|
374
|
+
host = _header(headers, "host") or ""
|
|
375
|
+
return urlsplit(origin).netloc.lower() == host.lower()
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def handle_internal(tap: "LLMTap", method: str, path: str, headers: Any, body: bytes, respond: Responder) -> bool:
|
|
379
|
+
"""Serve ``/_llmtap/*``. Returns True if the request was handled (always, for that prefix)."""
|
|
380
|
+
parts = urlsplit(path)
|
|
381
|
+
route = unquote(parts.path)
|
|
382
|
+
if not (route == PREFIX or route.startswith(PREFIX + "/")):
|
|
383
|
+
return False
|
|
384
|
+
qs = parse_qs(parts.query, keep_blank_values=False)
|
|
385
|
+
sub = route[len(PREFIX):] or ""
|
|
386
|
+
method = method.upper()
|
|
387
|
+
if not _host_allowed(tap, headers):
|
|
388
|
+
_error(respond, 403, "llmtap: request rejected (Host header is not this machine; "
|
|
389
|
+
"possible DNS-rebinding attack). Use http://localhost or http://127.0.0.1.")
|
|
390
|
+
return True
|
|
391
|
+
|
|
392
|
+
try:
|
|
393
|
+
if sub == "":
|
|
394
|
+
_send(respond, 302, b"", "text/plain", {"location": PREFIX + "/"})
|
|
395
|
+
return True
|
|
396
|
+
|
|
397
|
+
if method in ("GET", "HEAD"):
|
|
398
|
+
if sub in ("/", "/index.html"):
|
|
399
|
+
try:
|
|
400
|
+
html = UI_PATH.read_bytes()
|
|
401
|
+
except OSError:
|
|
402
|
+
html = b"<!doctype html><title>llmtap</title><p>llmtap UI missing (llmtap/ui/index.html)."
|
|
403
|
+
_send(respond, 200, b"" if method == "HEAD" else html, "text/html; charset=utf-8")
|
|
404
|
+
return True
|
|
405
|
+
if sub == "/health":
|
|
406
|
+
_json(respond, {"ok": True, "version": __version__})
|
|
407
|
+
return True
|
|
408
|
+
if sub == "/api/stats":
|
|
409
|
+
_json(respond, _stats(tap, qs))
|
|
410
|
+
return True
|
|
411
|
+
if sub == "/api/exchanges":
|
|
412
|
+
_json(respond, _exchanges(tap, qs))
|
|
413
|
+
return True
|
|
414
|
+
if sub.startswith("/api/exchanges/"):
|
|
415
|
+
ex_id = sub[len("/api/exchanges/"):].strip("/")
|
|
416
|
+
ex = tap.store.get(ex_id) if ex_id else None
|
|
417
|
+
if ex is None:
|
|
418
|
+
_error(respond, 404, f"exchange not found: {ex_id}")
|
|
419
|
+
return True
|
|
420
|
+
d = ex.to_dict()
|
|
421
|
+
d["preview"] = request_preview(ex.request_body)
|
|
422
|
+
_json(respond, d)
|
|
423
|
+
return True
|
|
424
|
+
if sub == "/api/timeseries":
|
|
425
|
+
_json(respond, _timeseries(tap, qs))
|
|
426
|
+
return True
|
|
427
|
+
if sub == "/api/events":
|
|
428
|
+
_events(tap, respond)
|
|
429
|
+
return True
|
|
430
|
+
elif method == "POST":
|
|
431
|
+
if sub == "/api/clear":
|
|
432
|
+
if not _same_origin(headers):
|
|
433
|
+
_error(respond, 403, "cross-origin request rejected")
|
|
434
|
+
return True
|
|
435
|
+
tap.store.clear()
|
|
436
|
+
_json(respond, {"ok": True})
|
|
437
|
+
return True
|
|
438
|
+
_error(respond, 404, f"no route for {method} {route}")
|
|
439
|
+
return True
|
|
440
|
+
except Exception as e: # never let the dashboard take down the proxy thread
|
|
441
|
+
try:
|
|
442
|
+
_error(respond, 500, f"dashboard error: {type(e).__name__}: {e}")
|
|
443
|
+
except Exception:
|
|
444
|
+
pass
|
|
445
|
+
return True
|