llmtap-proxy 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llmtap/__init__.py +2 -0
- llmtap/__main__.py +3 -0
- llmtap/budget.py +166 -0
- llmtap/cache.py +129 -0
- llmtap/cassette.py +172 -0
- llmtap/cli.py +1066 -0
- llmtap/config.py +112 -0
- llmtap/dashboard.py +445 -0
- llmtap/demo.py +244 -0
- llmtap/doctor.py +275 -0
- llmtap/guard.py +391 -0
- llmtap/mock.py +478 -0
- llmtap/models.py +75 -0
- llmtap/pricing.py +379 -0
- llmtap/providers.py +635 -0
- llmtap/proxy.py +1153 -0
- llmtap/pytest_plugin.py +444 -0
- llmtap/report.py +893 -0
- llmtap/store.py +319 -0
- llmtap/term.py +477 -0
- llmtap/ui/index.html +893 -0
- llmtap_proxy-0.2.0.dist-info/METADATA +609 -0
- llmtap_proxy-0.2.0.dist-info/RECORD +27 -0
- llmtap_proxy-0.2.0.dist-info/WHEEL +5 -0
- llmtap_proxy-0.2.0.dist-info/entry_points.txt +5 -0
- llmtap_proxy-0.2.0.dist-info/licenses/LICENSE +21 -0
- llmtap_proxy-0.2.0.dist-info/top_level.txt +1 -0
llmtap/proxy.py
ADDED
|
@@ -0,0 +1,1153 @@
|
|
|
1
|
+
"""The core: an HTTP proxy that records, prices, caches, replays, guards and budgets LLM calls.
|
|
2
|
+
|
|
3
|
+
``LLMTap.handle`` implements the whole per-request pipeline against an abstract ``Responder`` so
|
|
4
|
+
the dashboard and tests don't care about ``http.server``. ``_Handler`` adapts
|
|
5
|
+
``BaseHTTPRequestHandler`` (HTTP/1.1, keep-alive, Content-Length / chunked framing) to it.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import gzip
|
|
10
|
+
import http.client
|
|
11
|
+
import ipaddress
|
|
12
|
+
import json
|
|
13
|
+
import queue
|
|
14
|
+
import re
|
|
15
|
+
import socket
|
|
16
|
+
import socketserver
|
|
17
|
+
import ssl
|
|
18
|
+
import sys
|
|
19
|
+
import threading
|
|
20
|
+
import time
|
|
21
|
+
import zlib
|
|
22
|
+
from collections.abc import Mapping
|
|
23
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
24
|
+
from typing import Any, Protocol
|
|
25
|
+
from urllib.parse import parse_qsl, urlsplit
|
|
26
|
+
|
|
27
|
+
from . import __version__
|
|
28
|
+
from .budget import Budget
|
|
29
|
+
from .cache import ResponseCache, cache_key
|
|
30
|
+
from .cassette import Cassette
|
|
31
|
+
from .config import Config
|
|
32
|
+
from .guard import apply_guard
|
|
33
|
+
from .models import CachedResponse, Exchange, Route, Usage
|
|
34
|
+
from .pricing import cost_usd
|
|
35
|
+
from .providers import (
|
|
36
|
+
body_error,
|
|
37
|
+
error_body,
|
|
38
|
+
estimate_usage,
|
|
39
|
+
extract_text,
|
|
40
|
+
extract_usage,
|
|
41
|
+
forward_headers,
|
|
42
|
+
is_stream_request,
|
|
43
|
+
parse_json,
|
|
44
|
+
parse_sse,
|
|
45
|
+
model_from_path,
|
|
46
|
+
request_model,
|
|
47
|
+
resolve_route,
|
|
48
|
+
response_model,
|
|
49
|
+
sse_error,
|
|
50
|
+
tags_from_headers,
|
|
51
|
+
)
|
|
52
|
+
from .store import Store
|
|
53
|
+
|
|
54
|
+
__all__ = ["LLMTap", "EventBus", "Responder", "summarize"]
|
|
55
|
+
|
|
56
|
+
# Response headers never relayed to the client (we do our own framing).
|
|
57
|
+
_RESP_HOP = frozenset({
|
|
58
|
+
"connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailers",
|
|
59
|
+
"transfer-encoding", "upgrade", "content-length",
|
|
60
|
+
})
|
|
61
|
+
# Headers never stored for replay (body is stored decoded, framing is recomputed).
|
|
62
|
+
_STORE_DROP = _RESP_HOP | {"content-encoding", "date", "server", "x-llmtap-id", "x-llmtap-source",
|
|
63
|
+
"access-control-allow-origin", "access-control-expose-headers"}
|
|
64
|
+
_CLIENT_GONE = (BrokenPipeError, ConnectionResetError, ConnectionAbortedError, TimeoutError)
|
|
65
|
+
# Per-exchange cap (characters) for what is *stored* (db / cache / cassette). Relayed bytes are never capped.
|
|
66
|
+
MAX_STORED_CHARS = 2 * 1024 * 1024
|
|
67
|
+
_CORS_BASE = {
|
|
68
|
+
"Access-Control-Allow-Origin": "*",
|
|
69
|
+
"Access-Control-Expose-Headers": "x-llmtap-id, x-llmtap-source, request-id, x-request-id",
|
|
70
|
+
}
|
|
71
|
+
_INTERNAL = "/_llmtap"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class Responder(Protocol):
|
|
75
|
+
"""Minimal response sink used by the pipeline and the dashboard."""
|
|
76
|
+
|
|
77
|
+
def start(self, status: int, headers: dict[str, str]) -> None: ...
|
|
78
|
+
def write(self, chunk: bytes) -> None: ... # chunk frame when started with transfer-encoding: chunked
|
|
79
|
+
def end(self) -> None: ...
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class ClientGone(Exception):
|
|
83
|
+
"""The downstream client disconnected while we were writing to it."""
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
# ------------------------------------------------------------------------------------------------
|
|
87
|
+
# event bus
|
|
88
|
+
# ------------------------------------------------------------------------------------------------
|
|
89
|
+
class EventBus:
|
|
90
|
+
"""Thread-safe pub/sub fan-out; each subscriber gets its own bounded Queue (slow ones drop)."""
|
|
91
|
+
|
|
92
|
+
def __init__(self, maxsize: int = 1000) -> None:
|
|
93
|
+
self._lock = threading.Lock()
|
|
94
|
+
self._subs: list[queue.Queue] = []
|
|
95
|
+
self._maxsize = maxsize
|
|
96
|
+
|
|
97
|
+
def publish(self, event: dict) -> None:
|
|
98
|
+
with self._lock:
|
|
99
|
+
subs = list(self._subs)
|
|
100
|
+
for q in subs:
|
|
101
|
+
try:
|
|
102
|
+
q.put_nowait(event)
|
|
103
|
+
except queue.Full:
|
|
104
|
+
pass
|
|
105
|
+
|
|
106
|
+
def subscribe(self) -> queue.Queue:
|
|
107
|
+
q: queue.Queue = queue.Queue(maxsize=self._maxsize)
|
|
108
|
+
with self._lock:
|
|
109
|
+
self._subs.append(q)
|
|
110
|
+
return q
|
|
111
|
+
|
|
112
|
+
def unsubscribe(self, q: queue.Queue) -> None:
|
|
113
|
+
with self._lock:
|
|
114
|
+
try:
|
|
115
|
+
self._subs.remove(q)
|
|
116
|
+
except ValueError:
|
|
117
|
+
pass
|
|
118
|
+
|
|
119
|
+
def subscriber_count(self) -> int:
|
|
120
|
+
with self._lock:
|
|
121
|
+
return len(self._subs)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def summarize(ex: Exchange) -> dict[str, Any]:
|
|
125
|
+
"""Exchange dict minus bodies (dashboard's richer summary when available)."""
|
|
126
|
+
try:
|
|
127
|
+
from .dashboard import summary # lazy: dashboard is optional
|
|
128
|
+
return summary(ex)
|
|
129
|
+
except Exception:
|
|
130
|
+
d = ex.to_dict()
|
|
131
|
+
d.pop("request_body", None)
|
|
132
|
+
d.pop("response_body", None)
|
|
133
|
+
return d
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
# ------------------------------------------------------------------------------------------------
|
|
137
|
+
# helpers
|
|
138
|
+
# ------------------------------------------------------------------------------------------------
|
|
139
|
+
class _Tracked:
|
|
140
|
+
"""Wraps a Responder: remembers whether it started, injects headers, maps disconnects."""
|
|
141
|
+
|
|
142
|
+
def __init__(self, inner: Responder, extra: dict[str, str] | None = None) -> None:
|
|
143
|
+
self.inner = inner
|
|
144
|
+
self.extra = extra or {}
|
|
145
|
+
self.started = False
|
|
146
|
+
self.ended = False
|
|
147
|
+
|
|
148
|
+
def start(self, status: int, headers: dict[str, str]) -> None:
|
|
149
|
+
h = dict(headers)
|
|
150
|
+
low = {k.lower() for k in h}
|
|
151
|
+
for k, v in self.extra.items():
|
|
152
|
+
if k.lower() not in low:
|
|
153
|
+
h[k] = v
|
|
154
|
+
self.started = True
|
|
155
|
+
try:
|
|
156
|
+
self.inner.start(status, h)
|
|
157
|
+
except _CLIENT_GONE as e:
|
|
158
|
+
raise ClientGone(str(e)) from e
|
|
159
|
+
|
|
160
|
+
def write(self, chunk: bytes) -> None:
|
|
161
|
+
try:
|
|
162
|
+
self.inner.write(chunk)
|
|
163
|
+
except _CLIENT_GONE as e:
|
|
164
|
+
raise ClientGone(str(e)) from e
|
|
165
|
+
|
|
166
|
+
def end(self) -> None:
|
|
167
|
+
if self.ended:
|
|
168
|
+
return
|
|
169
|
+
self.ended = True
|
|
170
|
+
try:
|
|
171
|
+
self.inner.end()
|
|
172
|
+
except _CLIENT_GONE as e:
|
|
173
|
+
raise ClientGone(str(e)) from e
|
|
174
|
+
|
|
175
|
+
def abort(self) -> None:
|
|
176
|
+
"""Drop the connection without completing the body (so the client sees truncation)."""
|
|
177
|
+
self.ended = True
|
|
178
|
+
fn = getattr(self.inner, "abort", None)
|
|
179
|
+
if fn is not None:
|
|
180
|
+
try:
|
|
181
|
+
fn()
|
|
182
|
+
except Exception:
|
|
183
|
+
pass
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _lower(headers: Mapping[str, str] | None) -> dict[str, str]:
|
|
187
|
+
out: dict[str, str] = {}
|
|
188
|
+
try:
|
|
189
|
+
for k, v in (headers or {}).items():
|
|
190
|
+
out[str(k).lower()] = str(v)
|
|
191
|
+
except Exception:
|
|
192
|
+
pass
|
|
193
|
+
return out
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _try_decode(raw: bytes, enc: str) -> bytes | None:
|
|
197
|
+
"""Undo content-encoding; None when we can't (br, zstd, compress, corrupt gzip...)."""
|
|
198
|
+
try:
|
|
199
|
+
if enc in ("gzip", "x-gzip"):
|
|
200
|
+
return gzip.decompress(raw)
|
|
201
|
+
if enc == "deflate":
|
|
202
|
+
try:
|
|
203
|
+
return zlib.decompress(raw)
|
|
204
|
+
except zlib.error:
|
|
205
|
+
return zlib.decompress(raw, -zlib.MAX_WBITS)
|
|
206
|
+
except Exception:
|
|
207
|
+
return None
|
|
208
|
+
return None
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
_BINARY_PREFIXES = ("audio/", "image/", "video/", "font/")
|
|
212
|
+
_BINARY_TYPES = frozenset({"application/octet-stream", "application/zip", "application/pdf", "application/gzip",
|
|
213
|
+
"application/x-tar", "application/x-zip-compressed", "application/x-protobuf"})
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _decode_response(raw: bytes, encoding: str, ctype: str) -> tuple[str, bool]:
|
|
217
|
+
"""(text, parseable). Binary / undecodable bodies yield a short placeholder and parseable=False
|
|
218
|
+
so we never dump megabytes of mojibake into the db or cache (the client still gets the raw bytes)."""
|
|
219
|
+
if not raw:
|
|
220
|
+
return "", True
|
|
221
|
+
enc = (encoding or "").strip().lower()
|
|
222
|
+
data = raw
|
|
223
|
+
if enc not in ("", "identity"):
|
|
224
|
+
dec = _try_decode(raw, enc)
|
|
225
|
+
if dec is None:
|
|
226
|
+
return (f"[response body not stored: content-encoding '{enc}' is not decodable by llmtap, "
|
|
227
|
+
f"{len(raw)} bytes]"), False
|
|
228
|
+
data = dec
|
|
229
|
+
main = (ctype or "").split(";")[0].strip().lower()
|
|
230
|
+
if main.startswith(_BINARY_PREFIXES) or main in _BINARY_TYPES:
|
|
231
|
+
return f"[binary response body not stored: {main}, {len(data)} bytes]", False
|
|
232
|
+
try:
|
|
233
|
+
return data.decode("utf-8"), True
|
|
234
|
+
except UnicodeDecodeError:
|
|
235
|
+
if any(t in main for t in ("json", "text", "event-stream", "xml")):
|
|
236
|
+
return data.decode("utf-8", "replace"), True
|
|
237
|
+
return f"[binary response body not stored: {main or 'unknown content-type'}, {len(data)} bytes]", False
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _decode_request(body: bytes, ctype: str) -> tuple[str, bool]:
|
|
241
|
+
"""(text to store, truncated). Multipart / binary request bodies are stored as a short placeholder."""
|
|
242
|
+
main = (ctype or "").split(";")[0].strip().lower()
|
|
243
|
+
if main.startswith("multipart/") or main in _BINARY_TYPES or main.startswith(_BINARY_PREFIXES):
|
|
244
|
+
return f"[{main} request body not stored: {len(body)} bytes]", False
|
|
245
|
+
try:
|
|
246
|
+
text = body[:MAX_STORED_CHARS * 4].decode("utf-8")
|
|
247
|
+
except UnicodeDecodeError as e:
|
|
248
|
+
if e.start > MAX_STORED_CHARS * 4 - 8 or len(body) > MAX_STORED_CHARS * 4:
|
|
249
|
+
text = body[:MAX_STORED_CHARS * 4].decode("utf-8", "ignore")
|
|
250
|
+
else:
|
|
251
|
+
return f"[binary request body not stored: {main or 'unknown content-type'}, {len(body)} bytes]", False
|
|
252
|
+
if len(text) > MAX_STORED_CHARS or len(body) > MAX_STORED_CHARS * 4:
|
|
253
|
+
return text[:MAX_STORED_CHARS] + f"\n[llmtap: truncated, {len(body)} bytes in total]", True
|
|
254
|
+
return text, False
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
_SENSITIVE_QS = frozenset({"key", "api_key", "apikey", "api-key", "token", "access_token", "x-api-key",
|
|
258
|
+
"signature", "sig", "auth", "password", "secret", "client_secret"})
|
|
259
|
+
_SENSITIVE_QS_RE = re.compile(r"([?&](?:" + "|".join(sorted(re.escape(k) for k in _SENSITIVE_QS)) + r")=)[^&#]*",
|
|
260
|
+
re.IGNORECASE)
|
|
261
|
+
_CRED_HEADERS = ("authorization", "proxy-authorization", "x-api-key", "api-key", "x-goog-api-key", "cookie",
|
|
262
|
+
"x-auth-token", "openai-api-key", "anthropic-auth-token")
|
|
263
|
+
REDACTED = "[REDACTED]"
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _mask_path(path: str) -> str:
|
|
267
|
+
"""Mask secret-looking query parameters (``?key=AIza...``) before the path is ever stored."""
|
|
268
|
+
return _SENSITIVE_QS_RE.sub(lambda m: m.group(1) + REDACTED, path) if "?" in path else path
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _credentials(hdrs: Mapping[str, str], path: str) -> list[str]:
|
|
272
|
+
"""Credential strings of this request (>= 8 chars), longest first, so they can be scrubbed from anything stored."""
|
|
273
|
+
vals: set[str] = set()
|
|
274
|
+
for name in _CRED_HEADERS:
|
|
275
|
+
v = (hdrs.get(name) or "").strip()
|
|
276
|
+
if not v:
|
|
277
|
+
continue
|
|
278
|
+
vals.add(v)
|
|
279
|
+
if name == "cookie":
|
|
280
|
+
for part in v.split(";"):
|
|
281
|
+
vals.add(part.strip().partition("=")[2].strip())
|
|
282
|
+
else:
|
|
283
|
+
vals.add(v.split(None, 1)[-1]) # "Bearer abc" -> "abc"
|
|
284
|
+
for k, v in parse_qsl(path.partition("?")[2], keep_blank_values=True):
|
|
285
|
+
if k.lower() in _SENSITIVE_QS:
|
|
286
|
+
vals.add(v)
|
|
287
|
+
return sorted({v for v in vals if len(v) >= 8}, key=len, reverse=True)
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _scrub(text: str, secrets: list[str]) -> str:
|
|
291
|
+
for sec in secrets:
|
|
292
|
+
if sec in text:
|
|
293
|
+
text = text.replace(sec, REDACTED)
|
|
294
|
+
return text
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _storable_headers(headers: list[tuple[str, str]], secrets: list[str] | None = None) -> dict[str, str]:
|
|
298
|
+
out: dict[str, str] = {}
|
|
299
|
+
for k, v in headers:
|
|
300
|
+
lk = k.lower()
|
|
301
|
+
if lk in _STORE_DROP or lk.startswith("x-llmtap-") or lk == "set-cookie" or lk in _CRED_HEADERS:
|
|
302
|
+
continue
|
|
303
|
+
if secrets and any(sec in v for sec in secrets): # upstream echoed a credential back
|
|
304
|
+
continue
|
|
305
|
+
out[lk] = v
|
|
306
|
+
return out
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _note(ex: Exchange, text: str) -> None:
|
|
310
|
+
"""Attach a non-fatal note (e.g. truncation); folded into ``ex.error`` when the exchange is sealed."""
|
|
311
|
+
ex.__dict__.setdefault("_notes", []).append(text)
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _seal(ex: Exchange) -> None:
|
|
315
|
+
"""Final pass before an exchange is persisted: fold notes into ``error``, scrub credentials."""
|
|
316
|
+
notes = ex.__dict__.pop("_notes", None)
|
|
317
|
+
if notes:
|
|
318
|
+
ex.error = "; ".join(x for x in [ex.error, *notes] if x)
|
|
319
|
+
secrets = ex.__dict__.pop("_secrets", None)
|
|
320
|
+
if secrets:
|
|
321
|
+
for f in ("path", "request_body", "response_body", "error", "output_text"):
|
|
322
|
+
v = getattr(ex, f, None)
|
|
323
|
+
if v:
|
|
324
|
+
setattr(ex, f, _scrub(v, secrets))
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
_LLM_TAIL = re.compile(r"(?:/(?:chat/completions|completions|embeddings)|:(?:stream)?[gG]enerateContent|(?:^|/)v\d+[a-z0-9]*/(?:messages|complete|responses))$")
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _is_llm_call(method: str, upstream_path: str) -> bool:
|
|
331
|
+
"""True for billable generation endpoints (chat / messages / responses / embeddings / completions).
|
|
332
|
+
Everything else (models, files, batches, count_tokens, ...) is forwarded untouched: recorded, but
|
|
333
|
+
never cached, budget-checked, loop-checked or priced."""
|
|
334
|
+
if method != "POST":
|
|
335
|
+
return False
|
|
336
|
+
return bool(_LLM_TAIL.search(upstream_path.split("?", 1)[0].rstrip("/")))
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _is_json_doc(body: bytes) -> bool:
|
|
340
|
+
"""Cheap check that a (non-object) body is a JSON document, e.g. an array."""
|
|
341
|
+
head = body.lstrip()[:1]
|
|
342
|
+
if head not in (b"[", b'"'):
|
|
343
|
+
return False
|
|
344
|
+
try:
|
|
345
|
+
json.loads(body.decode("utf-8-sig"))
|
|
346
|
+
return True
|
|
347
|
+
except Exception:
|
|
348
|
+
return False
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _split_sse(body: str) -> list[str]:
|
|
352
|
+
"""Split a stored SSE transcript into whole events (each ending with a blank line)."""
|
|
353
|
+
text = body.replace("\r\n", "\n")
|
|
354
|
+
parts = [p for p in text.split("\n\n") if p.strip()]
|
|
355
|
+
return [p + "\n\n" for p in parts]
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _upstream_target(route: Route) -> tuple[str, str, int | None, str]:
|
|
359
|
+
"""(scheme, host, port, full_path) for a route, honoring base paths like https://x/api."""
|
|
360
|
+
u = urlsplit(route.upstream_base)
|
|
361
|
+
scheme = (u.scheme or "http").lower()
|
|
362
|
+
base_path = (u.path or "").rstrip("/")
|
|
363
|
+
up = route.upstream_path or "/"
|
|
364
|
+
if not up.startswith("/"):
|
|
365
|
+
up = "/" + up
|
|
366
|
+
return scheme, u.hostname or "localhost", u.port, base_path + up
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _is_streaming_response(ctype: str, req_stream: bool, has_length: bool) -> bool:
|
|
370
|
+
c = ctype.lower()
|
|
371
|
+
if "text/event-stream" in c or "ndjson" in c or "jsonl" in c:
|
|
372
|
+
return True
|
|
373
|
+
return req_stream and not has_length and "json" not in c
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
# ------------------------------------------------------------------------------------------------
|
|
377
|
+
# the proxy
|
|
378
|
+
# ------------------------------------------------------------------------------------------------
|
|
379
|
+
class LLMTap:
|
|
380
|
+
"""The llmtap proxy: shared state + per-request pipeline + server lifecycle."""
|
|
381
|
+
|
|
382
|
+
def __init__(self, cfg: Config, store: Store | None = None) -> None:
|
|
383
|
+
self.cfg = cfg
|
|
384
|
+
self.store: Store = store if store is not None else Store(cfg.resolved_db_path())
|
|
385
|
+
self.cache: ResponseCache | None = (
|
|
386
|
+
ResponseCache(cfg.resolved_db_path(), cfg.cache_ttl_s) if cfg.cache else None
|
|
387
|
+
)
|
|
388
|
+
self.cassette: Cassette | None = Cassette(cfg.cassette, cfg.mode) if cfg.cassette else None
|
|
389
|
+
self.budget: Budget = Budget(cfg, self.store.spent_since)
|
|
390
|
+
self.bus: EventBus = EventBus()
|
|
391
|
+
self.port: int = cfg.port
|
|
392
|
+
self.started_at: float = time.time()
|
|
393
|
+
self._server: _Server | None = None
|
|
394
|
+
self._thread: threading.Thread | None = None
|
|
395
|
+
self._serving = threading.Event()
|
|
396
|
+
self._closed = False
|
|
397
|
+
|
|
398
|
+
# ---------------------------------------------------------------- lifecycle
|
|
399
|
+
@property
|
|
400
|
+
def base_url(self) -> str:
|
|
401
|
+
host = self.cfg.host
|
|
402
|
+
if host in ("", "0.0.0.0", "::"):
|
|
403
|
+
host = "127.0.0.1"
|
|
404
|
+
if ":" in host and not host.startswith("["):
|
|
405
|
+
host = f"[{host}]"
|
|
406
|
+
return f"http://{host}:{self.port}"
|
|
407
|
+
|
|
408
|
+
@property
|
|
409
|
+
def is_exposed(self) -> bool:
|
|
410
|
+
"""True when the listen address is reachable beyond this machine (0.0.0.0, ::, a LAN address...)."""
|
|
411
|
+
return _host_is_exposed(self.cfg.host)
|
|
412
|
+
|
|
413
|
+
def _bind(self) -> "_Server":
|
|
414
|
+
if self._server is None:
|
|
415
|
+
self._server = _Server((self.cfg.host, self.cfg.port), _Handler, self)
|
|
416
|
+
self.port = int(self._server.server_address[1])
|
|
417
|
+
if self.is_exposed:
|
|
418
|
+
self._warn_exposed()
|
|
419
|
+
return self._server
|
|
420
|
+
|
|
421
|
+
def _warn_exposed(self) -> None:
|
|
422
|
+
"""Prominent stderr warning: prompts, responses and the dashboard are served without authentication."""
|
|
423
|
+
try:
|
|
424
|
+
from .term import color_enabled, paint
|
|
425
|
+
msg = (f"WARNING: llmtap is listening on {self.cfg.host or '0.0.0.0'}:{self.port} and is exposed to "
|
|
426
|
+
"your network. The dashboard and every recorded prompt/response are readable by anyone who "
|
|
427
|
+
"can reach this port, and nothing is authenticated. Bind to 127.0.0.1 unless you mean this.")
|
|
428
|
+
sys.stderr.write(paint(msg, "bold", "yellow", color=color_enabled(sys.stderr)) + "\n")
|
|
429
|
+
sys.stderr.flush()
|
|
430
|
+
except Exception:
|
|
431
|
+
pass
|
|
432
|
+
|
|
433
|
+
def serve_forever(self) -> None:
|
|
434
|
+
"""Serve in the calling thread until shutdown() (or KeyboardInterrupt propagates)."""
|
|
435
|
+
srv = self._bind()
|
|
436
|
+
self._serving.set()
|
|
437
|
+
try:
|
|
438
|
+
srv.serve_forever(poll_interval=0.25)
|
|
439
|
+
finally:
|
|
440
|
+
self._serving.clear()
|
|
441
|
+
|
|
442
|
+
def start_background(self) -> "LLMTap":
|
|
443
|
+
"""Bind now (so .port/.base_url are valid on return) and serve in a daemon thread."""
|
|
444
|
+
self._bind()
|
|
445
|
+
self._thread = threading.Thread(target=self.serve_forever, name="llmtap-proxy", daemon=True)
|
|
446
|
+
self._thread.start()
|
|
447
|
+
self._serving.wait(5)
|
|
448
|
+
return self
|
|
449
|
+
|
|
450
|
+
def shutdown(self) -> None:
|
|
451
|
+
"""Stop serving and close the server socket, store and cache. Idempotent."""
|
|
452
|
+
if self._closed:
|
|
453
|
+
return
|
|
454
|
+
self._closed = True
|
|
455
|
+
srv = self._server
|
|
456
|
+
if srv is not None:
|
|
457
|
+
if self._serving.is_set():
|
|
458
|
+
try:
|
|
459
|
+
srv.shutdown()
|
|
460
|
+
except Exception:
|
|
461
|
+
pass
|
|
462
|
+
try:
|
|
463
|
+
srv.server_close()
|
|
464
|
+
except Exception:
|
|
465
|
+
pass
|
|
466
|
+
if self._thread is not None:
|
|
467
|
+
self._thread.join(timeout=5)
|
|
468
|
+
for obj in (self.cache, self.store):
|
|
469
|
+
close = getattr(obj, "close", None)
|
|
470
|
+
if close is not None:
|
|
471
|
+
try:
|
|
472
|
+
close()
|
|
473
|
+
except Exception:
|
|
474
|
+
pass
|
|
475
|
+
|
|
476
|
+
def __enter__(self) -> "LLMTap":
|
|
477
|
+
return self.start_background()
|
|
478
|
+
|
|
479
|
+
def __exit__(self, *exc: Any) -> None:
|
|
480
|
+
self.shutdown()
|
|
481
|
+
|
|
482
|
+
# ---------------------------------------------------------------- pipeline
|
|
483
|
+
def handle(self, method: str, path: str, headers: Mapping[str, str], body: bytes,
|
|
484
|
+
respond: Responder) -> Exchange:
|
|
485
|
+
"""Process one request end-to-end and return the (recorded) Exchange."""
|
|
486
|
+
method = (method or "GET").upper()
|
|
487
|
+
path = path or "/"
|
|
488
|
+
hdrs = _lower(headers)
|
|
489
|
+
body = body or b""
|
|
490
|
+
bare = path.split("?", 1)[0]
|
|
491
|
+
|
|
492
|
+
# -- internal routes (not recorded) -------------------------------------------------
|
|
493
|
+
if bare == _INTERNAL or bare.startswith(_INTERNAL + "/"):
|
|
494
|
+
ex = Exchange(method=method, path=path, provider="llmtap", source="internal")
|
|
495
|
+
self._internal(method, path, headers, body, respond, ex)
|
|
496
|
+
return ex
|
|
497
|
+
if bare == "/" and method in ("GET", "HEAD"):
|
|
498
|
+
ex = Exchange(method=method, path=path, provider="llmtap", source="internal", status=302)
|
|
499
|
+
self._simple(respond, 302, b"", {"Location": _INTERNAL + "/"})
|
|
500
|
+
return ex
|
|
501
|
+
if bare == "/favicon.ico":
|
|
502
|
+
ex = Exchange(method=method, path=path, provider="llmtap", source="internal", status=204)
|
|
503
|
+
self._simple(respond, 204, b"", {})
|
|
504
|
+
return ex
|
|
505
|
+
if method == "OPTIONS" and "access-control-request-method" in hdrs:
|
|
506
|
+
ex = Exchange(method=method, path=path, provider="llmtap", source="internal", status=204)
|
|
507
|
+
self._simple(respond, 204, b"", {
|
|
508
|
+
"Access-Control-Allow-Origin": "*",
|
|
509
|
+
"Access-Control-Allow-Methods": "GET, POST, PUT, PATCH, DELETE, OPTIONS",
|
|
510
|
+
"Access-Control-Allow-Headers": hdrs.get("access-control-request-headers") or "*",
|
|
511
|
+
"Access-Control-Max-Age": "86400",
|
|
512
|
+
})
|
|
513
|
+
return ex
|
|
514
|
+
|
|
515
|
+
# -- proxied LLM call ---------------------------------------------------------------
|
|
516
|
+
ex = Exchange(method=method, path=_mask_path(path))
|
|
517
|
+
ex.__dict__["_secrets"] = _credentials(hdrs, path)
|
|
518
|
+
t0 = time.perf_counter()
|
|
519
|
+
out = _Tracked(respond, dict(_CORS_BASE))
|
|
520
|
+
style = "openai"
|
|
521
|
+
try:
|
|
522
|
+
route = resolve_route(path, hdrs, self.cfg)
|
|
523
|
+
style = route.style
|
|
524
|
+
ex.provider, ex.style = route.provider, route.style
|
|
525
|
+
self._pipeline(ex, route, method, hdrs, body, out, t0)
|
|
526
|
+
except ClientGone:
|
|
527
|
+
ex.error = ex.error or "client disconnected"
|
|
528
|
+
except Exception as e: # never let a bug kill the connection silently
|
|
529
|
+
ex.error = f"llmtap internal error: {type(e).__name__}: {e}"
|
|
530
|
+
if ex.source == "upstream":
|
|
531
|
+
ex.source = "error"
|
|
532
|
+
if not out.started:
|
|
533
|
+
ex.status = 500
|
|
534
|
+
self._error(out, ex, style, 500, ex.error, "llmtap_internal_error")
|
|
535
|
+
else:
|
|
536
|
+
out.abort()
|
|
537
|
+
ex.latency_ms = round((time.perf_counter() - t0) * 1000.0, 2)
|
|
538
|
+
self._finish(ex)
|
|
539
|
+
return ex
|
|
540
|
+
|
|
541
|
+
def _pipeline(self, ex: Exchange, route: Route, method: str, hdrs: dict[str, str], body: bytes,
|
|
542
|
+
out: _Tracked, t0: float) -> None:
|
|
543
|
+
cfg = self.cfg
|
|
544
|
+
req = parse_json(body)
|
|
545
|
+
ex.model = request_model(req) or model_from_path(route.upstream_path)
|
|
546
|
+
stream_req = is_stream_request(req)
|
|
547
|
+
ex.tags = tags_from_headers(hdrs)
|
|
548
|
+
|
|
549
|
+
llm = _is_llm_call(method, route.upstream_path)
|
|
550
|
+
ctype = hdrs.get("content-type", "")
|
|
551
|
+
|
|
552
|
+
# guard. Only JSON documents may ever be rewritten: multipart / binary / unparseable bodies are
|
|
553
|
+
# scanned as text (warn / block) but forwarded byte-for-byte even in redact mode.
|
|
554
|
+
send_body = body
|
|
555
|
+
mode = cfg.guard or "off"
|
|
556
|
+
if body and mode != "off":
|
|
557
|
+
if mode == "redact" and req is None and not _is_json_doc(body):
|
|
558
|
+
mode = "warn"
|
|
559
|
+
send_body, findings, blocked = apply_guard(body, mode, cfg.guard_pii)
|
|
560
|
+
ex.guard_findings = [f.to_dict() if hasattr(f, "to_dict") else dict(f) for f in findings]
|
|
561
|
+
if blocked:
|
|
562
|
+
ex.request_body, trunc = _decode_request(body, ctype)
|
|
563
|
+
if trunc:
|
|
564
|
+
_note(ex, f"request body truncated to {MAX_STORED_CHARS} characters in llmtap storage")
|
|
565
|
+
kinds = ", ".join(sorted({str(f.get("kind", "?")) for f in ex.guard_findings}))
|
|
566
|
+
ex.source = "blocked"
|
|
567
|
+
msg = (f"llmtap guard blocked this request: detected {kinds} in the prompt. "
|
|
568
|
+
"Remove the secret, or run llmtap with --guard redact / warn.")
|
|
569
|
+
self._error(out, ex, route.style, 400, msg, "llmtap_guard_blocked")
|
|
570
|
+
return
|
|
571
|
+
if send_body is not body:
|
|
572
|
+
req = parse_json(send_body) if req is not None else req
|
|
573
|
+
ex.request_body, trunc = _decode_request(send_body, ctype)
|
|
574
|
+
if trunc:
|
|
575
|
+
_note(ex, f"request body truncated to {MAX_STORED_CHARS} characters in llmtap storage (forwarded in full)")
|
|
576
|
+
|
|
577
|
+
cacheable = llm
|
|
578
|
+
key = cache_key(route.provider, route.upstream_path, send_body) if cacheable else ""
|
|
579
|
+
|
|
580
|
+
# budget / loop breaker (generation calls only: models/files/batches/count_tokens are not chat traffic)
|
|
581
|
+
if cacheable:
|
|
582
|
+
verdict = self.budget.check(key)
|
|
583
|
+
if not verdict.allowed:
|
|
584
|
+
ex.source = verdict.kind or "budget"
|
|
585
|
+
err_type = "llmtap_loop_detected" if ex.source == "loop" else "llmtap_budget_exceeded"
|
|
586
|
+
self._error(out, ex, route.style, 429, verdict.reason or "llmtap budget exceeded", err_type)
|
|
587
|
+
return
|
|
588
|
+
|
|
589
|
+
# cassette / cache
|
|
590
|
+
if cacheable:
|
|
591
|
+
stored: CachedResponse | None = None
|
|
592
|
+
if self.cassette is not None:
|
|
593
|
+
stored = self.cassette.lookup(key)
|
|
594
|
+
if stored is not None:
|
|
595
|
+
ex.source = "cassette"
|
|
596
|
+
if stored is None and self.cache is not None:
|
|
597
|
+
stored = self.cache.get(key)
|
|
598
|
+
if stored is not None:
|
|
599
|
+
ex.source = "cache"
|
|
600
|
+
if stored is not None:
|
|
601
|
+
self._replay(ex, route, req, stored, out, t0)
|
|
602
|
+
return
|
|
603
|
+
if self.cassette is not None and self.cassette.mode == "replay" and not cfg.allow_miss:
|
|
604
|
+
ex.source = "error"
|
|
605
|
+
msg = ("llmtap cassette miss (replay mode): no recording for this request. "
|
|
606
|
+
"Re-record with --mode record")
|
|
607
|
+
self._error(out, ex, route.style, 409, msg, "llmtap_cassette_miss")
|
|
608
|
+
return
|
|
609
|
+
|
|
610
|
+
self._forward(ex, route, method, hdrs, send_body, req, stream_req, key, out, t0)
|
|
611
|
+
|
|
612
|
+
# ---------------------------------------------------------------- upstream
|
|
613
|
+
def _forward(self, ex: Exchange, route: Route, method: str, hdrs: dict[str, str], body: bytes,
|
|
614
|
+
req: dict | None, stream_req: bool, key: str, out: _Tracked, t0: float) -> None:
|
|
615
|
+
llm = _is_llm_call(method, route.upstream_path)
|
|
616
|
+
scheme, host, port, full_path = _upstream_target(route)
|
|
617
|
+
timeout = float(self.cfg.upstream_timeout_s or 600.0)
|
|
618
|
+
conn: http.client.HTTPConnection
|
|
619
|
+
if scheme == "https":
|
|
620
|
+
conn = http.client.HTTPSConnection(host, port, timeout=timeout, context=ssl.create_default_context())
|
|
621
|
+
else:
|
|
622
|
+
conn = http.client.HTTPConnection(host, port, timeout=timeout)
|
|
623
|
+
fwd = forward_headers(hdrs)
|
|
624
|
+
try:
|
|
625
|
+
try:
|
|
626
|
+
conn.request(method, full_path, body=body if (body or method in ("POST", "PUT", "PATCH")) else None,
|
|
627
|
+
headers=fwd)
|
|
628
|
+
resp = conn.getresponse()
|
|
629
|
+
except (OSError, http.client.HTTPException) as e:
|
|
630
|
+
ex.source = "error"
|
|
631
|
+
msg = f"llmtap: upstream {route.upstream_base} unreachable: {type(e).__name__}: {e}"
|
|
632
|
+
status = 504 if isinstance(e, (socket.timeout, TimeoutError)) else 502
|
|
633
|
+
self._error(out, ex, route.style, status, msg, "llmtap_upstream_error")
|
|
634
|
+
return
|
|
635
|
+
|
|
636
|
+
ex.status = resp.status
|
|
637
|
+
up_headers = resp.getheaders()
|
|
638
|
+
low = {k.lower(): v for k, v in up_headers}
|
|
639
|
+
ctype = low.get("content-type", "")
|
|
640
|
+
encoding = low.get("content-encoding", "")
|
|
641
|
+
relay = [(k, v) for k, v in up_headers if k.lower() not in _RESP_HOP]
|
|
642
|
+
client_h = _merge_headers(relay, {"x-llmtap-id": ex.id, "x-llmtap-source": "upstream"})
|
|
643
|
+
bodyless = method == "HEAD" or resp.status in (204, 304) or resp.status < 200
|
|
644
|
+
if bodyless and resp.status != 204 and "content-length" in low:
|
|
645
|
+
client_h["Content-Length"] = low["content-length"] # HEAD / 304 describe the entity: keep its length
|
|
646
|
+
streaming = not bodyless and _is_streaming_response(ctype, stream_req, "content-length" in low)
|
|
647
|
+
ex.streamed = streaming
|
|
648
|
+
raw = bytearray()
|
|
649
|
+
client_gone = False
|
|
650
|
+
finalized = False
|
|
651
|
+
upstream_err: str | None = None
|
|
652
|
+
|
|
653
|
+
if streaming:
|
|
654
|
+
client_h["Transfer-Encoding"] = "chunked"
|
|
655
|
+
out.start(resp.status, client_h)
|
|
656
|
+
while True:
|
|
657
|
+
try:
|
|
658
|
+
chunk = resp.read1(65536)
|
|
659
|
+
except (OSError, http.client.HTTPException) as e:
|
|
660
|
+
upstream_err = f"upstream read error: {type(e).__name__}: {e}"
|
|
661
|
+
break
|
|
662
|
+
if not chunk:
|
|
663
|
+
break
|
|
664
|
+
if ex.ttft_ms is None:
|
|
665
|
+
ex.ttft_ms = round((time.perf_counter() - t0) * 1000.0, 2)
|
|
666
|
+
raw += chunk
|
|
667
|
+
try:
|
|
668
|
+
out.write(chunk)
|
|
669
|
+
except ClientGone:
|
|
670
|
+
client_gone = True
|
|
671
|
+
break
|
|
672
|
+
if client_gone or upstream_err:
|
|
673
|
+
out.abort()
|
|
674
|
+
else:
|
|
675
|
+
self._finalize(ex, route, req, key, llm, bytes(raw), encoding, ctype, streaming, up_headers)
|
|
676
|
+
finalized = True
|
|
677
|
+
try:
|
|
678
|
+
out.end()
|
|
679
|
+
except ClientGone:
|
|
680
|
+
client_gone = True
|
|
681
|
+
else:
|
|
682
|
+
try:
|
|
683
|
+
data = resp.read() if not bodyless else b""
|
|
684
|
+
except (OSError, http.client.HTTPException) as e:
|
|
685
|
+
ex.source = "error"
|
|
686
|
+
msg = f"llmtap: upstream read failed: {type(e).__name__}: {e}"
|
|
687
|
+
status = 504 if isinstance(e, (socket.timeout, TimeoutError)) else 502
|
|
688
|
+
self._error(out, ex, route.style, status, msg, "llmtap_upstream_error")
|
|
689
|
+
return
|
|
690
|
+
raw += data
|
|
691
|
+
ex.ttft_ms = round((time.perf_counter() - t0) * 1000.0, 2)
|
|
692
|
+
if not bodyless:
|
|
693
|
+
client_h["Content-Length"] = str(len(raw))
|
|
694
|
+
self._finalize(ex, route, req, key, llm, bytes(raw), encoding, ctype, streaming, up_headers)
|
|
695
|
+
finalized = True
|
|
696
|
+
try:
|
|
697
|
+
out.start(resp.status, client_h)
|
|
698
|
+
if raw and not bodyless:
|
|
699
|
+
out.write(bytes(raw))
|
|
700
|
+
out.end()
|
|
701
|
+
except ClientGone:
|
|
702
|
+
client_gone = True
|
|
703
|
+
out.abort()
|
|
704
|
+
finally:
|
|
705
|
+
try:
|
|
706
|
+
conn.close()
|
|
707
|
+
except Exception:
|
|
708
|
+
pass
|
|
709
|
+
|
|
710
|
+
if not finalized:
|
|
711
|
+
self._finalize(ex, route, req, key, llm, bytes(raw), encoding, ctype, streaming, up_headers,
|
|
712
|
+
client_gone, upstream_err)
|
|
713
|
+
elif client_gone:
|
|
714
|
+
ex.error = "client disconnected"
|
|
715
|
+
|
|
716
|
+
def _finalize(self, ex: Exchange, route: Route, req: dict | None, key: str, llm: bool, raw: bytes,
|
|
717
|
+
encoding: str, ctype: str, streaming: bool, up_headers: list[tuple[str, str]],
|
|
718
|
+
client_gone: bool = False, upstream_err: str | None = None) -> None:
|
|
719
|
+
"""Usage, cost, budget, cache and cassette for a finished upstream response. Runs *before* the last
|
|
720
|
+
byte reaches the client (when possible) so a client that immediately sends its next request already
|
|
721
|
+
sees the updated budget / cache."""
|
|
722
|
+
raw = bytes(raw)
|
|
723
|
+
text, parseable = _decode_response(bytes(raw), encoding, ctype)
|
|
724
|
+
ex.response_body = text
|
|
725
|
+
if client_gone:
|
|
726
|
+
ex.error = "client disconnected"
|
|
727
|
+
elif upstream_err:
|
|
728
|
+
ex.error = upstream_err
|
|
729
|
+
if ex.status >= 400 and not ex.error:
|
|
730
|
+
ex.error = _upstream_error_message(text) or f"upstream HTTP {ex.status}"
|
|
731
|
+
|
|
732
|
+
in_band_err = self._account(ex, route, req, text, streaming, llm=llm, parseable=parseable)
|
|
733
|
+
ex.cost_usd = self._price(ex) if ex.usage.total() else 0.0
|
|
734
|
+
try:
|
|
735
|
+
self.budget.add_spend(ex.cost_usd)
|
|
736
|
+
except Exception:
|
|
737
|
+
pass
|
|
738
|
+
|
|
739
|
+
oversize = len(text) > MAX_STORED_CHARS or len(ex.output_text) > MAX_STORED_CHARS
|
|
740
|
+
if oversize:
|
|
741
|
+
ex.response_body = text[:MAX_STORED_CHARS] + f"\n[llmtap: truncated, {len(raw)} bytes in total]"
|
|
742
|
+
ex.output_text = ex.output_text[:MAX_STORED_CHARS]
|
|
743
|
+
_note(ex, f"response body truncated to {MAX_STORED_CHARS} characters in llmtap storage "
|
|
744
|
+
"(relayed to the client in full; not cached)")
|
|
745
|
+
|
|
746
|
+
if (200 <= ex.status < 300 and not (streaming and client_gone) and not upstream_err and key and parseable
|
|
747
|
+
and not oversize and not in_band_err):
|
|
748
|
+
secrets = ex.__dict__.get("_secrets") or []
|
|
749
|
+
cached = CachedResponse(status=ex.status, headers=_storable_headers(up_headers, secrets),
|
|
750
|
+
body=_scrub(text, secrets), streamed=streaming, cost_usd=ex.cost_usd)
|
|
751
|
+
if self.cache is not None:
|
|
752
|
+
try:
|
|
753
|
+
self.cache.put(key, cached)
|
|
754
|
+
except Exception as e:
|
|
755
|
+
ex.error = ex.error or f"cache write failed: {e}"
|
|
756
|
+
if self.cassette is not None and self.cassette.should_record():
|
|
757
|
+
try:
|
|
758
|
+
rec_req: Any = req if req is not None else ex.request_body
|
|
759
|
+
if secrets and isinstance(rec_req, dict):
|
|
760
|
+
rec_req = json.loads(_scrub(json.dumps(rec_req, ensure_ascii=False), secrets))
|
|
761
|
+
self.cassette.record(key, rec_req, cached)
|
|
762
|
+
except Exception as e:
|
|
763
|
+
ex.error = ex.error or f"cassette write failed: {e}"
|
|
764
|
+
|
|
765
|
+
# ---------------------------------------------------------------- replay
|
|
766
|
+
def _replay(self, ex: Exchange, route: Route, req: dict | None, stored: CachedResponse,
|
|
767
|
+
out: _Tracked, t0: float) -> None:
|
|
768
|
+
ex.status = int(stored.status)
|
|
769
|
+
ex.streamed = bool(stored.streamed)
|
|
770
|
+
ex.response_body = stored.body
|
|
771
|
+
ex.saved_usd = float(stored.cost_usd or 0.0)
|
|
772
|
+
ex.cost_usd = 0.0
|
|
773
|
+
h: dict[str, str] = {}
|
|
774
|
+
for k, v in (stored.headers or {}).items():
|
|
775
|
+
lk = str(k).lower()
|
|
776
|
+
if lk in _STORE_DROP or lk.startswith("x-llmtap-") or lk == "host":
|
|
777
|
+
continue
|
|
778
|
+
h[lk] = str(v)
|
|
779
|
+
h["x-llmtap-id"] = ex.id
|
|
780
|
+
h["x-llmtap-source"] = ex.source
|
|
781
|
+
data = stored.body.encode("utf-8")
|
|
782
|
+
try:
|
|
783
|
+
if ex.streamed:
|
|
784
|
+
h["content-type"] = h.get("content-type") or "text/event-stream"
|
|
785
|
+
if "event-stream" not in h["content-type"] and "ndjson" not in h["content-type"]:
|
|
786
|
+
h["content-type"] = "text/event-stream"
|
|
787
|
+
h["cache-control"] = "no-cache"
|
|
788
|
+
h["Transfer-Encoding"] = "chunked"
|
|
789
|
+
out.start(ex.status, h)
|
|
790
|
+
events = _split_sse(stored.body) if "event-stream" in h["content-type"] else (
|
|
791
|
+
[ln + "\n" for ln in stored.body.splitlines() if ln.strip()])
|
|
792
|
+
for i, evt in enumerate(events):
|
|
793
|
+
out.write(evt.encode("utf-8"))
|
|
794
|
+
if i == 0:
|
|
795
|
+
ex.ttft_ms = round((time.perf_counter() - t0) * 1000.0, 2)
|
|
796
|
+
out.end()
|
|
797
|
+
else:
|
|
798
|
+
h.setdefault("content-type", "application/json")
|
|
799
|
+
h["Content-Length"] = str(len(data))
|
|
800
|
+
out.start(ex.status, h)
|
|
801
|
+
if data:
|
|
802
|
+
out.write(data)
|
|
803
|
+
out.end()
|
|
804
|
+
ex.ttft_ms = round((time.perf_counter() - t0) * 1000.0, 2)
|
|
805
|
+
except ClientGone:
|
|
806
|
+
ex.error = "client disconnected"
|
|
807
|
+
out.abort()
|
|
808
|
+
self._account(ex, route, req, stored.body, ex.streamed)
|
|
809
|
+
|
|
810
|
+
# ---------------------------------------------------------------- accounting
|
|
811
|
+
def _account(self, ex: Exchange, route: Route, req: dict | None, text: str, streamed: bool,
|
|
812
|
+
llm: bool = True, parseable: bool = True) -> str | None:
|
|
813
|
+
"""Fill usage / output_text / model from a response body (never raises).
|
|
814
|
+
|
|
815
|
+
Returns an in-band error message (stream ``error`` event, or HTTP 200 with an ``error`` body) if any.
|
|
816
|
+
Usage is only *estimated* for successful generation calls whose body carried no usage and no error.
|
|
817
|
+
"""
|
|
818
|
+
usage, out_text, model, err = Usage(), "", None, None
|
|
819
|
+
if parseable:
|
|
820
|
+
try:
|
|
821
|
+
if streamed:
|
|
822
|
+
usage, out_text, model = parse_sse(route.style, text)
|
|
823
|
+
err = sse_error(text)
|
|
824
|
+
else:
|
|
825
|
+
j = parse_json(text.encode("utf-8"))
|
|
826
|
+
if j is not None:
|
|
827
|
+
usage = extract_usage(route.style, j)
|
|
828
|
+
out_text = extract_text(route.style, j)
|
|
829
|
+
model = response_model(route.style, j)
|
|
830
|
+
err = body_error(j)
|
|
831
|
+
except Exception:
|
|
832
|
+
pass
|
|
833
|
+
ok = 200 <= ex.status < 300
|
|
834
|
+
if err and ok:
|
|
835
|
+
ex.error = ex.error or err
|
|
836
|
+
if ok and llm and not err and usage.input_tokens == 0 and usage.output_tokens == 0:
|
|
837
|
+
try:
|
|
838
|
+
usage = estimate_usage(req, out_text)
|
|
839
|
+
except Exception:
|
|
840
|
+
pass
|
|
841
|
+
ex.usage = usage
|
|
842
|
+
ex.output_text = out_text or ""
|
|
843
|
+
if model:
|
|
844
|
+
ex.model = model
|
|
845
|
+
return err if ok else None
|
|
846
|
+
|
|
847
|
+
def _price(self, ex: Exchange) -> float:
|
|
848
|
+
try:
|
|
849
|
+
return float(cost_usd(ex.model, ex.usage, self.cfg.pricing_overrides) or 0.0)
|
|
850
|
+
except Exception:
|
|
851
|
+
return 0.0
|
|
852
|
+
|
|
853
|
+
# ---------------------------------------------------------------- responses
|
|
854
|
+
def _simple(self, respond: Responder, status: int, data: bytes, headers: dict[str, str]) -> None:
|
|
855
|
+
h = dict(headers)
|
|
856
|
+
h["Content-Length"] = str(len(data))
|
|
857
|
+
try:
|
|
858
|
+
respond.start(status, h)
|
|
859
|
+
if data:
|
|
860
|
+
respond.write(data)
|
|
861
|
+
respond.end()
|
|
862
|
+
except _CLIENT_GONE:
|
|
863
|
+
pass
|
|
864
|
+
|
|
865
|
+
def _error(self, out: _Tracked, ex: Exchange, style: str, status: int, message: str, err_type: str) -> None:
|
|
866
|
+
"""Send a provider-shaped JSON error and fill the exchange accordingly."""
|
|
867
|
+
data = error_body(style, status, message, err_type)
|
|
868
|
+
ex.status = status
|
|
869
|
+
ex.error = ex.error or message
|
|
870
|
+
ex.response_body = data.decode("utf-8", "replace")
|
|
871
|
+
if out.started:
|
|
872
|
+
return
|
|
873
|
+
try:
|
|
874
|
+
hdrs = {"Content-Type": "application/json", "Content-Length": str(len(data)),
|
|
875
|
+
"x-llmtap-id": ex.id, "x-llmtap-source": ex.source}
|
|
876
|
+
if status in (400, 409, 429):
|
|
877
|
+
# deliberate llmtap blocks (guard/cassette miss/budget/loop): tell official SDKs not to retry
|
|
878
|
+
hdrs["x-should-retry"] = "false"
|
|
879
|
+
out.start(status, hdrs)
|
|
880
|
+
out.write(data)
|
|
881
|
+
out.end()
|
|
882
|
+
except ClientGone:
|
|
883
|
+
pass
|
|
884
|
+
|
|
885
|
+
def _internal(self, method: str, path: str, headers: Mapping[str, str], body: bytes,
|
|
886
|
+
respond: Responder, ex: Exchange) -> None:
|
|
887
|
+
out = _Tracked(respond)
|
|
888
|
+
try:
|
|
889
|
+
from .dashboard import handle_internal
|
|
890
|
+
except Exception:
|
|
891
|
+
handle_internal = None # type: ignore[assignment]
|
|
892
|
+
try:
|
|
893
|
+
handled = bool(handle_internal(self, method, path, headers, body, out)) if handle_internal else False
|
|
894
|
+
except ClientGone:
|
|
895
|
+
return
|
|
896
|
+
except Exception as e:
|
|
897
|
+
ex.error = f"dashboard error: {type(e).__name__}: {e}"
|
|
898
|
+
if not out.started:
|
|
899
|
+
data = json.dumps({"error": ex.error}).encode()
|
|
900
|
+
ex.status = 500
|
|
901
|
+
self._simple(respond, 500, data, {"Content-Type": "application/json"})
|
|
902
|
+
return
|
|
903
|
+
if not handled and not out.started:
|
|
904
|
+
ex.status = 404
|
|
905
|
+
data = json.dumps({"error": f"not found: {path}", "version": __version__}).encode()
|
|
906
|
+
self._simple(respond, 404, data, {"Content-Type": "application/json"})
|
|
907
|
+
|
|
908
|
+
# ---------------------------------------------------------------- finish
|
|
909
|
+
def _finish(self, ex: Exchange) -> None:
|
|
910
|
+
"""Persist, publish and log; each step isolated so none can break the request."""
|
|
911
|
+
try:
|
|
912
|
+
_seal(ex)
|
|
913
|
+
except Exception:
|
|
914
|
+
ex.__dict__.pop("_secrets", None)
|
|
915
|
+
try:
|
|
916
|
+
self.store.record(ex)
|
|
917
|
+
except Exception as e:
|
|
918
|
+
ex.error = ex.error or f"store write failed: {e}"
|
|
919
|
+
try:
|
|
920
|
+
self.bus.publish({"type": "exchange", "exchange": summarize(ex)})
|
|
921
|
+
except Exception:
|
|
922
|
+
pass
|
|
923
|
+
if not self.cfg.quiet:
|
|
924
|
+
try:
|
|
925
|
+
from .term import log_exchange
|
|
926
|
+
log_exchange(ex)
|
|
927
|
+
except Exception:
|
|
928
|
+
pass
|
|
929
|
+
|
|
930
|
+
|
|
931
|
+
def _host_is_exposed(host: str) -> bool:
|
|
932
|
+
h = (host or "").strip().strip("[]").lower()
|
|
933
|
+
if h == "localhost":
|
|
934
|
+
return False
|
|
935
|
+
try:
|
|
936
|
+
return not ipaddress.ip_address(h).is_loopback
|
|
937
|
+
except ValueError:
|
|
938
|
+
return True # "" (all interfaces) or a hostname we can't prove is loopback
|
|
939
|
+
|
|
940
|
+
|
|
941
|
+
def _merge_headers(relay: list[tuple[str, str]], extra: dict[str, str]) -> dict[str, str]:
|
|
942
|
+
"""Relay headers as a dict (duplicate names joined with ', ', except set-cookie last-wins)."""
|
|
943
|
+
out: dict[str, str] = {}
|
|
944
|
+
lower_map: dict[str, str] = {}
|
|
945
|
+
for k, v in relay:
|
|
946
|
+
lk = k.lower()
|
|
947
|
+
if lk in lower_map and lk != "set-cookie":
|
|
948
|
+
out[lower_map[lk]] = out[lower_map[lk]] + ", " + v
|
|
949
|
+
else:
|
|
950
|
+
lower_map[lk] = k
|
|
951
|
+
out[k] = v
|
|
952
|
+
for k, v in extra.items():
|
|
953
|
+
out[k] = v
|
|
954
|
+
return out
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
def _upstream_error_message(text: str) -> str | None:
|
|
958
|
+
j = parse_json(text.encode("utf-8")) if text else None
|
|
959
|
+
if not j:
|
|
960
|
+
return None
|
|
961
|
+
err = j.get("error")
|
|
962
|
+
if isinstance(err, dict) and isinstance(err.get("message"), str):
|
|
963
|
+
return err["message"][:500]
|
|
964
|
+
if isinstance(err, str):
|
|
965
|
+
return err[:500]
|
|
966
|
+
if isinstance(j.get("message"), str):
|
|
967
|
+
return j["message"][:500]
|
|
968
|
+
return None
|
|
969
|
+
|
|
970
|
+
|
|
971
|
+
# ------------------------------------------------------------------------------------------------
|
|
972
|
+
# http.server adapter
|
|
973
|
+
# ------------------------------------------------------------------------------------------------
|
|
974
|
+
class _Server(ThreadingHTTPServer):
|
|
975
|
+
daemon_threads = True
|
|
976
|
+
allow_reuse_address = True
|
|
977
|
+
block_on_close = False
|
|
978
|
+
request_queue_size = 128
|
|
979
|
+
|
|
980
|
+
def __init__(self, addr: tuple[str, int], handler: type, tap: LLMTap) -> None:
|
|
981
|
+
self.tap = tap
|
|
982
|
+
if ":" in (addr[0] or ""):
|
|
983
|
+
self.address_family = socket.AF_INET6
|
|
984
|
+
super().__init__(addr, handler)
|
|
985
|
+
|
|
986
|
+
def server_bind(self) -> None:
|
|
987
|
+
# Skip HTTPServer.server_bind's socket.getfqdn(): it can stall for seconds on some hosts.
|
|
988
|
+
socketserver.TCPServer.server_bind(self)
|
|
989
|
+
host, port = self.server_address[:2]
|
|
990
|
+
self.server_name = str(host)
|
|
991
|
+
self.server_port = int(port)
|
|
992
|
+
|
|
993
|
+
def handle_error(self, request: Any, client_address: Any) -> None: # quiet on client resets
|
|
994
|
+
pass
|
|
995
|
+
|
|
996
|
+
|
|
997
|
+
class _HandlerResponder:
|
|
998
|
+
"""Responder over a BaseHTTPRequestHandler with Content-Length or chunked framing."""
|
|
999
|
+
|
|
1000
|
+
def __init__(self, handler: "_Handler") -> None:
|
|
1001
|
+
self.h = handler
|
|
1002
|
+
self.started = False
|
|
1003
|
+
self.ended = False
|
|
1004
|
+
self.chunked = False
|
|
1005
|
+
self.no_body = False
|
|
1006
|
+
|
|
1007
|
+
def start(self, status: int, headers: dict[str, str]) -> None:
|
|
1008
|
+
if self.started:
|
|
1009
|
+
raise RuntimeError("response already started")
|
|
1010
|
+
h = self.h
|
|
1011
|
+
self.started = True
|
|
1012
|
+
self.no_body = h.command == "HEAD" or status in (204, 304) or 100 <= status < 200
|
|
1013
|
+
want_chunked = False
|
|
1014
|
+
has_length = False
|
|
1015
|
+
h.send_response(status)
|
|
1016
|
+
for k, v in headers.items():
|
|
1017
|
+
lk = k.lower()
|
|
1018
|
+
if lk == "transfer-encoding":
|
|
1019
|
+
want_chunked = "chunked" in str(v).lower()
|
|
1020
|
+
continue
|
|
1021
|
+
if lk == "content-length":
|
|
1022
|
+
has_length = True
|
|
1023
|
+
if lk == "connection":
|
|
1024
|
+
continue
|
|
1025
|
+
h.send_header(k, str(v))
|
|
1026
|
+
if self.no_body:
|
|
1027
|
+
if not has_length and status not in (204, 304) and not (100 <= status < 200):
|
|
1028
|
+
h.send_header("Content-Length", "0")
|
|
1029
|
+
elif want_chunked or not has_length:
|
|
1030
|
+
if h.request_version == "HTTP/1.0":
|
|
1031
|
+
h.close_connection = True
|
|
1032
|
+
else:
|
|
1033
|
+
self.chunked = True
|
|
1034
|
+
h.send_header("Transfer-Encoding", "chunked")
|
|
1035
|
+
if h.close_connection:
|
|
1036
|
+
h.send_header("Connection", "close")
|
|
1037
|
+
h.end_headers()
|
|
1038
|
+
h.wfile.flush()
|
|
1039
|
+
|
|
1040
|
+
def write(self, chunk: bytes) -> None:
|
|
1041
|
+
if not self.started:
|
|
1042
|
+
raise RuntimeError("write() before start()")
|
|
1043
|
+
if not chunk or self.no_body:
|
|
1044
|
+
return
|
|
1045
|
+
w = self.h.wfile
|
|
1046
|
+
if self.chunked:
|
|
1047
|
+
w.write(b"%x\r\n" % len(chunk) + bytes(chunk) + b"\r\n")
|
|
1048
|
+
else:
|
|
1049
|
+
w.write(chunk)
|
|
1050
|
+
w.flush()
|
|
1051
|
+
|
|
1052
|
+
def end(self) -> None:
|
|
1053
|
+
if self.ended:
|
|
1054
|
+
return
|
|
1055
|
+
self.ended = True
|
|
1056
|
+
if self.chunked and not self.no_body:
|
|
1057
|
+
self.h.wfile.write(b"0\r\n\r\n")
|
|
1058
|
+
self.h.wfile.flush()
|
|
1059
|
+
|
|
1060
|
+
def abort(self) -> None:
|
|
1061
|
+
"""Mark done without the terminating frame and close the connection afterwards."""
|
|
1062
|
+
self.ended = True
|
|
1063
|
+
self.h.close_connection = True
|
|
1064
|
+
|
|
1065
|
+
|
|
1066
|
+
class _BadRequest(Exception):
|
|
1067
|
+
pass
|
|
1068
|
+
|
|
1069
|
+
|
|
1070
|
+
class _Handler(BaseHTTPRequestHandler):
|
|
1071
|
+
protocol_version = "HTTP/1.1"
|
|
1072
|
+
server_version = "llmtap/" + __version__
|
|
1073
|
+
sys_version = ""
|
|
1074
|
+
timeout = 900 # reap idle keep-alive connections / stalled clients instead of leaking threads
|
|
1075
|
+
server: _Server
|
|
1076
|
+
|
|
1077
|
+
def log_message(self, format: str, *args: Any) -> None: # noqa: A002 - silence default logging
|
|
1078
|
+
pass
|
|
1079
|
+
|
|
1080
|
+
def _read_body(self) -> bytes:
|
|
1081
|
+
te = (self.headers.get("Transfer-Encoding") or "").lower()
|
|
1082
|
+
if "chunked" in te:
|
|
1083
|
+
buf = bytearray()
|
|
1084
|
+
while True:
|
|
1085
|
+
line = self.rfile.readline(65537)
|
|
1086
|
+
if not line:
|
|
1087
|
+
raise _BadRequest("truncated chunked body")
|
|
1088
|
+
size_s = line.split(b";", 1)[0].strip()
|
|
1089
|
+
try:
|
|
1090
|
+
size = int(size_s, 16)
|
|
1091
|
+
except ValueError:
|
|
1092
|
+
raise _BadRequest("bad chunk size") from None
|
|
1093
|
+
if size == 0:
|
|
1094
|
+
while True: # trailers
|
|
1095
|
+
t = self.rfile.readline(65537)
|
|
1096
|
+
if t in (b"\r\n", b"\n", b""):
|
|
1097
|
+
break
|
|
1098
|
+
return bytes(buf)
|
|
1099
|
+
data = self.rfile.read(size)
|
|
1100
|
+
if len(data) < size:
|
|
1101
|
+
raise _BadRequest("truncated chunk")
|
|
1102
|
+
buf += data
|
|
1103
|
+
self.rfile.readline(3) # CRLF after chunk
|
|
1104
|
+
cl = self.headers.get("Content-Length")
|
|
1105
|
+
if cl:
|
|
1106
|
+
try:
|
|
1107
|
+
n = int(cl)
|
|
1108
|
+
except ValueError:
|
|
1109
|
+
raise _BadRequest("bad Content-Length") from None
|
|
1110
|
+
if n < 0:
|
|
1111
|
+
raise _BadRequest("bad Content-Length")
|
|
1112
|
+
data = self.rfile.read(n)
|
|
1113
|
+
if len(data) < n:
|
|
1114
|
+
raise _BadRequest("truncated body")
|
|
1115
|
+
return data
|
|
1116
|
+
return b""
|
|
1117
|
+
|
|
1118
|
+
def _dispatch(self) -> None:
|
|
1119
|
+
r = _HandlerResponder(self)
|
|
1120
|
+
try:
|
|
1121
|
+
try:
|
|
1122
|
+
body = self._read_body()
|
|
1123
|
+
except _BadRequest as e:
|
|
1124
|
+
self.close_connection = True
|
|
1125
|
+
msg = json.dumps({"error": {"message": f"llmtap: bad request: {e}", "type": "llmtap_bad_request"}})
|
|
1126
|
+
data = msg.encode()
|
|
1127
|
+
r.start(400, {"Content-Type": "application/json", "Content-Length": str(len(data))})
|
|
1128
|
+
r.write(data)
|
|
1129
|
+
r.end()
|
|
1130
|
+
return
|
|
1131
|
+
headers = {k: v for k, v in self.headers.items()}
|
|
1132
|
+
self.server.tap.handle(self.command, self.path, headers, body, r)
|
|
1133
|
+
if not r.started:
|
|
1134
|
+
data = b'{"error":{"message":"llmtap: no response produced","type":"llmtap_error"}}'
|
|
1135
|
+
r.start(500, {"Content-Type": "application/json", "Content-Length": str(len(data))})
|
|
1136
|
+
r.write(data)
|
|
1137
|
+
r.end()
|
|
1138
|
+
elif not r.ended:
|
|
1139
|
+
r.end()
|
|
1140
|
+
except _CLIENT_GONE:
|
|
1141
|
+
self.close_connection = True
|
|
1142
|
+
except Exception:
|
|
1143
|
+
self.close_connection = True
|
|
1144
|
+
if not r.started:
|
|
1145
|
+
try:
|
|
1146
|
+
data = b'{"error":{"message":"llmtap internal error","type":"llmtap_error"}}'
|
|
1147
|
+
r.start(500, {"Content-Type": "application/json", "Content-Length": str(len(data))})
|
|
1148
|
+
r.write(data)
|
|
1149
|
+
r.end()
|
|
1150
|
+
except Exception:
|
|
1151
|
+
pass
|
|
1152
|
+
|
|
1153
|
+
do_GET = do_POST = do_PUT = do_PATCH = do_DELETE = do_OPTIONS = do_HEAD = _dispatch
|