llmtap-proxy 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
llmtap/proxy.py ADDED
@@ -0,0 +1,1153 @@
1
+ """The core: an HTTP proxy that records, prices, caches, replays, guards and budgets LLM calls.
2
+
3
+ ``LLMTap.handle`` implements the whole per-request pipeline against an abstract ``Responder`` so
4
+ the dashboard and tests don't care about ``http.server``. ``_Handler`` adapts
5
+ ``BaseHTTPRequestHandler`` (HTTP/1.1, keep-alive, Content-Length / chunked framing) to it.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import gzip
10
+ import http.client
11
+ import ipaddress
12
+ import json
13
+ import queue
14
+ import re
15
+ import socket
16
+ import socketserver
17
+ import ssl
18
+ import sys
19
+ import threading
20
+ import time
21
+ import zlib
22
+ from collections.abc import Mapping
23
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
24
+ from typing import Any, Protocol
25
+ from urllib.parse import parse_qsl, urlsplit
26
+
27
+ from . import __version__
28
+ from .budget import Budget
29
+ from .cache import ResponseCache, cache_key
30
+ from .cassette import Cassette
31
+ from .config import Config
32
+ from .guard import apply_guard
33
+ from .models import CachedResponse, Exchange, Route, Usage
34
+ from .pricing import cost_usd
35
+ from .providers import (
36
+ body_error,
37
+ error_body,
38
+ estimate_usage,
39
+ extract_text,
40
+ extract_usage,
41
+ forward_headers,
42
+ is_stream_request,
43
+ parse_json,
44
+ parse_sse,
45
+ model_from_path,
46
+ request_model,
47
+ resolve_route,
48
+ response_model,
49
+ sse_error,
50
+ tags_from_headers,
51
+ )
52
+ from .store import Store
53
+
54
+ __all__ = ["LLMTap", "EventBus", "Responder", "summarize"]
55
+
56
+ # Response headers never relayed to the client (we do our own framing).
57
+ _RESP_HOP = frozenset({
58
+ "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailers",
59
+ "transfer-encoding", "upgrade", "content-length",
60
+ })
61
+ # Headers never stored for replay (body is stored decoded, framing is recomputed).
62
+ _STORE_DROP = _RESP_HOP | {"content-encoding", "date", "server", "x-llmtap-id", "x-llmtap-source",
63
+ "access-control-allow-origin", "access-control-expose-headers"}
64
+ _CLIENT_GONE = (BrokenPipeError, ConnectionResetError, ConnectionAbortedError, TimeoutError)
65
+ # Per-exchange cap (characters) for what is *stored* (db / cache / cassette). Relayed bytes are never capped.
66
+ MAX_STORED_CHARS = 2 * 1024 * 1024
67
+ _CORS_BASE = {
68
+ "Access-Control-Allow-Origin": "*",
69
+ "Access-Control-Expose-Headers": "x-llmtap-id, x-llmtap-source, request-id, x-request-id",
70
+ }
71
+ _INTERNAL = "/_llmtap"
72
+
73
+
74
+ class Responder(Protocol):
75
+ """Minimal response sink used by the pipeline and the dashboard."""
76
+
77
+ def start(self, status: int, headers: dict[str, str]) -> None: ...
78
+ def write(self, chunk: bytes) -> None: ... # chunk frame when started with transfer-encoding: chunked
79
+ def end(self) -> None: ...
80
+
81
+
82
+ class ClientGone(Exception):
83
+ """The downstream client disconnected while we were writing to it."""
84
+
85
+
86
+ # ------------------------------------------------------------------------------------------------
87
+ # event bus
88
+ # ------------------------------------------------------------------------------------------------
89
+ class EventBus:
90
+ """Thread-safe pub/sub fan-out; each subscriber gets its own bounded Queue (slow ones drop)."""
91
+
92
+ def __init__(self, maxsize: int = 1000) -> None:
93
+ self._lock = threading.Lock()
94
+ self._subs: list[queue.Queue] = []
95
+ self._maxsize = maxsize
96
+
97
+ def publish(self, event: dict) -> None:
98
+ with self._lock:
99
+ subs = list(self._subs)
100
+ for q in subs:
101
+ try:
102
+ q.put_nowait(event)
103
+ except queue.Full:
104
+ pass
105
+
106
+ def subscribe(self) -> queue.Queue:
107
+ q: queue.Queue = queue.Queue(maxsize=self._maxsize)
108
+ with self._lock:
109
+ self._subs.append(q)
110
+ return q
111
+
112
+ def unsubscribe(self, q: queue.Queue) -> None:
113
+ with self._lock:
114
+ try:
115
+ self._subs.remove(q)
116
+ except ValueError:
117
+ pass
118
+
119
+ def subscriber_count(self) -> int:
120
+ with self._lock:
121
+ return len(self._subs)
122
+
123
+
124
+ def summarize(ex: Exchange) -> dict[str, Any]:
125
+ """Exchange dict minus bodies (dashboard's richer summary when available)."""
126
+ try:
127
+ from .dashboard import summary # lazy: dashboard is optional
128
+ return summary(ex)
129
+ except Exception:
130
+ d = ex.to_dict()
131
+ d.pop("request_body", None)
132
+ d.pop("response_body", None)
133
+ return d
134
+
135
+
136
+ # ------------------------------------------------------------------------------------------------
137
+ # helpers
138
+ # ------------------------------------------------------------------------------------------------
139
+ class _Tracked:
140
+ """Wraps a Responder: remembers whether it started, injects headers, maps disconnects."""
141
+
142
+ def __init__(self, inner: Responder, extra: dict[str, str] | None = None) -> None:
143
+ self.inner = inner
144
+ self.extra = extra or {}
145
+ self.started = False
146
+ self.ended = False
147
+
148
+ def start(self, status: int, headers: dict[str, str]) -> None:
149
+ h = dict(headers)
150
+ low = {k.lower() for k in h}
151
+ for k, v in self.extra.items():
152
+ if k.lower() not in low:
153
+ h[k] = v
154
+ self.started = True
155
+ try:
156
+ self.inner.start(status, h)
157
+ except _CLIENT_GONE as e:
158
+ raise ClientGone(str(e)) from e
159
+
160
+ def write(self, chunk: bytes) -> None:
161
+ try:
162
+ self.inner.write(chunk)
163
+ except _CLIENT_GONE as e:
164
+ raise ClientGone(str(e)) from e
165
+
166
+ def end(self) -> None:
167
+ if self.ended:
168
+ return
169
+ self.ended = True
170
+ try:
171
+ self.inner.end()
172
+ except _CLIENT_GONE as e:
173
+ raise ClientGone(str(e)) from e
174
+
175
+ def abort(self) -> None:
176
+ """Drop the connection without completing the body (so the client sees truncation)."""
177
+ self.ended = True
178
+ fn = getattr(self.inner, "abort", None)
179
+ if fn is not None:
180
+ try:
181
+ fn()
182
+ except Exception:
183
+ pass
184
+
185
+
186
+ def _lower(headers: Mapping[str, str] | None) -> dict[str, str]:
187
+ out: dict[str, str] = {}
188
+ try:
189
+ for k, v in (headers or {}).items():
190
+ out[str(k).lower()] = str(v)
191
+ except Exception:
192
+ pass
193
+ return out
194
+
195
+
196
+ def _try_decode(raw: bytes, enc: str) -> bytes | None:
197
+ """Undo content-encoding; None when we can't (br, zstd, compress, corrupt gzip...)."""
198
+ try:
199
+ if enc in ("gzip", "x-gzip"):
200
+ return gzip.decompress(raw)
201
+ if enc == "deflate":
202
+ try:
203
+ return zlib.decompress(raw)
204
+ except zlib.error:
205
+ return zlib.decompress(raw, -zlib.MAX_WBITS)
206
+ except Exception:
207
+ return None
208
+ return None
209
+
210
+
211
+ _BINARY_PREFIXES = ("audio/", "image/", "video/", "font/")
212
+ _BINARY_TYPES = frozenset({"application/octet-stream", "application/zip", "application/pdf", "application/gzip",
213
+ "application/x-tar", "application/x-zip-compressed", "application/x-protobuf"})
214
+
215
+
216
+ def _decode_response(raw: bytes, encoding: str, ctype: str) -> tuple[str, bool]:
217
+ """(text, parseable). Binary / undecodable bodies yield a short placeholder and parseable=False
218
+ so we never dump megabytes of mojibake into the db or cache (the client still gets the raw bytes)."""
219
+ if not raw:
220
+ return "", True
221
+ enc = (encoding or "").strip().lower()
222
+ data = raw
223
+ if enc not in ("", "identity"):
224
+ dec = _try_decode(raw, enc)
225
+ if dec is None:
226
+ return (f"[response body not stored: content-encoding '{enc}' is not decodable by llmtap, "
227
+ f"{len(raw)} bytes]"), False
228
+ data = dec
229
+ main = (ctype or "").split(";")[0].strip().lower()
230
+ if main.startswith(_BINARY_PREFIXES) or main in _BINARY_TYPES:
231
+ return f"[binary response body not stored: {main}, {len(data)} bytes]", False
232
+ try:
233
+ return data.decode("utf-8"), True
234
+ except UnicodeDecodeError:
235
+ if any(t in main for t in ("json", "text", "event-stream", "xml")):
236
+ return data.decode("utf-8", "replace"), True
237
+ return f"[binary response body not stored: {main or 'unknown content-type'}, {len(data)} bytes]", False
238
+
239
+
240
+ def _decode_request(body: bytes, ctype: str) -> tuple[str, bool]:
241
+ """(text to store, truncated). Multipart / binary request bodies are stored as a short placeholder."""
242
+ main = (ctype or "").split(";")[0].strip().lower()
243
+ if main.startswith("multipart/") or main in _BINARY_TYPES or main.startswith(_BINARY_PREFIXES):
244
+ return f"[{main} request body not stored: {len(body)} bytes]", False
245
+ try:
246
+ text = body[:MAX_STORED_CHARS * 4].decode("utf-8")
247
+ except UnicodeDecodeError as e:
248
+ if e.start > MAX_STORED_CHARS * 4 - 8 or len(body) > MAX_STORED_CHARS * 4:
249
+ text = body[:MAX_STORED_CHARS * 4].decode("utf-8", "ignore")
250
+ else:
251
+ return f"[binary request body not stored: {main or 'unknown content-type'}, {len(body)} bytes]", False
252
+ if len(text) > MAX_STORED_CHARS or len(body) > MAX_STORED_CHARS * 4:
253
+ return text[:MAX_STORED_CHARS] + f"\n[llmtap: truncated, {len(body)} bytes in total]", True
254
+ return text, False
255
+
256
+
257
+ _SENSITIVE_QS = frozenset({"key", "api_key", "apikey", "api-key", "token", "access_token", "x-api-key",
258
+ "signature", "sig", "auth", "password", "secret", "client_secret"})
259
+ _SENSITIVE_QS_RE = re.compile(r"([?&](?:" + "|".join(sorted(re.escape(k) for k in _SENSITIVE_QS)) + r")=)[^&#]*",
260
+ re.IGNORECASE)
261
+ _CRED_HEADERS = ("authorization", "proxy-authorization", "x-api-key", "api-key", "x-goog-api-key", "cookie",
262
+ "x-auth-token", "openai-api-key", "anthropic-auth-token")
263
+ REDACTED = "[REDACTED]"
264
+
265
+
266
+ def _mask_path(path: str) -> str:
267
+ """Mask secret-looking query parameters (``?key=AIza...``) before the path is ever stored."""
268
+ return _SENSITIVE_QS_RE.sub(lambda m: m.group(1) + REDACTED, path) if "?" in path else path
269
+
270
+
271
+ def _credentials(hdrs: Mapping[str, str], path: str) -> list[str]:
272
+ """Credential strings of this request (>= 8 chars), longest first, so they can be scrubbed from anything stored."""
273
+ vals: set[str] = set()
274
+ for name in _CRED_HEADERS:
275
+ v = (hdrs.get(name) or "").strip()
276
+ if not v:
277
+ continue
278
+ vals.add(v)
279
+ if name == "cookie":
280
+ for part in v.split(";"):
281
+ vals.add(part.strip().partition("=")[2].strip())
282
+ else:
283
+ vals.add(v.split(None, 1)[-1]) # "Bearer abc" -> "abc"
284
+ for k, v in parse_qsl(path.partition("?")[2], keep_blank_values=True):
285
+ if k.lower() in _SENSITIVE_QS:
286
+ vals.add(v)
287
+ return sorted({v for v in vals if len(v) >= 8}, key=len, reverse=True)
288
+
289
+
290
+ def _scrub(text: str, secrets: list[str]) -> str:
291
+ for sec in secrets:
292
+ if sec in text:
293
+ text = text.replace(sec, REDACTED)
294
+ return text
295
+
296
+
297
+ def _storable_headers(headers: list[tuple[str, str]], secrets: list[str] | None = None) -> dict[str, str]:
298
+ out: dict[str, str] = {}
299
+ for k, v in headers:
300
+ lk = k.lower()
301
+ if lk in _STORE_DROP or lk.startswith("x-llmtap-") or lk == "set-cookie" or lk in _CRED_HEADERS:
302
+ continue
303
+ if secrets and any(sec in v for sec in secrets): # upstream echoed a credential back
304
+ continue
305
+ out[lk] = v
306
+ return out
307
+
308
+
309
+ def _note(ex: Exchange, text: str) -> None:
310
+ """Attach a non-fatal note (e.g. truncation); folded into ``ex.error`` when the exchange is sealed."""
311
+ ex.__dict__.setdefault("_notes", []).append(text)
312
+
313
+
314
+ def _seal(ex: Exchange) -> None:
315
+ """Final pass before an exchange is persisted: fold notes into ``error``, scrub credentials."""
316
+ notes = ex.__dict__.pop("_notes", None)
317
+ if notes:
318
+ ex.error = "; ".join(x for x in [ex.error, *notes] if x)
319
+ secrets = ex.__dict__.pop("_secrets", None)
320
+ if secrets:
321
+ for f in ("path", "request_body", "response_body", "error", "output_text"):
322
+ v = getattr(ex, f, None)
323
+ if v:
324
+ setattr(ex, f, _scrub(v, secrets))
325
+
326
+
327
+ _LLM_TAIL = re.compile(r"(?:/(?:chat/completions|completions|embeddings)|:(?:stream)?[gG]enerateContent|(?:^|/)v\d+[a-z0-9]*/(?:messages|complete|responses))$")
328
+
329
+
330
+ def _is_llm_call(method: str, upstream_path: str) -> bool:
331
+ """True for billable generation endpoints (chat / messages / responses / embeddings / completions).
332
+ Everything else (models, files, batches, count_tokens, ...) is forwarded untouched: recorded, but
333
+ never cached, budget-checked, loop-checked or priced."""
334
+ if method != "POST":
335
+ return False
336
+ return bool(_LLM_TAIL.search(upstream_path.split("?", 1)[0].rstrip("/")))
337
+
338
+
339
+ def _is_json_doc(body: bytes) -> bool:
340
+ """Cheap check that a (non-object) body is a JSON document, e.g. an array."""
341
+ head = body.lstrip()[:1]
342
+ if head not in (b"[", b'"'):
343
+ return False
344
+ try:
345
+ json.loads(body.decode("utf-8-sig"))
346
+ return True
347
+ except Exception:
348
+ return False
349
+
350
+
351
+ def _split_sse(body: str) -> list[str]:
352
+ """Split a stored SSE transcript into whole events (each ending with a blank line)."""
353
+ text = body.replace("\r\n", "\n")
354
+ parts = [p for p in text.split("\n\n") if p.strip()]
355
+ return [p + "\n\n" for p in parts]
356
+
357
+
358
+ def _upstream_target(route: Route) -> tuple[str, str, int | None, str]:
359
+ """(scheme, host, port, full_path) for a route, honoring base paths like https://x/api."""
360
+ u = urlsplit(route.upstream_base)
361
+ scheme = (u.scheme or "http").lower()
362
+ base_path = (u.path or "").rstrip("/")
363
+ up = route.upstream_path or "/"
364
+ if not up.startswith("/"):
365
+ up = "/" + up
366
+ return scheme, u.hostname or "localhost", u.port, base_path + up
367
+
368
+
369
+ def _is_streaming_response(ctype: str, req_stream: bool, has_length: bool) -> bool:
370
+ c = ctype.lower()
371
+ if "text/event-stream" in c or "ndjson" in c or "jsonl" in c:
372
+ return True
373
+ return req_stream and not has_length and "json" not in c
374
+
375
+
376
+ # ------------------------------------------------------------------------------------------------
377
+ # the proxy
378
+ # ------------------------------------------------------------------------------------------------
379
+ class LLMTap:
380
+ """The llmtap proxy: shared state + per-request pipeline + server lifecycle."""
381
+
382
+ def __init__(self, cfg: Config, store: Store | None = None) -> None:
383
+ self.cfg = cfg
384
+ self.store: Store = store if store is not None else Store(cfg.resolved_db_path())
385
+ self.cache: ResponseCache | None = (
386
+ ResponseCache(cfg.resolved_db_path(), cfg.cache_ttl_s) if cfg.cache else None
387
+ )
388
+ self.cassette: Cassette | None = Cassette(cfg.cassette, cfg.mode) if cfg.cassette else None
389
+ self.budget: Budget = Budget(cfg, self.store.spent_since)
390
+ self.bus: EventBus = EventBus()
391
+ self.port: int = cfg.port
392
+ self.started_at: float = time.time()
393
+ self._server: _Server | None = None
394
+ self._thread: threading.Thread | None = None
395
+ self._serving = threading.Event()
396
+ self._closed = False
397
+
398
+ # ---------------------------------------------------------------- lifecycle
399
+ @property
400
+ def base_url(self) -> str:
401
+ host = self.cfg.host
402
+ if host in ("", "0.0.0.0", "::"):
403
+ host = "127.0.0.1"
404
+ if ":" in host and not host.startswith("["):
405
+ host = f"[{host}]"
406
+ return f"http://{host}:{self.port}"
407
+
408
+ @property
409
+ def is_exposed(self) -> bool:
410
+ """True when the listen address is reachable beyond this machine (0.0.0.0, ::, a LAN address...)."""
411
+ return _host_is_exposed(self.cfg.host)
412
+
413
+ def _bind(self) -> "_Server":
414
+ if self._server is None:
415
+ self._server = _Server((self.cfg.host, self.cfg.port), _Handler, self)
416
+ self.port = int(self._server.server_address[1])
417
+ if self.is_exposed:
418
+ self._warn_exposed()
419
+ return self._server
420
+
421
+ def _warn_exposed(self) -> None:
422
+ """Prominent stderr warning: prompts, responses and the dashboard are served without authentication."""
423
+ try:
424
+ from .term import color_enabled, paint
425
+ msg = (f"WARNING: llmtap is listening on {self.cfg.host or '0.0.0.0'}:{self.port} and is exposed to "
426
+ "your network. The dashboard and every recorded prompt/response are readable by anyone who "
427
+ "can reach this port, and nothing is authenticated. Bind to 127.0.0.1 unless you mean this.")
428
+ sys.stderr.write(paint(msg, "bold", "yellow", color=color_enabled(sys.stderr)) + "\n")
429
+ sys.stderr.flush()
430
+ except Exception:
431
+ pass
432
+
433
+ def serve_forever(self) -> None:
434
+ """Serve in the calling thread until shutdown() (or KeyboardInterrupt propagates)."""
435
+ srv = self._bind()
436
+ self._serving.set()
437
+ try:
438
+ srv.serve_forever(poll_interval=0.25)
439
+ finally:
440
+ self._serving.clear()
441
+
442
+ def start_background(self) -> "LLMTap":
443
+ """Bind now (so .port/.base_url are valid on return) and serve in a daemon thread."""
444
+ self._bind()
445
+ self._thread = threading.Thread(target=self.serve_forever, name="llmtap-proxy", daemon=True)
446
+ self._thread.start()
447
+ self._serving.wait(5)
448
+ return self
449
+
450
+ def shutdown(self) -> None:
451
+ """Stop serving and close the server socket, store and cache. Idempotent."""
452
+ if self._closed:
453
+ return
454
+ self._closed = True
455
+ srv = self._server
456
+ if srv is not None:
457
+ if self._serving.is_set():
458
+ try:
459
+ srv.shutdown()
460
+ except Exception:
461
+ pass
462
+ try:
463
+ srv.server_close()
464
+ except Exception:
465
+ pass
466
+ if self._thread is not None:
467
+ self._thread.join(timeout=5)
468
+ for obj in (self.cache, self.store):
469
+ close = getattr(obj, "close", None)
470
+ if close is not None:
471
+ try:
472
+ close()
473
+ except Exception:
474
+ pass
475
+
476
+ def __enter__(self) -> "LLMTap":
477
+ return self.start_background()
478
+
479
+ def __exit__(self, *exc: Any) -> None:
480
+ self.shutdown()
481
+
482
+ # ---------------------------------------------------------------- pipeline
483
+ def handle(self, method: str, path: str, headers: Mapping[str, str], body: bytes,
484
+ respond: Responder) -> Exchange:
485
+ """Process one request end-to-end and return the (recorded) Exchange."""
486
+ method = (method or "GET").upper()
487
+ path = path or "/"
488
+ hdrs = _lower(headers)
489
+ body = body or b""
490
+ bare = path.split("?", 1)[0]
491
+
492
+ # -- internal routes (not recorded) -------------------------------------------------
493
+ if bare == _INTERNAL or bare.startswith(_INTERNAL + "/"):
494
+ ex = Exchange(method=method, path=path, provider="llmtap", source="internal")
495
+ self._internal(method, path, headers, body, respond, ex)
496
+ return ex
497
+ if bare == "/" and method in ("GET", "HEAD"):
498
+ ex = Exchange(method=method, path=path, provider="llmtap", source="internal", status=302)
499
+ self._simple(respond, 302, b"", {"Location": _INTERNAL + "/"})
500
+ return ex
501
+ if bare == "/favicon.ico":
502
+ ex = Exchange(method=method, path=path, provider="llmtap", source="internal", status=204)
503
+ self._simple(respond, 204, b"", {})
504
+ return ex
505
+ if method == "OPTIONS" and "access-control-request-method" in hdrs:
506
+ ex = Exchange(method=method, path=path, provider="llmtap", source="internal", status=204)
507
+ self._simple(respond, 204, b"", {
508
+ "Access-Control-Allow-Origin": "*",
509
+ "Access-Control-Allow-Methods": "GET, POST, PUT, PATCH, DELETE, OPTIONS",
510
+ "Access-Control-Allow-Headers": hdrs.get("access-control-request-headers") or "*",
511
+ "Access-Control-Max-Age": "86400",
512
+ })
513
+ return ex
514
+
515
+ # -- proxied LLM call ---------------------------------------------------------------
516
+ ex = Exchange(method=method, path=_mask_path(path))
517
+ ex.__dict__["_secrets"] = _credentials(hdrs, path)
518
+ t0 = time.perf_counter()
519
+ out = _Tracked(respond, dict(_CORS_BASE))
520
+ style = "openai"
521
+ try:
522
+ route = resolve_route(path, hdrs, self.cfg)
523
+ style = route.style
524
+ ex.provider, ex.style = route.provider, route.style
525
+ self._pipeline(ex, route, method, hdrs, body, out, t0)
526
+ except ClientGone:
527
+ ex.error = ex.error or "client disconnected"
528
+ except Exception as e: # never let a bug kill the connection silently
529
+ ex.error = f"llmtap internal error: {type(e).__name__}: {e}"
530
+ if ex.source == "upstream":
531
+ ex.source = "error"
532
+ if not out.started:
533
+ ex.status = 500
534
+ self._error(out, ex, style, 500, ex.error, "llmtap_internal_error")
535
+ else:
536
+ out.abort()
537
+ ex.latency_ms = round((time.perf_counter() - t0) * 1000.0, 2)
538
+ self._finish(ex)
539
+ return ex
540
+
541
+ def _pipeline(self, ex: Exchange, route: Route, method: str, hdrs: dict[str, str], body: bytes,
542
+ out: _Tracked, t0: float) -> None:
543
+ cfg = self.cfg
544
+ req = parse_json(body)
545
+ ex.model = request_model(req) or model_from_path(route.upstream_path)
546
+ stream_req = is_stream_request(req)
547
+ ex.tags = tags_from_headers(hdrs)
548
+
549
+ llm = _is_llm_call(method, route.upstream_path)
550
+ ctype = hdrs.get("content-type", "")
551
+
552
+ # guard. Only JSON documents may ever be rewritten: multipart / binary / unparseable bodies are
553
+ # scanned as text (warn / block) but forwarded byte-for-byte even in redact mode.
554
+ send_body = body
555
+ mode = cfg.guard or "off"
556
+ if body and mode != "off":
557
+ if mode == "redact" and req is None and not _is_json_doc(body):
558
+ mode = "warn"
559
+ send_body, findings, blocked = apply_guard(body, mode, cfg.guard_pii)
560
+ ex.guard_findings = [f.to_dict() if hasattr(f, "to_dict") else dict(f) for f in findings]
561
+ if blocked:
562
+ ex.request_body, trunc = _decode_request(body, ctype)
563
+ if trunc:
564
+ _note(ex, f"request body truncated to {MAX_STORED_CHARS} characters in llmtap storage")
565
+ kinds = ", ".join(sorted({str(f.get("kind", "?")) for f in ex.guard_findings}))
566
+ ex.source = "blocked"
567
+ msg = (f"llmtap guard blocked this request: detected {kinds} in the prompt. "
568
+ "Remove the secret, or run llmtap with --guard redact / warn.")
569
+ self._error(out, ex, route.style, 400, msg, "llmtap_guard_blocked")
570
+ return
571
+ if send_body is not body:
572
+ req = parse_json(send_body) if req is not None else req
573
+ ex.request_body, trunc = _decode_request(send_body, ctype)
574
+ if trunc:
575
+ _note(ex, f"request body truncated to {MAX_STORED_CHARS} characters in llmtap storage (forwarded in full)")
576
+
577
+ cacheable = llm
578
+ key = cache_key(route.provider, route.upstream_path, send_body) if cacheable else ""
579
+
580
+ # budget / loop breaker (generation calls only: models/files/batches/count_tokens are not chat traffic)
581
+ if cacheable:
582
+ verdict = self.budget.check(key)
583
+ if not verdict.allowed:
584
+ ex.source = verdict.kind or "budget"
585
+ err_type = "llmtap_loop_detected" if ex.source == "loop" else "llmtap_budget_exceeded"
586
+ self._error(out, ex, route.style, 429, verdict.reason or "llmtap budget exceeded", err_type)
587
+ return
588
+
589
+ # cassette / cache
590
+ if cacheable:
591
+ stored: CachedResponse | None = None
592
+ if self.cassette is not None:
593
+ stored = self.cassette.lookup(key)
594
+ if stored is not None:
595
+ ex.source = "cassette"
596
+ if stored is None and self.cache is not None:
597
+ stored = self.cache.get(key)
598
+ if stored is not None:
599
+ ex.source = "cache"
600
+ if stored is not None:
601
+ self._replay(ex, route, req, stored, out, t0)
602
+ return
603
+ if self.cassette is not None and self.cassette.mode == "replay" and not cfg.allow_miss:
604
+ ex.source = "error"
605
+ msg = ("llmtap cassette miss (replay mode): no recording for this request. "
606
+ "Re-record with --mode record")
607
+ self._error(out, ex, route.style, 409, msg, "llmtap_cassette_miss")
608
+ return
609
+
610
+ self._forward(ex, route, method, hdrs, send_body, req, stream_req, key, out, t0)
611
+
612
+ # ---------------------------------------------------------------- upstream
613
+ def _forward(self, ex: Exchange, route: Route, method: str, hdrs: dict[str, str], body: bytes,
614
+ req: dict | None, stream_req: bool, key: str, out: _Tracked, t0: float) -> None:
615
+ llm = _is_llm_call(method, route.upstream_path)
616
+ scheme, host, port, full_path = _upstream_target(route)
617
+ timeout = float(self.cfg.upstream_timeout_s or 600.0)
618
+ conn: http.client.HTTPConnection
619
+ if scheme == "https":
620
+ conn = http.client.HTTPSConnection(host, port, timeout=timeout, context=ssl.create_default_context())
621
+ else:
622
+ conn = http.client.HTTPConnection(host, port, timeout=timeout)
623
+ fwd = forward_headers(hdrs)
624
+ try:
625
+ try:
626
+ conn.request(method, full_path, body=body if (body or method in ("POST", "PUT", "PATCH")) else None,
627
+ headers=fwd)
628
+ resp = conn.getresponse()
629
+ except (OSError, http.client.HTTPException) as e:
630
+ ex.source = "error"
631
+ msg = f"llmtap: upstream {route.upstream_base} unreachable: {type(e).__name__}: {e}"
632
+ status = 504 if isinstance(e, (socket.timeout, TimeoutError)) else 502
633
+ self._error(out, ex, route.style, status, msg, "llmtap_upstream_error")
634
+ return
635
+
636
+ ex.status = resp.status
637
+ up_headers = resp.getheaders()
638
+ low = {k.lower(): v for k, v in up_headers}
639
+ ctype = low.get("content-type", "")
640
+ encoding = low.get("content-encoding", "")
641
+ relay = [(k, v) for k, v in up_headers if k.lower() not in _RESP_HOP]
642
+ client_h = _merge_headers(relay, {"x-llmtap-id": ex.id, "x-llmtap-source": "upstream"})
643
+ bodyless = method == "HEAD" or resp.status in (204, 304) or resp.status < 200
644
+ if bodyless and resp.status != 204 and "content-length" in low:
645
+ client_h["Content-Length"] = low["content-length"] # HEAD / 304 describe the entity: keep its length
646
+ streaming = not bodyless and _is_streaming_response(ctype, stream_req, "content-length" in low)
647
+ ex.streamed = streaming
648
+ raw = bytearray()
649
+ client_gone = False
650
+ finalized = False
651
+ upstream_err: str | None = None
652
+
653
+ if streaming:
654
+ client_h["Transfer-Encoding"] = "chunked"
655
+ out.start(resp.status, client_h)
656
+ while True:
657
+ try:
658
+ chunk = resp.read1(65536)
659
+ except (OSError, http.client.HTTPException) as e:
660
+ upstream_err = f"upstream read error: {type(e).__name__}: {e}"
661
+ break
662
+ if not chunk:
663
+ break
664
+ if ex.ttft_ms is None:
665
+ ex.ttft_ms = round((time.perf_counter() - t0) * 1000.0, 2)
666
+ raw += chunk
667
+ try:
668
+ out.write(chunk)
669
+ except ClientGone:
670
+ client_gone = True
671
+ break
672
+ if client_gone or upstream_err:
673
+ out.abort()
674
+ else:
675
+ self._finalize(ex, route, req, key, llm, bytes(raw), encoding, ctype, streaming, up_headers)
676
+ finalized = True
677
+ try:
678
+ out.end()
679
+ except ClientGone:
680
+ client_gone = True
681
+ else:
682
+ try:
683
+ data = resp.read() if not bodyless else b""
684
+ except (OSError, http.client.HTTPException) as e:
685
+ ex.source = "error"
686
+ msg = f"llmtap: upstream read failed: {type(e).__name__}: {e}"
687
+ status = 504 if isinstance(e, (socket.timeout, TimeoutError)) else 502
688
+ self._error(out, ex, route.style, status, msg, "llmtap_upstream_error")
689
+ return
690
+ raw += data
691
+ ex.ttft_ms = round((time.perf_counter() - t0) * 1000.0, 2)
692
+ if not bodyless:
693
+ client_h["Content-Length"] = str(len(raw))
694
+ self._finalize(ex, route, req, key, llm, bytes(raw), encoding, ctype, streaming, up_headers)
695
+ finalized = True
696
+ try:
697
+ out.start(resp.status, client_h)
698
+ if raw and not bodyless:
699
+ out.write(bytes(raw))
700
+ out.end()
701
+ except ClientGone:
702
+ client_gone = True
703
+ out.abort()
704
+ finally:
705
+ try:
706
+ conn.close()
707
+ except Exception:
708
+ pass
709
+
710
+ if not finalized:
711
+ self._finalize(ex, route, req, key, llm, bytes(raw), encoding, ctype, streaming, up_headers,
712
+ client_gone, upstream_err)
713
+ elif client_gone:
714
+ ex.error = "client disconnected"
715
+
716
+ def _finalize(self, ex: Exchange, route: Route, req: dict | None, key: str, llm: bool, raw: bytes,
717
+ encoding: str, ctype: str, streaming: bool, up_headers: list[tuple[str, str]],
718
+ client_gone: bool = False, upstream_err: str | None = None) -> None:
719
+ """Usage, cost, budget, cache and cassette for a finished upstream response. Runs *before* the last
720
+ byte reaches the client (when possible) so a client that immediately sends its next request already
721
+ sees the updated budget / cache."""
722
+ raw = bytes(raw)
723
+ text, parseable = _decode_response(bytes(raw), encoding, ctype)
724
+ ex.response_body = text
725
+ if client_gone:
726
+ ex.error = "client disconnected"
727
+ elif upstream_err:
728
+ ex.error = upstream_err
729
+ if ex.status >= 400 and not ex.error:
730
+ ex.error = _upstream_error_message(text) or f"upstream HTTP {ex.status}"
731
+
732
+ in_band_err = self._account(ex, route, req, text, streaming, llm=llm, parseable=parseable)
733
+ ex.cost_usd = self._price(ex) if ex.usage.total() else 0.0
734
+ try:
735
+ self.budget.add_spend(ex.cost_usd)
736
+ except Exception:
737
+ pass
738
+
739
+ oversize = len(text) > MAX_STORED_CHARS or len(ex.output_text) > MAX_STORED_CHARS
740
+ if oversize:
741
+ ex.response_body = text[:MAX_STORED_CHARS] + f"\n[llmtap: truncated, {len(raw)} bytes in total]"
742
+ ex.output_text = ex.output_text[:MAX_STORED_CHARS]
743
+ _note(ex, f"response body truncated to {MAX_STORED_CHARS} characters in llmtap storage "
744
+ "(relayed to the client in full; not cached)")
745
+
746
+ if (200 <= ex.status < 300 and not (streaming and client_gone) and not upstream_err and key and parseable
747
+ and not oversize and not in_band_err):
748
+ secrets = ex.__dict__.get("_secrets") or []
749
+ cached = CachedResponse(status=ex.status, headers=_storable_headers(up_headers, secrets),
750
+ body=_scrub(text, secrets), streamed=streaming, cost_usd=ex.cost_usd)
751
+ if self.cache is not None:
752
+ try:
753
+ self.cache.put(key, cached)
754
+ except Exception as e:
755
+ ex.error = ex.error or f"cache write failed: {e}"
756
+ if self.cassette is not None and self.cassette.should_record():
757
+ try:
758
+ rec_req: Any = req if req is not None else ex.request_body
759
+ if secrets and isinstance(rec_req, dict):
760
+ rec_req = json.loads(_scrub(json.dumps(rec_req, ensure_ascii=False), secrets))
761
+ self.cassette.record(key, rec_req, cached)
762
+ except Exception as e:
763
+ ex.error = ex.error or f"cassette write failed: {e}"
764
+
765
+ # ---------------------------------------------------------------- replay
766
+ def _replay(self, ex: Exchange, route: Route, req: dict | None, stored: CachedResponse,
767
+ out: _Tracked, t0: float) -> None:
768
+ ex.status = int(stored.status)
769
+ ex.streamed = bool(stored.streamed)
770
+ ex.response_body = stored.body
771
+ ex.saved_usd = float(stored.cost_usd or 0.0)
772
+ ex.cost_usd = 0.0
773
+ h: dict[str, str] = {}
774
+ for k, v in (stored.headers or {}).items():
775
+ lk = str(k).lower()
776
+ if lk in _STORE_DROP or lk.startswith("x-llmtap-") or lk == "host":
777
+ continue
778
+ h[lk] = str(v)
779
+ h["x-llmtap-id"] = ex.id
780
+ h["x-llmtap-source"] = ex.source
781
+ data = stored.body.encode("utf-8")
782
+ try:
783
+ if ex.streamed:
784
+ h["content-type"] = h.get("content-type") or "text/event-stream"
785
+ if "event-stream" not in h["content-type"] and "ndjson" not in h["content-type"]:
786
+ h["content-type"] = "text/event-stream"
787
+ h["cache-control"] = "no-cache"
788
+ h["Transfer-Encoding"] = "chunked"
789
+ out.start(ex.status, h)
790
+ events = _split_sse(stored.body) if "event-stream" in h["content-type"] else (
791
+ [ln + "\n" for ln in stored.body.splitlines() if ln.strip()])
792
+ for i, evt in enumerate(events):
793
+ out.write(evt.encode("utf-8"))
794
+ if i == 0:
795
+ ex.ttft_ms = round((time.perf_counter() - t0) * 1000.0, 2)
796
+ out.end()
797
+ else:
798
+ h.setdefault("content-type", "application/json")
799
+ h["Content-Length"] = str(len(data))
800
+ out.start(ex.status, h)
801
+ if data:
802
+ out.write(data)
803
+ out.end()
804
+ ex.ttft_ms = round((time.perf_counter() - t0) * 1000.0, 2)
805
+ except ClientGone:
806
+ ex.error = "client disconnected"
807
+ out.abort()
808
+ self._account(ex, route, req, stored.body, ex.streamed)
809
+
810
+ # ---------------------------------------------------------------- accounting
811
+ def _account(self, ex: Exchange, route: Route, req: dict | None, text: str, streamed: bool,
812
+ llm: bool = True, parseable: bool = True) -> str | None:
813
+ """Fill usage / output_text / model from a response body (never raises).
814
+
815
+ Returns an in-band error message (stream ``error`` event, or HTTP 200 with an ``error`` body) if any.
816
+ Usage is only *estimated* for successful generation calls whose body carried no usage and no error.
817
+ """
818
+ usage, out_text, model, err = Usage(), "", None, None
819
+ if parseable:
820
+ try:
821
+ if streamed:
822
+ usage, out_text, model = parse_sse(route.style, text)
823
+ err = sse_error(text)
824
+ else:
825
+ j = parse_json(text.encode("utf-8"))
826
+ if j is not None:
827
+ usage = extract_usage(route.style, j)
828
+ out_text = extract_text(route.style, j)
829
+ model = response_model(route.style, j)
830
+ err = body_error(j)
831
+ except Exception:
832
+ pass
833
+ ok = 200 <= ex.status < 300
834
+ if err and ok:
835
+ ex.error = ex.error or err
836
+ if ok and llm and not err and usage.input_tokens == 0 and usage.output_tokens == 0:
837
+ try:
838
+ usage = estimate_usage(req, out_text)
839
+ except Exception:
840
+ pass
841
+ ex.usage = usage
842
+ ex.output_text = out_text or ""
843
+ if model:
844
+ ex.model = model
845
+ return err if ok else None
846
+
847
+ def _price(self, ex: Exchange) -> float:
848
+ try:
849
+ return float(cost_usd(ex.model, ex.usage, self.cfg.pricing_overrides) or 0.0)
850
+ except Exception:
851
+ return 0.0
852
+
853
+ # ---------------------------------------------------------------- responses
854
+ def _simple(self, respond: Responder, status: int, data: bytes, headers: dict[str, str]) -> None:
855
+ h = dict(headers)
856
+ h["Content-Length"] = str(len(data))
857
+ try:
858
+ respond.start(status, h)
859
+ if data:
860
+ respond.write(data)
861
+ respond.end()
862
+ except _CLIENT_GONE:
863
+ pass
864
+
865
+ def _error(self, out: _Tracked, ex: Exchange, style: str, status: int, message: str, err_type: str) -> None:
866
+ """Send a provider-shaped JSON error and fill the exchange accordingly."""
867
+ data = error_body(style, status, message, err_type)
868
+ ex.status = status
869
+ ex.error = ex.error or message
870
+ ex.response_body = data.decode("utf-8", "replace")
871
+ if out.started:
872
+ return
873
+ try:
874
+ hdrs = {"Content-Type": "application/json", "Content-Length": str(len(data)),
875
+ "x-llmtap-id": ex.id, "x-llmtap-source": ex.source}
876
+ if status in (400, 409, 429):
877
+ # deliberate llmtap blocks (guard/cassette miss/budget/loop): tell official SDKs not to retry
878
+ hdrs["x-should-retry"] = "false"
879
+ out.start(status, hdrs)
880
+ out.write(data)
881
+ out.end()
882
+ except ClientGone:
883
+ pass
884
+
885
+ def _internal(self, method: str, path: str, headers: Mapping[str, str], body: bytes,
886
+ respond: Responder, ex: Exchange) -> None:
887
+ out = _Tracked(respond)
888
+ try:
889
+ from .dashboard import handle_internal
890
+ except Exception:
891
+ handle_internal = None # type: ignore[assignment]
892
+ try:
893
+ handled = bool(handle_internal(self, method, path, headers, body, out)) if handle_internal else False
894
+ except ClientGone:
895
+ return
896
+ except Exception as e:
897
+ ex.error = f"dashboard error: {type(e).__name__}: {e}"
898
+ if not out.started:
899
+ data = json.dumps({"error": ex.error}).encode()
900
+ ex.status = 500
901
+ self._simple(respond, 500, data, {"Content-Type": "application/json"})
902
+ return
903
+ if not handled and not out.started:
904
+ ex.status = 404
905
+ data = json.dumps({"error": f"not found: {path}", "version": __version__}).encode()
906
+ self._simple(respond, 404, data, {"Content-Type": "application/json"})
907
+
908
+ # ---------------------------------------------------------------- finish
909
+ def _finish(self, ex: Exchange) -> None:
910
+ """Persist, publish and log; each step isolated so none can break the request."""
911
+ try:
912
+ _seal(ex)
913
+ except Exception:
914
+ ex.__dict__.pop("_secrets", None)
915
+ try:
916
+ self.store.record(ex)
917
+ except Exception as e:
918
+ ex.error = ex.error or f"store write failed: {e}"
919
+ try:
920
+ self.bus.publish({"type": "exchange", "exchange": summarize(ex)})
921
+ except Exception:
922
+ pass
923
+ if not self.cfg.quiet:
924
+ try:
925
+ from .term import log_exchange
926
+ log_exchange(ex)
927
+ except Exception:
928
+ pass
929
+
930
+
931
+ def _host_is_exposed(host: str) -> bool:
932
+ h = (host or "").strip().strip("[]").lower()
933
+ if h == "localhost":
934
+ return False
935
+ try:
936
+ return not ipaddress.ip_address(h).is_loopback
937
+ except ValueError:
938
+ return True # "" (all interfaces) or a hostname we can't prove is loopback
939
+
940
+
941
+ def _merge_headers(relay: list[tuple[str, str]], extra: dict[str, str]) -> dict[str, str]:
942
+ """Relay headers as a dict (duplicate names joined with ', ', except set-cookie last-wins)."""
943
+ out: dict[str, str] = {}
944
+ lower_map: dict[str, str] = {}
945
+ for k, v in relay:
946
+ lk = k.lower()
947
+ if lk in lower_map and lk != "set-cookie":
948
+ out[lower_map[lk]] = out[lower_map[lk]] + ", " + v
949
+ else:
950
+ lower_map[lk] = k
951
+ out[k] = v
952
+ for k, v in extra.items():
953
+ out[k] = v
954
+ return out
955
+
956
+
957
+ def _upstream_error_message(text: str) -> str | None:
958
+ j = parse_json(text.encode("utf-8")) if text else None
959
+ if not j:
960
+ return None
961
+ err = j.get("error")
962
+ if isinstance(err, dict) and isinstance(err.get("message"), str):
963
+ return err["message"][:500]
964
+ if isinstance(err, str):
965
+ return err[:500]
966
+ if isinstance(j.get("message"), str):
967
+ return j["message"][:500]
968
+ return None
969
+
970
+
971
+ # ------------------------------------------------------------------------------------------------
972
+ # http.server adapter
973
+ # ------------------------------------------------------------------------------------------------
974
+ class _Server(ThreadingHTTPServer):
975
+ daemon_threads = True
976
+ allow_reuse_address = True
977
+ block_on_close = False
978
+ request_queue_size = 128
979
+
980
+ def __init__(self, addr: tuple[str, int], handler: type, tap: LLMTap) -> None:
981
+ self.tap = tap
982
+ if ":" in (addr[0] or ""):
983
+ self.address_family = socket.AF_INET6
984
+ super().__init__(addr, handler)
985
+
986
+ def server_bind(self) -> None:
987
+ # Skip HTTPServer.server_bind's socket.getfqdn(): it can stall for seconds on some hosts.
988
+ socketserver.TCPServer.server_bind(self)
989
+ host, port = self.server_address[:2]
990
+ self.server_name = str(host)
991
+ self.server_port = int(port)
992
+
993
+ def handle_error(self, request: Any, client_address: Any) -> None: # quiet on client resets
994
+ pass
995
+
996
+
997
+ class _HandlerResponder:
998
+ """Responder over a BaseHTTPRequestHandler with Content-Length or chunked framing."""
999
+
1000
+ def __init__(self, handler: "_Handler") -> None:
1001
+ self.h = handler
1002
+ self.started = False
1003
+ self.ended = False
1004
+ self.chunked = False
1005
+ self.no_body = False
1006
+
1007
+ def start(self, status: int, headers: dict[str, str]) -> None:
1008
+ if self.started:
1009
+ raise RuntimeError("response already started")
1010
+ h = self.h
1011
+ self.started = True
1012
+ self.no_body = h.command == "HEAD" or status in (204, 304) or 100 <= status < 200
1013
+ want_chunked = False
1014
+ has_length = False
1015
+ h.send_response(status)
1016
+ for k, v in headers.items():
1017
+ lk = k.lower()
1018
+ if lk == "transfer-encoding":
1019
+ want_chunked = "chunked" in str(v).lower()
1020
+ continue
1021
+ if lk == "content-length":
1022
+ has_length = True
1023
+ if lk == "connection":
1024
+ continue
1025
+ h.send_header(k, str(v))
1026
+ if self.no_body:
1027
+ if not has_length and status not in (204, 304) and not (100 <= status < 200):
1028
+ h.send_header("Content-Length", "0")
1029
+ elif want_chunked or not has_length:
1030
+ if h.request_version == "HTTP/1.0":
1031
+ h.close_connection = True
1032
+ else:
1033
+ self.chunked = True
1034
+ h.send_header("Transfer-Encoding", "chunked")
1035
+ if h.close_connection:
1036
+ h.send_header("Connection", "close")
1037
+ h.end_headers()
1038
+ h.wfile.flush()
1039
+
1040
+ def write(self, chunk: bytes) -> None:
1041
+ if not self.started:
1042
+ raise RuntimeError("write() before start()")
1043
+ if not chunk or self.no_body:
1044
+ return
1045
+ w = self.h.wfile
1046
+ if self.chunked:
1047
+ w.write(b"%x\r\n" % len(chunk) + bytes(chunk) + b"\r\n")
1048
+ else:
1049
+ w.write(chunk)
1050
+ w.flush()
1051
+
1052
+ def end(self) -> None:
1053
+ if self.ended:
1054
+ return
1055
+ self.ended = True
1056
+ if self.chunked and not self.no_body:
1057
+ self.h.wfile.write(b"0\r\n\r\n")
1058
+ self.h.wfile.flush()
1059
+
1060
+ def abort(self) -> None:
1061
+ """Mark done without the terminating frame and close the connection afterwards."""
1062
+ self.ended = True
1063
+ self.h.close_connection = True
1064
+
1065
+
1066
+ class _BadRequest(Exception):
1067
+ pass
1068
+
1069
+
1070
+ class _Handler(BaseHTTPRequestHandler):
1071
+ protocol_version = "HTTP/1.1"
1072
+ server_version = "llmtap/" + __version__
1073
+ sys_version = ""
1074
+ timeout = 900 # reap idle keep-alive connections / stalled clients instead of leaking threads
1075
+ server: _Server
1076
+
1077
+ def log_message(self, format: str, *args: Any) -> None: # noqa: A002 - silence default logging
1078
+ pass
1079
+
1080
+ def _read_body(self) -> bytes:
1081
+ te = (self.headers.get("Transfer-Encoding") or "").lower()
1082
+ if "chunked" in te:
1083
+ buf = bytearray()
1084
+ while True:
1085
+ line = self.rfile.readline(65537)
1086
+ if not line:
1087
+ raise _BadRequest("truncated chunked body")
1088
+ size_s = line.split(b";", 1)[0].strip()
1089
+ try:
1090
+ size = int(size_s, 16)
1091
+ except ValueError:
1092
+ raise _BadRequest("bad chunk size") from None
1093
+ if size == 0:
1094
+ while True: # trailers
1095
+ t = self.rfile.readline(65537)
1096
+ if t in (b"\r\n", b"\n", b""):
1097
+ break
1098
+ return bytes(buf)
1099
+ data = self.rfile.read(size)
1100
+ if len(data) < size:
1101
+ raise _BadRequest("truncated chunk")
1102
+ buf += data
1103
+ self.rfile.readline(3) # CRLF after chunk
1104
+ cl = self.headers.get("Content-Length")
1105
+ if cl:
1106
+ try:
1107
+ n = int(cl)
1108
+ except ValueError:
1109
+ raise _BadRequest("bad Content-Length") from None
1110
+ if n < 0:
1111
+ raise _BadRequest("bad Content-Length")
1112
+ data = self.rfile.read(n)
1113
+ if len(data) < n:
1114
+ raise _BadRequest("truncated body")
1115
+ return data
1116
+ return b""
1117
+
1118
+ def _dispatch(self) -> None:
1119
+ r = _HandlerResponder(self)
1120
+ try:
1121
+ try:
1122
+ body = self._read_body()
1123
+ except _BadRequest as e:
1124
+ self.close_connection = True
1125
+ msg = json.dumps({"error": {"message": f"llmtap: bad request: {e}", "type": "llmtap_bad_request"}})
1126
+ data = msg.encode()
1127
+ r.start(400, {"Content-Type": "application/json", "Content-Length": str(len(data))})
1128
+ r.write(data)
1129
+ r.end()
1130
+ return
1131
+ headers = {k: v for k, v in self.headers.items()}
1132
+ self.server.tap.handle(self.command, self.path, headers, body, r)
1133
+ if not r.started:
1134
+ data = b'{"error":{"message":"llmtap: no response produced","type":"llmtap_error"}}'
1135
+ r.start(500, {"Content-Type": "application/json", "Content-Length": str(len(data))})
1136
+ r.write(data)
1137
+ r.end()
1138
+ elif not r.ended:
1139
+ r.end()
1140
+ except _CLIENT_GONE:
1141
+ self.close_connection = True
1142
+ except Exception:
1143
+ self.close_connection = True
1144
+ if not r.started:
1145
+ try:
1146
+ data = b'{"error":{"message":"llmtap internal error","type":"llmtap_error"}}'
1147
+ r.start(500, {"Content-Type": "application/json", "Content-Length": str(len(data))})
1148
+ r.write(data)
1149
+ r.end()
1150
+ except Exception:
1151
+ pass
1152
+
1153
+ do_GET = do_POST = do_PUT = do_PATCH = do_DELETE = do_OPTIONS = do_HEAD = _dispatch