llmtap-proxy 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llmtap/__init__.py +2 -0
- llmtap/__main__.py +3 -0
- llmtap/budget.py +166 -0
- llmtap/cache.py +129 -0
- llmtap/cassette.py +172 -0
- llmtap/cli.py +1066 -0
- llmtap/config.py +112 -0
- llmtap/dashboard.py +445 -0
- llmtap/demo.py +244 -0
- llmtap/doctor.py +275 -0
- llmtap/guard.py +391 -0
- llmtap/mock.py +478 -0
- llmtap/models.py +75 -0
- llmtap/pricing.py +379 -0
- llmtap/providers.py +635 -0
- llmtap/proxy.py +1153 -0
- llmtap/pytest_plugin.py +444 -0
- llmtap/report.py +893 -0
- llmtap/store.py +319 -0
- llmtap/term.py +477 -0
- llmtap/ui/index.html +893 -0
- llmtap_proxy-0.2.0.dist-info/METADATA +609 -0
- llmtap_proxy-0.2.0.dist-info/RECORD +27 -0
- llmtap_proxy-0.2.0.dist-info/WHEEL +5 -0
- llmtap_proxy-0.2.0.dist-info/entry_points.txt +5 -0
- llmtap_proxy-0.2.0.dist-info/licenses/LICENSE +21 -0
- llmtap_proxy-0.2.0.dist-info/top_level.txt +1 -0
llmtap/mock.py
ADDED
|
@@ -0,0 +1,478 @@
|
|
|
1
|
+
"""Realistic fake LLM upstream (Anthropic + OpenAI wire formats) for tests and `llmtap demo`.
|
|
2
|
+
|
|
3
|
+
Speaks HTTP/1.1 on 127.0.0.1 with proper chunked transfer-encoding for SSE streams. Replies are
|
|
4
|
+
deterministic (``"Echo: <last user text>"``) unless the prompt contains a magic word:
|
|
5
|
+
``MOCK_ERROR_500`` (500 provider error), ``MOCK_ERROR_429`` (429 rate limit), ``MOCK_SLOW`` (sleep 1s).
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import re
|
|
12
|
+
import struct
|
|
13
|
+
import threading
|
|
14
|
+
import time
|
|
15
|
+
import uuid
|
|
16
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
17
|
+
from typing import Any, Iterator
|
|
18
|
+
|
|
19
|
+
MOCK_MODELS: tuple[str, ...] = (
|
|
20
|
+
"claude-opus-4-1", "claude-sonnet-4-5", "claude-haiku-4-5",
|
|
21
|
+
"gpt-4o", "gpt-4o-mini", "gpt-5", "text-embedding-3-small",
|
|
22
|
+
)
|
|
23
|
+
EMBED_DIM = 8
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# --------------------------------------------------------------------------------------- helpers
|
|
27
|
+
def _collect_strings(obj: Any, out: list[str]) -> None:
|
|
28
|
+
"""Append every string found in a JSON-ish object (depth first)."""
|
|
29
|
+
if isinstance(obj, str):
|
|
30
|
+
out.append(obj)
|
|
31
|
+
elif isinstance(obj, dict):
|
|
32
|
+
for v in obj.values():
|
|
33
|
+
_collect_strings(v, out)
|
|
34
|
+
elif isinstance(obj, list):
|
|
35
|
+
for v in obj:
|
|
36
|
+
_collect_strings(v, out)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _content_text(content: Any) -> str:
|
|
40
|
+
"""Text of a message `content` (string, or list of blocks with text / input_text)."""
|
|
41
|
+
if isinstance(content, str):
|
|
42
|
+
return content
|
|
43
|
+
if isinstance(content, list):
|
|
44
|
+
parts: list[str] = []
|
|
45
|
+
for block in content:
|
|
46
|
+
if isinstance(block, str):
|
|
47
|
+
parts.append(block)
|
|
48
|
+
elif isinstance(block, dict):
|
|
49
|
+
t = block.get("text")
|
|
50
|
+
if isinstance(t, str):
|
|
51
|
+
parts.append(t)
|
|
52
|
+
elif isinstance(block.get("content"), (str, list)): # tool_result blocks
|
|
53
|
+
parts.append(_content_text(block["content"]))
|
|
54
|
+
return "\n".join(p for p in parts if p)
|
|
55
|
+
return ""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def prompt_text(req: dict | None) -> str:
|
|
59
|
+
"""All prompt text of a request (system + messages / input / prompt), joined by newlines."""
|
|
60
|
+
if not isinstance(req, dict):
|
|
61
|
+
return ""
|
|
62
|
+
out: list[str] = []
|
|
63
|
+
for key in ("system", "instructions", "prompt"):
|
|
64
|
+
val = req.get(key)
|
|
65
|
+
if isinstance(val, (str, list)):
|
|
66
|
+
out.append(_content_text(val))
|
|
67
|
+
for key in ("messages", "input"):
|
|
68
|
+
val = req.get(key)
|
|
69
|
+
if isinstance(val, str):
|
|
70
|
+
out.append(val)
|
|
71
|
+
elif isinstance(val, list):
|
|
72
|
+
for m in val:
|
|
73
|
+
if isinstance(m, dict) and "content" in m:
|
|
74
|
+
c = m["content"]
|
|
75
|
+
out.append(_content_text(c) if isinstance(c, (str, list)) else json.dumps(c))
|
|
76
|
+
elif isinstance(m, dict):
|
|
77
|
+
strs: list[str] = []
|
|
78
|
+
_collect_strings(m, strs) # e.g. function_call_output items
|
|
79
|
+
out.append(" ".join(strs))
|
|
80
|
+
elif isinstance(m, str):
|
|
81
|
+
out.append(m)
|
|
82
|
+
return "\n".join(p for p in out if p)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def last_user_text(req: dict | None) -> str:
|
|
86
|
+
"""Text of the last user turn (messages API, chat completions, or responses API `input`)."""
|
|
87
|
+
if not isinstance(req, dict):
|
|
88
|
+
return ""
|
|
89
|
+
msgs = req.get("messages")
|
|
90
|
+
if msgs is None:
|
|
91
|
+
msgs = req.get("input")
|
|
92
|
+
if isinstance(msgs, str):
|
|
93
|
+
return msgs
|
|
94
|
+
if isinstance(msgs, list):
|
|
95
|
+
for m in reversed(msgs):
|
|
96
|
+
if isinstance(m, dict) and m.get("role", "user") == "user":
|
|
97
|
+
txt = _content_text(m.get("content"))
|
|
98
|
+
if txt:
|
|
99
|
+
return txt
|
|
100
|
+
elif isinstance(m, str):
|
|
101
|
+
return m
|
|
102
|
+
p = req.get("prompt")
|
|
103
|
+
return p if isinstance(p, str) else ""
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def reply_for(req: dict | None) -> str:
|
|
107
|
+
"""Deterministic assistant reply for a request."""
|
|
108
|
+
return ("Echo: " + last_user_text(req))[:200]
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def count_tokens(text: str) -> int:
|
|
112
|
+
"""Mock tokenizer: len(text)//4 + 1."""
|
|
113
|
+
return len(text) // 4 + 1
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def split_chunks(text: str) -> list[str]:
|
|
117
|
+
"""Split text into word-ish streaming chunks that concatenate back to `text`."""
|
|
118
|
+
parts = re.findall(r"\s*\S+", text)
|
|
119
|
+
if not parts:
|
|
120
|
+
return [text] if text else []
|
|
121
|
+
tail = text[len("".join(parts)):]
|
|
122
|
+
if tail:
|
|
123
|
+
parts[-1] += tail
|
|
124
|
+
return parts
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def embedding_for(text: str, dim: int = EMBED_DIM) -> list[float]:
|
|
128
|
+
"""Deterministic unit-ish pseudo-vector derived from sha256(text)."""
|
|
129
|
+
digest = hashlib.sha256(text.encode("utf-8")).digest()
|
|
130
|
+
raw = [struct.unpack(">H", digest[2 * i:2 * i + 2])[0] / 65535.0 * 2 - 1 for i in range(dim)]
|
|
131
|
+
norm = sum(x * x for x in raw) ** 0.5 or 1.0
|
|
132
|
+
return [round(x / norm, 6) for x in raw]
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _anthropic_error(status: int, message: str) -> dict:
|
|
136
|
+
etype = {429: "rate_limit_error", 500: "api_error", 400: "invalid_request_error",
|
|
137
|
+
404: "not_found_error"}.get(status, "api_error")
|
|
138
|
+
return {"type": "error", "error": {"type": etype, "message": message}}
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _openai_error(status: int, message: str) -> dict:
|
|
142
|
+
etype = {429: "rate_limit_exceeded", 500: "server_error", 400: "invalid_request_error",
|
|
143
|
+
404: "not_found"}.get(status, "server_error")
|
|
144
|
+
return {"error": {"message": message, "type": etype, "param": None, "code": etype}}
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
# --------------------------------------------------------------------------------------- server
|
|
148
|
+
class _Handler(BaseHTTPRequestHandler):
|
|
149
|
+
protocol_version = "HTTP/1.1"
|
|
150
|
+
server_version = "llmtap-mock/0.1"
|
|
151
|
+
mock: "MockUpstream" # set on the subclass created per MockUpstream
|
|
152
|
+
|
|
153
|
+
def log_message(self, format: str, *args: Any) -> None: # noqa: A002 - silence stderr
|
|
154
|
+
return
|
|
155
|
+
|
|
156
|
+
# ---- low level writers
|
|
157
|
+
def _send_json(self, status: int, obj: Any, extra: dict[str, str] | None = None) -> None:
|
|
158
|
+
data = json.dumps(obj).encode("utf-8")
|
|
159
|
+
self.send_response(status)
|
|
160
|
+
self.send_header("Content-Type", "application/json")
|
|
161
|
+
self.send_header("Content-Length", str(len(data)))
|
|
162
|
+
self.send_header("request-id", "req_mock_" + uuid.uuid4().hex[:16])
|
|
163
|
+
for k, v in (extra or {}).items():
|
|
164
|
+
self.send_header(k, v)
|
|
165
|
+
self.end_headers()
|
|
166
|
+
self.wfile.write(data)
|
|
167
|
+
self.wfile.flush()
|
|
168
|
+
|
|
169
|
+
def _start_stream(self) -> None:
|
|
170
|
+
self.send_response(200)
|
|
171
|
+
self.send_header("Content-Type", "text/event-stream; charset=utf-8")
|
|
172
|
+
self.send_header("Cache-Control", "no-cache")
|
|
173
|
+
self.send_header("Transfer-Encoding", "chunked")
|
|
174
|
+
self.send_header("request-id", "req_mock_" + uuid.uuid4().hex[:16])
|
|
175
|
+
self.end_headers()
|
|
176
|
+
self.wfile.flush()
|
|
177
|
+
|
|
178
|
+
def _chunk(self, data: bytes) -> None:
|
|
179
|
+
if not data:
|
|
180
|
+
return
|
|
181
|
+
self.wfile.write(b"%x\r\n%s\r\n" % (len(data), data))
|
|
182
|
+
self.wfile.flush()
|
|
183
|
+
|
|
184
|
+
def _end_stream(self) -> None:
|
|
185
|
+
self.wfile.write(b"0\r\n\r\n")
|
|
186
|
+
self.wfile.flush()
|
|
187
|
+
|
|
188
|
+
def _stream(self, frames: Iterator[bytes]) -> None:
|
|
189
|
+
self._start_stream()
|
|
190
|
+
try:
|
|
191
|
+
for frame in frames:
|
|
192
|
+
self._chunk(frame)
|
|
193
|
+
self._end_stream()
|
|
194
|
+
except (BrokenPipeError, ConnectionResetError):
|
|
195
|
+
self.close_connection = True
|
|
196
|
+
|
|
197
|
+
def _sleep_token(self) -> None:
|
|
198
|
+
if self.mock.token_delay_s > 0:
|
|
199
|
+
time.sleep(self.mock.token_delay_s)
|
|
200
|
+
|
|
201
|
+
# ---- request plumbing
|
|
202
|
+
def _read_body(self) -> bytes:
|
|
203
|
+
length = int(self.headers.get("Content-Length") or 0)
|
|
204
|
+
if length:
|
|
205
|
+
return self.rfile.read(length)
|
|
206
|
+
if "chunked" in (self.headers.get("Transfer-Encoding") or "").lower():
|
|
207
|
+
buf = b""
|
|
208
|
+
while True:
|
|
209
|
+
line = self.rfile.readline().strip()
|
|
210
|
+
size = int(line.split(b";")[0] or b"0", 16)
|
|
211
|
+
if size == 0:
|
|
212
|
+
self.rfile.readline()
|
|
213
|
+
break
|
|
214
|
+
buf += self.rfile.read(size)
|
|
215
|
+
self.rfile.readline()
|
|
216
|
+
return buf
|
|
217
|
+
return b""
|
|
218
|
+
|
|
219
|
+
def _record(self, method: str, raw: bytes) -> dict | None:
|
|
220
|
+
try:
|
|
221
|
+
parsed = json.loads(raw) if raw else None
|
|
222
|
+
except (ValueError, UnicodeDecodeError):
|
|
223
|
+
parsed = None
|
|
224
|
+
entry = {
|
|
225
|
+
"method": method,
|
|
226
|
+
"path": self.path,
|
|
227
|
+
"headers": {k.lower(): v for k, v in self.headers.items()},
|
|
228
|
+
"body": parsed,
|
|
229
|
+
"raw": raw,
|
|
230
|
+
}
|
|
231
|
+
with self.mock._lock:
|
|
232
|
+
self.mock.requests.append(entry)
|
|
233
|
+
return parsed if isinstance(parsed, dict) else None
|
|
234
|
+
|
|
235
|
+
def do_GET(self) -> None: # noqa: N802
|
|
236
|
+
self._record("GET", b"")
|
|
237
|
+
path = self.path.split("?", 1)[0]
|
|
238
|
+
if path in ("/v1/models", "/models"):
|
|
239
|
+
now = int(time.time())
|
|
240
|
+
self._send_json(200, {"object": "list", "data": [
|
|
241
|
+
{"id": m, "object": "model", "type": "model", "created": now, "owned_by": "llmtap-mock",
|
|
242
|
+
"display_name": m} for m in MOCK_MODELS]})
|
|
243
|
+
else:
|
|
244
|
+
self._send_json(404, _openai_error(404, f"mock: no route for GET {path}"))
|
|
245
|
+
|
|
246
|
+
def do_POST(self) -> None: # noqa: N802
|
|
247
|
+
raw = self._read_body()
|
|
248
|
+
req = self._record("POST", raw)
|
|
249
|
+
path = self.path.split("?", 1)[0]
|
|
250
|
+
if self.mock.latency_s > 0:
|
|
251
|
+
time.sleep(self.mock.latency_s)
|
|
252
|
+
style = "anthropic" if path.startswith("/v1/messages") or path.startswith("/v1/complete") else "openai"
|
|
253
|
+
if req is None:
|
|
254
|
+
err = _anthropic_error if style == "anthropic" else _openai_error
|
|
255
|
+
self._send_json(400, err(400, "mock: request body must be a JSON object"))
|
|
256
|
+
return
|
|
257
|
+
text = prompt_text(req)
|
|
258
|
+
if "MOCK_SLOW" in text:
|
|
259
|
+
time.sleep(1.0)
|
|
260
|
+
if "MOCK_ERROR_500" in text:
|
|
261
|
+
err = _anthropic_error if style == "anthropic" else _openai_error
|
|
262
|
+
self._send_json(500, err(500, "mock: internal server error (MOCK_ERROR_500)"))
|
|
263
|
+
return
|
|
264
|
+
if "MOCK_ERROR_429" in text:
|
|
265
|
+
err = _anthropic_error if style == "anthropic" else _openai_error
|
|
266
|
+
self._send_json(429, err(429, "mock: rate limited (MOCK_ERROR_429)"), {"retry-after": "1"})
|
|
267
|
+
return
|
|
268
|
+
if path == "/v1/messages/count_tokens":
|
|
269
|
+
self._send_json(200, {"input_tokens": count_tokens(text)})
|
|
270
|
+
elif path == "/v1/messages":
|
|
271
|
+
self._anthropic_messages(req, text)
|
|
272
|
+
elif path in ("/v1/chat/completions", "/chat/completions"):
|
|
273
|
+
self._openai_chat(req, text)
|
|
274
|
+
elif path in ("/v1/responses", "/responses"):
|
|
275
|
+
self._openai_responses(req, text)
|
|
276
|
+
elif path in ("/v1/embeddings", "/embeddings"):
|
|
277
|
+
self._openai_embeddings(req)
|
|
278
|
+
else:
|
|
279
|
+
err = _anthropic_error if style == "anthropic" else _openai_error
|
|
280
|
+
self._send_json(404, err(404, f"mock: no route for POST {path}"))
|
|
281
|
+
|
|
282
|
+
# ---- anthropic messages
|
|
283
|
+
def _anthropic_messages(self, req: dict, text: str) -> None:
|
|
284
|
+
model = req.get("model") or "claude-sonnet-4-5"
|
|
285
|
+
reply = reply_for(req)
|
|
286
|
+
in_tok, out_tok = count_tokens(text), count_tokens(reply)
|
|
287
|
+
msg_id = "msg_mock_" + uuid.uuid4().hex[:24]
|
|
288
|
+
if req.get("stream") is not True:
|
|
289
|
+
self._send_json(200, {
|
|
290
|
+
"id": msg_id, "type": "message", "role": "assistant",
|
|
291
|
+
"content": [{"type": "text", "text": reply}],
|
|
292
|
+
"model": model, "stop_reason": "end_turn", "stop_sequence": None,
|
|
293
|
+
"usage": {"input_tokens": in_tok, "output_tokens": out_tok,
|
|
294
|
+
"cache_read_input_tokens": 0, "cache_creation_input_tokens": 0},
|
|
295
|
+
})
|
|
296
|
+
return
|
|
297
|
+
|
|
298
|
+
def ev(name: str, data: dict) -> bytes:
|
|
299
|
+
return f"event: {name}\ndata: {json.dumps(data)}\n\n".encode("utf-8")
|
|
300
|
+
|
|
301
|
+
def frames() -> Iterator[bytes]:
|
|
302
|
+
yield ev("message_start", {"type": "message_start", "message": {
|
|
303
|
+
"id": msg_id, "type": "message", "role": "assistant", "content": [], "model": model,
|
|
304
|
+
"stop_reason": None, "stop_sequence": None,
|
|
305
|
+
"usage": {"input_tokens": in_tok, "output_tokens": 1,
|
|
306
|
+
"cache_read_input_tokens": 0, "cache_creation_input_tokens": 0}}})
|
|
307
|
+
yield ev("content_block_start", {"type": "content_block_start", "index": 0,
|
|
308
|
+
"content_block": {"type": "text", "text": ""}})
|
|
309
|
+
yield ev("ping", {"type": "ping"})
|
|
310
|
+
for piece in split_chunks(reply):
|
|
311
|
+
self._sleep_token()
|
|
312
|
+
yield ev("content_block_delta", {"type": "content_block_delta", "index": 0,
|
|
313
|
+
"delta": {"type": "text_delta", "text": piece}})
|
|
314
|
+
yield ev("content_block_stop", {"type": "content_block_stop", "index": 0})
|
|
315
|
+
yield ev("message_delta", {"type": "message_delta",
|
|
316
|
+
"delta": {"stop_reason": "end_turn", "stop_sequence": None},
|
|
317
|
+
"usage": {"output_tokens": out_tok}})
|
|
318
|
+
yield ev("message_stop", {"type": "message_stop"})
|
|
319
|
+
|
|
320
|
+
self._stream(frames())
|
|
321
|
+
|
|
322
|
+
# ---- openai chat completions
|
|
323
|
+
def _openai_chat(self, req: dict, text: str) -> None:
|
|
324
|
+
model = req.get("model") or "gpt-4o-mini"
|
|
325
|
+
reply = reply_for(req)
|
|
326
|
+
in_tok, out_tok = count_tokens(text), count_tokens(reply)
|
|
327
|
+
cid = "chatcmpl-mock" + uuid.uuid4().hex[:20]
|
|
328
|
+
created = int(time.time())
|
|
329
|
+
usage = {"prompt_tokens": in_tok, "completion_tokens": out_tok, "total_tokens": in_tok + out_tok,
|
|
330
|
+
"prompt_tokens_details": {"cached_tokens": 0, "audio_tokens": 0},
|
|
331
|
+
"completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0}}
|
|
332
|
+
if req.get("stream") is not True:
|
|
333
|
+
self._send_json(200, {
|
|
334
|
+
"id": cid, "object": "chat.completion", "created": created, "model": model,
|
|
335
|
+
"choices": [{"index": 0, "message": {"role": "assistant", "content": reply, "refusal": None},
|
|
336
|
+
"logprobs": None, "finish_reason": "stop"}],
|
|
337
|
+
"usage": usage, "system_fingerprint": "fp_mock",
|
|
338
|
+
})
|
|
339
|
+
return
|
|
340
|
+
include_usage = bool((req.get("stream_options") or {}).get("include_usage"))
|
|
341
|
+
|
|
342
|
+
def chunk(delta: dict, finish: str | None = None, with_usage: bool = False) -> bytes:
|
|
343
|
+
obj: dict[str, Any] = {"id": cid, "object": "chat.completion.chunk", "created": created,
|
|
344
|
+
"model": model, "system_fingerprint": "fp_mock",
|
|
345
|
+
"choices": [{"index": 0, "delta": delta, "logprobs": None,
|
|
346
|
+
"finish_reason": finish}]}
|
|
347
|
+
if include_usage:
|
|
348
|
+
obj["usage"] = None
|
|
349
|
+
return f"data: {json.dumps(obj)}\n\n".encode("utf-8")
|
|
350
|
+
|
|
351
|
+
def frames() -> Iterator[bytes]:
|
|
352
|
+
yield chunk({"role": "assistant", "content": "", "refusal": None})
|
|
353
|
+
for piece in split_chunks(reply):
|
|
354
|
+
self._sleep_token()
|
|
355
|
+
yield chunk({"content": piece})
|
|
356
|
+
yield chunk({}, "stop")
|
|
357
|
+
if include_usage:
|
|
358
|
+
final = {"id": cid, "object": "chat.completion.chunk", "created": created, "model": model,
|
|
359
|
+
"system_fingerprint": "fp_mock", "choices": [], "usage": usage}
|
|
360
|
+
yield f"data: {json.dumps(final)}\n\n".encode("utf-8")
|
|
361
|
+
yield b"data: [DONE]\n\n"
|
|
362
|
+
|
|
363
|
+
self._stream(frames())
|
|
364
|
+
|
|
365
|
+
# ---- openai responses API
|
|
366
|
+
def _openai_responses(self, req: dict, text: str) -> None:
|
|
367
|
+
model = req.get("model") or "gpt-4o-mini"
|
|
368
|
+
reply = reply_for(req)
|
|
369
|
+
in_tok, out_tok = count_tokens(text), count_tokens(reply)
|
|
370
|
+
rid = "resp_mock_" + uuid.uuid4().hex[:24]
|
|
371
|
+
mid = "msg_mock_" + uuid.uuid4().hex[:24]
|
|
372
|
+
created = int(time.time())
|
|
373
|
+
usage = {"input_tokens": in_tok, "input_tokens_details": {"cached_tokens": 0},
|
|
374
|
+
"output_tokens": out_tok, "output_tokens_details": {"reasoning_tokens": 0},
|
|
375
|
+
"total_tokens": in_tok + out_tok}
|
|
376
|
+
item = {"type": "message", "id": mid, "status": "completed", "role": "assistant",
|
|
377
|
+
"content": [{"type": "output_text", "text": reply, "annotations": []}]}
|
|
378
|
+
|
|
379
|
+
def response(status: str, output: list, use: dict | None) -> dict:
|
|
380
|
+
return {"id": rid, "object": "response", "created_at": created, "status": status, "model": model,
|
|
381
|
+
"output": output, "usage": use, "error": None, "incomplete_details": None}
|
|
382
|
+
|
|
383
|
+
if req.get("stream") is not True:
|
|
384
|
+
body = response("completed", [item], usage)
|
|
385
|
+
body["output_text"] = reply
|
|
386
|
+
self._send_json(200, body)
|
|
387
|
+
return
|
|
388
|
+
seq = iter(range(10_000))
|
|
389
|
+
|
|
390
|
+
def ev(name: str, data: dict) -> bytes:
|
|
391
|
+
data = {"type": name, "sequence_number": next(seq), **data}
|
|
392
|
+
return f"event: {name}\ndata: {json.dumps(data)}\n\n".encode("utf-8")
|
|
393
|
+
|
|
394
|
+
def frames() -> Iterator[bytes]:
|
|
395
|
+
yield ev("response.created", {"response": response("in_progress", [], None)})
|
|
396
|
+
yield ev("response.in_progress", {"response": response("in_progress", [], None)})
|
|
397
|
+
yield ev("response.output_item.added", {"output_index": 0, "item": {
|
|
398
|
+
"type": "message", "id": mid, "status": "in_progress", "role": "assistant", "content": []}})
|
|
399
|
+
yield ev("response.content_part.added", {"item_id": mid, "output_index": 0, "content_index": 0,
|
|
400
|
+
"part": {"type": "output_text", "text": "", "annotations": []}})
|
|
401
|
+
for piece in split_chunks(reply):
|
|
402
|
+
self._sleep_token()
|
|
403
|
+
yield ev("response.output_text.delta", {"item_id": mid, "output_index": 0,
|
|
404
|
+
"content_index": 0, "delta": piece})
|
|
405
|
+
yield ev("response.output_text.done", {"item_id": mid, "output_index": 0, "content_index": 0,
|
|
406
|
+
"text": reply})
|
|
407
|
+
yield ev("response.content_part.done", {"item_id": mid, "output_index": 0, "content_index": 0,
|
|
408
|
+
"part": item["content"][0]})
|
|
409
|
+
yield ev("response.output_item.done", {"output_index": 0, "item": item})
|
|
410
|
+
yield ev("response.completed", {"response": response("completed", [item], usage)})
|
|
411
|
+
|
|
412
|
+
self._stream(frames())
|
|
413
|
+
|
|
414
|
+
# ---- openai embeddings
|
|
415
|
+
def _openai_embeddings(self, req: dict) -> None:
|
|
416
|
+
model = req.get("model") or "text-embedding-3-small"
|
|
417
|
+
inp = req.get("input", "")
|
|
418
|
+
items = inp if isinstance(inp, list) else [inp]
|
|
419
|
+
texts = [x if isinstance(x, str) else json.dumps(x) for x in items]
|
|
420
|
+
tokens = sum(count_tokens(t) for t in texts)
|
|
421
|
+
self._send_json(200, {
|
|
422
|
+
"object": "list", "model": model,
|
|
423
|
+
"data": [{"object": "embedding", "index": i, "embedding": embedding_for(t)} for i, t in enumerate(texts)],
|
|
424
|
+
"usage": {"prompt_tokens": tokens, "total_tokens": tokens},
|
|
425
|
+
})
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
class _MockServer(ThreadingHTTPServer):
|
|
429
|
+
daemon_threads = True
|
|
430
|
+
request_queue_size = 128 # bursts of concurrent clients must not overflow the listen backlog
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
class MockUpstream:
|
|
434
|
+
"""A local fake Anthropic/OpenAI server. ``requests`` logs everything received (thread-safe append)."""
|
|
435
|
+
|
|
436
|
+
def __init__(self, latency_s: float = 0.0, token_delay_s: float = 0.0) -> None:
|
|
437
|
+
self.latency_s = latency_s
|
|
438
|
+
self.token_delay_s = token_delay_s
|
|
439
|
+
self.requests: list[dict] = []
|
|
440
|
+
self._lock = threading.Lock()
|
|
441
|
+
self._server: ThreadingHTTPServer | None = None
|
|
442
|
+
self._thread: threading.Thread | None = None
|
|
443
|
+
self.port: int = 0
|
|
444
|
+
|
|
445
|
+
@property
|
|
446
|
+
def base_url(self) -> str:
|
|
447
|
+
return f"http://127.0.0.1:{self.port}"
|
|
448
|
+
|
|
449
|
+
def start(self) -> "MockUpstream":
|
|
450
|
+
"""Bind 127.0.0.1:0 and serve in a daemon thread."""
|
|
451
|
+
handler = type("MockHandler", (_Handler,), {"mock": self})
|
|
452
|
+
self._server = _MockServer(("127.0.0.1", 0), handler)
|
|
453
|
+
self.port = self._server.server_address[1]
|
|
454
|
+
self._thread = threading.Thread(target=self._server.serve_forever, args=(0.05,),
|
|
455
|
+
name="llmtap-mock", daemon=True)
|
|
456
|
+
self._thread.start()
|
|
457
|
+
return self
|
|
458
|
+
|
|
459
|
+
def stop(self) -> None:
|
|
460
|
+
"""Stop serving and release the port (idempotent)."""
|
|
461
|
+
srv, self._server = self._server, None
|
|
462
|
+
if srv is not None:
|
|
463
|
+
srv.shutdown()
|
|
464
|
+
srv.server_close()
|
|
465
|
+
if self._thread is not None:
|
|
466
|
+
self._thread.join(timeout=5)
|
|
467
|
+
self._thread = None
|
|
468
|
+
|
|
469
|
+
def reset(self) -> None:
|
|
470
|
+
"""Forget logged requests."""
|
|
471
|
+
with self._lock:
|
|
472
|
+
self.requests.clear()
|
|
473
|
+
|
|
474
|
+
def __enter__(self) -> "MockUpstream":
|
|
475
|
+
return self.start()
|
|
476
|
+
|
|
477
|
+
def __exit__(self, *exc: Any) -> None:
|
|
478
|
+
self.stop()
|
llmtap/models.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""Shared data model. Every module speaks in these types."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import time
|
|
5
|
+
import uuid
|
|
6
|
+
from dataclasses import asdict, dataclass, field
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class Usage:
|
|
12
|
+
input_tokens: int = 0
|
|
13
|
+
output_tokens: int = 0
|
|
14
|
+
cache_read_tokens: int = 0 # anthropic cache_read_input_tokens / openai cached_tokens
|
|
15
|
+
cache_write_tokens: int = 0 # anthropic cache_creation_input_tokens
|
|
16
|
+
estimated: bool = False # True when tokens were guessed (chars/4), not reported
|
|
17
|
+
|
|
18
|
+
def total(self) -> int:
|
|
19
|
+
return self.input_tokens + self.output_tokens
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class Route:
|
|
24
|
+
provider: str # "anthropic" | "openai" | custom upstream name (e.g. "ollama")
|
|
25
|
+
upstream_base: str # e.g. "https://api.anthropic.com" (no trailing slash)
|
|
26
|
+
upstream_path: str # path + query to send upstream, e.g. "/v1/messages?beta=true"
|
|
27
|
+
style: str # wire format: "anthropic" | "openai"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class Exchange:
|
|
32
|
+
"""One request/response through the proxy."""
|
|
33
|
+
id: str = field(default_factory=lambda: uuid.uuid4().hex)
|
|
34
|
+
ts: float = field(default_factory=time.time) # request start, epoch seconds
|
|
35
|
+
provider: str = "unknown"
|
|
36
|
+
style: str = "openai"
|
|
37
|
+
method: str = "POST"
|
|
38
|
+
path: str = "" # path as received by the proxy
|
|
39
|
+
model: str | None = None
|
|
40
|
+
request_body: str = "" # as sent upstream (post-redaction), utf-8 text
|
|
41
|
+
status: int = 0
|
|
42
|
+
response_body: str = "" # raw JSON text, or raw SSE text when streamed
|
|
43
|
+
streamed: bool = False
|
|
44
|
+
latency_ms: float = 0.0
|
|
45
|
+
ttft_ms: float | None = None # time to first byte of body (streaming)
|
|
46
|
+
usage: Usage = field(default_factory=Usage)
|
|
47
|
+
cost_usd: float = 0.0
|
|
48
|
+
saved_usd: float = 0.0 # cost avoided when served from cache/cassette
|
|
49
|
+
source: str = "upstream" # "upstream" | "cache" | "cassette" | "blocked" | "budget" | "loop" | "error"
|
|
50
|
+
tags: dict[str, str] = field(default_factory=dict) # from X-LLMTap-* headers
|
|
51
|
+
guard_findings: list[dict[str, Any]] = field(default_factory=list)
|
|
52
|
+
error: str | None = None
|
|
53
|
+
output_text: str = "" # assistant text extracted from response (for UI/search)
|
|
54
|
+
|
|
55
|
+
def to_dict(self) -> dict[str, Any]:
|
|
56
|
+
return asdict(self)
|
|
57
|
+
|
|
58
|
+
@classmethod
|
|
59
|
+
def from_dict(cls, d: dict[str, Any]) -> "Exchange":
|
|
60
|
+
d = dict(d)
|
|
61
|
+
u = d.pop("usage", None) or {}
|
|
62
|
+
known = {f for f in cls.__dataclass_fields__}
|
|
63
|
+
ex = cls(**{k: v for k, v in d.items() if k in known})
|
|
64
|
+
ex.usage = Usage(**{k: v for k, v in u.items() if k in Usage.__dataclass_fields__})
|
|
65
|
+
return ex
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass
|
|
69
|
+
class CachedResponse:
|
|
70
|
+
"""A response that can be replayed verbatim (from cache or cassette)."""
|
|
71
|
+
status: int
|
|
72
|
+
headers: dict[str, str]
|
|
73
|
+
body: str # raw JSON text or raw SSE text
|
|
74
|
+
streamed: bool
|
|
75
|
+
cost_usd: float = 0.0 # original cost, used to compute saved_usd
|