llmtap-proxy 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llmtap/__init__.py +2 -0
- llmtap/__main__.py +3 -0
- llmtap/budget.py +166 -0
- llmtap/cache.py +129 -0
- llmtap/cassette.py +172 -0
- llmtap/cli.py +1066 -0
- llmtap/config.py +112 -0
- llmtap/dashboard.py +445 -0
- llmtap/demo.py +244 -0
- llmtap/doctor.py +275 -0
- llmtap/guard.py +391 -0
- llmtap/mock.py +478 -0
- llmtap/models.py +75 -0
- llmtap/pricing.py +379 -0
- llmtap/providers.py +635 -0
- llmtap/proxy.py +1153 -0
- llmtap/pytest_plugin.py +444 -0
- llmtap/report.py +893 -0
- llmtap/store.py +319 -0
- llmtap/term.py +477 -0
- llmtap/ui/index.html +893 -0
- llmtap_proxy-0.2.0.dist-info/METADATA +609 -0
- llmtap_proxy-0.2.0.dist-info/RECORD +27 -0
- llmtap_proxy-0.2.0.dist-info/WHEEL +5 -0
- llmtap_proxy-0.2.0.dist-info/entry_points.txt +5 -0
- llmtap_proxy-0.2.0.dist-info/licenses/LICENSE +21 -0
- llmtap_proxy-0.2.0.dist-info/top_level.txt +1 -0
llmtap/providers.py
ADDED
|
@@ -0,0 +1,635 @@
|
|
|
1
|
+
"""Routing + wire-format knowledge for Anthropic / OpenAI / OpenAI-compatible APIs.
|
|
2
|
+
|
|
3
|
+
Every function here is defensive: malformed or unexpected input never raises, it yields a
|
|
4
|
+
safe default (``None``, ``""``, ``Usage()``...).
|
|
5
|
+
|
|
6
|
+
Path normalization (openai style only): if the upstream path does not start with a version
|
|
7
|
+
segment (``/v1``, ``/v2``, ...), ``/v1`` is prepended -- e.g. ``/chat/completions`` becomes
|
|
8
|
+
``/v1/chat/completions`` (users often set ``OPENAI_BASE_URL`` without ``/v1``). The exception is an
|
|
9
|
+
explicit-prefix route whose upstream base already ends in a version segment (e.g. a custom upstream
|
|
10
|
+
``http://host/v1``); there the path is passed through unchanged. Anthropic-style paths are never
|
|
11
|
+
rewritten.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
from collections.abc import Mapping
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from .config import Config
|
|
21
|
+
from .models import Route, Usage
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"HOP_BY_HOP",
|
|
25
|
+
"body_error",
|
|
26
|
+
"error_body",
|
|
27
|
+
"estimate_tokens",
|
|
28
|
+
"estimate_usage",
|
|
29
|
+
"extract_text",
|
|
30
|
+
"extract_usage",
|
|
31
|
+
"forward_headers",
|
|
32
|
+
"is_stream_request",
|
|
33
|
+
"parse_json",
|
|
34
|
+
"parse_sse",
|
|
35
|
+
"request_model",
|
|
36
|
+
"resolve_route",
|
|
37
|
+
"response_model",
|
|
38
|
+
"sse_error",
|
|
39
|
+
"tags_from_headers",
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
_VERSION_SEG = re.compile(r"^/v\d+(?:[a-z0-9]*)?(?:[/?#]|$)")
|
|
43
|
+
_BASE_ENDS_WITH_VERSION = re.compile(r"/v\d+[a-z0-9]*$")
|
|
44
|
+
|
|
45
|
+
_DEFAULT_BASES = {
|
|
46
|
+
"anthropic": "https://api.anthropic.com",
|
|
47
|
+
"openai": "https://api.openai.com",
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# --------------------------------------------------------------------------------------------
|
|
52
|
+
# routing
|
|
53
|
+
# --------------------------------------------------------------------------------------------
|
|
54
|
+
def _header_names(headers: Mapping[str, str] | None) -> set[str]:
|
|
55
|
+
try:
|
|
56
|
+
return {str(k).lower() for k in (headers or {}).keys()}
|
|
57
|
+
except Exception:
|
|
58
|
+
return set()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _upstream(cfg: Config, name: str) -> tuple[str, str]:
|
|
62
|
+
"""(base, style) for a named upstream, with sensible fallbacks."""
|
|
63
|
+
spec = {}
|
|
64
|
+
try:
|
|
65
|
+
spec = (cfg.upstreams or {}).get(name) or {}
|
|
66
|
+
except Exception:
|
|
67
|
+
spec = {}
|
|
68
|
+
base = str(spec.get("base") or _DEFAULT_BASES.get(name, "")).rstrip("/")
|
|
69
|
+
style = str(spec.get("style") or ("anthropic" if name == "anthropic" else "openai"))
|
|
70
|
+
return base, style
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _normalize_openai_path(path: str, base: str) -> str:
|
|
74
|
+
if _VERSION_SEG.match(path) or _BASE_ENDS_WITH_VERSION.search(base):
|
|
75
|
+
return path
|
|
76
|
+
return "/v1" + path
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def resolve_route(path: str, headers: Mapping[str, str], cfg: Config) -> Route:
|
|
80
|
+
"""Decide which upstream a proxied request goes to (see module docstring for path rules)."""
|
|
81
|
+
path = path or "/"
|
|
82
|
+
if not path.startswith("/"):
|
|
83
|
+
path = "/" + path
|
|
84
|
+
pure, sep, query = path.partition("?")
|
|
85
|
+
|
|
86
|
+
# 1. explicit prefix: /<name>/rest
|
|
87
|
+
parts = pure.split("/", 2) # ["", name, rest]
|
|
88
|
+
name = parts[1] if len(parts) > 1 else ""
|
|
89
|
+
upstreams = getattr(cfg, "upstreams", None) or {}
|
|
90
|
+
if name and name in upstreams:
|
|
91
|
+
base, style = _upstream(cfg, name)
|
|
92
|
+
rest = "/" + (parts[2] if len(parts) > 2 else "")
|
|
93
|
+
up_path = rest + (sep + query if sep else "")
|
|
94
|
+
if style == "openai":
|
|
95
|
+
up_path = _normalize_openai_path(up_path, base)
|
|
96
|
+
return Route(provider=name, upstream_base=base, upstream_path=up_path, style=style)
|
|
97
|
+
|
|
98
|
+
# 2./3. anthropic by path or headers
|
|
99
|
+
hdrs = _header_names(headers)
|
|
100
|
+
if (
|
|
101
|
+
pure.startswith("/v1/messages")
|
|
102
|
+
or pure.startswith("/v1/complete")
|
|
103
|
+
or "x-api-key" in hdrs
|
|
104
|
+
# any anthropic-* header (anthropic-version, anthropic-beta, anthropic-dangerous-direct-browser-access):
|
|
105
|
+
# Claude Code with an OAuth bearer token sends no x-api-key / anthropic-version on e.g. /api/oauth/*
|
|
106
|
+
or any(h.startswith("anthropic-") for h in hdrs)
|
|
107
|
+
):
|
|
108
|
+
base, style = _upstream(cfg, "anthropic")
|
|
109
|
+
return Route(provider="anthropic", upstream_base=base, upstream_path=path, style=style)
|
|
110
|
+
|
|
111
|
+
#Check if path is a Gemini ednpoit or Google API key header is present
|
|
112
|
+
if(
|
|
113
|
+
":generateContent" in pure
|
|
114
|
+
or ":streamGenerateContent" in pure
|
|
115
|
+
or "x-goog-api-key" in hdrs
|
|
116
|
+
):
|
|
117
|
+
base, style = _upstream(cfg,"gemini")
|
|
118
|
+
return Route(provider="gemini", upstream_base=base, upstream_path=path,style=style)
|
|
119
|
+
|
|
120
|
+
# 4. openai
|
|
121
|
+
base, style = _upstream(cfg, "openai")
|
|
122
|
+
up_path = _normalize_openai_path(path, base) if style == "openai" else path
|
|
123
|
+
return Route(provider="openai", upstream_base=base, upstream_path=up_path, style=style)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# --------------------------------------------------------------------------------------------
|
|
127
|
+
# request helpers
|
|
128
|
+
# --------------------------------------------------------------------------------------------
|
|
129
|
+
def parse_json(body: bytes) -> dict | None:
|
|
130
|
+
"""Parse a JSON object body; None if empty, invalid, or not an object."""
|
|
131
|
+
if not body:
|
|
132
|
+
return None
|
|
133
|
+
try:
|
|
134
|
+
if isinstance(body, (bytes, bytearray, memoryview)):
|
|
135
|
+
text = bytes(body).decode("utf-8-sig")
|
|
136
|
+
else:
|
|
137
|
+
text = str(body)
|
|
138
|
+
obj = json.loads(text)
|
|
139
|
+
except Exception:
|
|
140
|
+
return None
|
|
141
|
+
return obj if isinstance(obj, dict) else None
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def request_model(req: dict | None) -> str | None:
|
|
145
|
+
"""The requested model name, if any."""
|
|
146
|
+
if not isinstance(req, dict):
|
|
147
|
+
return None
|
|
148
|
+
m = req.get("model")
|
|
149
|
+
return m if isinstance(m, str) and m else None
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
_PATH_MODEL_RE = re.compile(r"/models/([^/:?#]+)(?::[A-Za-z]+)?")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def model_from_path(path: str | None) -> str | None:
|
|
156
|
+
"""Model name embedded in a URL path, e.g. /v1beta/models/gemini-2.5-flash:generateContent -> gemini-2.5-flash.
|
|
157
|
+
Gemini carries the model in the URL rather than the request body."""
|
|
158
|
+
if not isinstance(path, str):
|
|
159
|
+
return None
|
|
160
|
+
m = _PATH_MODEL_RE.search(path.split("?", 1)[0])
|
|
161
|
+
return m.group(1) if m else None
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def is_stream_request(req: dict | None) -> bool:
|
|
166
|
+
"""True when the request asks for a streamed (SSE) response."""
|
|
167
|
+
return isinstance(req, dict) and req.get("stream") is True
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# --------------------------------------------------------------------------------------------
|
|
171
|
+
# response parsing
|
|
172
|
+
# --------------------------------------------------------------------------------------------
|
|
173
|
+
def _int(v: Any) -> int:
|
|
174
|
+
if isinstance(v, bool) or v is None:
|
|
175
|
+
return 0
|
|
176
|
+
if isinstance(v, int):
|
|
177
|
+
return max(0, v)
|
|
178
|
+
if isinstance(v, float):
|
|
179
|
+
return max(0, int(v))
|
|
180
|
+
if isinstance(v, str):
|
|
181
|
+
try:
|
|
182
|
+
return max(0, int(float(v)))
|
|
183
|
+
except ValueError:
|
|
184
|
+
return 0
|
|
185
|
+
return 0
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _dict(v: Any) -> dict:
|
|
189
|
+
return v if isinstance(v, dict) else {}
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _usage_from_dict(u: Any) -> Usage:
|
|
193
|
+
"""Map any provider's usage object onto Usage (anthropic, openai chat, responses, embeddings)."""
|
|
194
|
+
u = _dict(u)
|
|
195
|
+
inp = u.get("input_tokens")
|
|
196
|
+
if inp is None:
|
|
197
|
+
inp = u.get("prompt_tokens")
|
|
198
|
+
out = u.get("output_tokens")
|
|
199
|
+
if out is None:
|
|
200
|
+
out = u.get("completion_tokens")
|
|
201
|
+
cache_read = u.get("cache_read_input_tokens")
|
|
202
|
+
if cache_read is None:
|
|
203
|
+
cache_read = _dict(u.get("prompt_tokens_details")).get("cached_tokens")
|
|
204
|
+
if cache_read is None:
|
|
205
|
+
cache_read = _dict(u.get("input_tokens_details")).get("cached_tokens")
|
|
206
|
+
return Usage(
|
|
207
|
+
input_tokens=_int(inp),
|
|
208
|
+
output_tokens=_int(out),
|
|
209
|
+
cache_read_tokens=_int(cache_read),
|
|
210
|
+
cache_write_tokens=_int(u.get("cache_creation_input_tokens")),
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _gemini_usage(meta: dict) -> Usage:
|
|
215
|
+
"""Gemini usageMetadata -> Usage. promptTokenCount already includes cached tokens; thinking tokens
|
|
216
|
+
(thoughtsTokenCount) are billed as output but reported separately from candidatesTokenCount."""
|
|
217
|
+
return Usage(
|
|
218
|
+
input_tokens=_int(meta.get("promptTokenCount")) + _int(meta.get("toolUsePromptTokenCount")),
|
|
219
|
+
output_tokens=_int(meta.get("candidatesTokenCount")) + _int(meta.get("thoughtsTokenCount")),
|
|
220
|
+
cache_read_tokens=_int(meta.get("cachedContentTokenCount")),
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def extract_usage(style: str, resp: dict) -> Usage:
|
|
226
|
+
"""Token usage from a non-streaming response JSON. Missing fields -> 0."""
|
|
227
|
+
if not isinstance(resp, dict):
|
|
228
|
+
return Usage()
|
|
229
|
+
if "usageMetadata" in resp or style=="gemini":
|
|
230
|
+
meta = resp.get("usageMetadata")
|
|
231
|
+
if isinstance(meta,dict):
|
|
232
|
+
return _gemini_usage(meta)
|
|
233
|
+
usage = resp.get("usage")
|
|
234
|
+
if usage is None and isinstance(resp.get("response"), dict): # wrapped responses-API object
|
|
235
|
+
usage = resp["response"].get("usage")
|
|
236
|
+
return _usage_from_dict(usage)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _anthropic_text(content: Any) -> str:
|
|
240
|
+
if isinstance(content, str):
|
|
241
|
+
return content
|
|
242
|
+
if not isinstance(content, list):
|
|
243
|
+
return ""
|
|
244
|
+
out = []
|
|
245
|
+
for block in content:
|
|
246
|
+
if isinstance(block, dict) and block.get("type") == "text" and isinstance(block.get("text"), str):
|
|
247
|
+
out.append(block["text"])
|
|
248
|
+
return "".join(out)
|
|
249
|
+
|
|
250
|
+
def _openai_message_content(content: Any) -> str:
|
|
251
|
+
if isinstance(content, str):
|
|
252
|
+
return content
|
|
253
|
+
if isinstance(content, list): # content parts
|
|
254
|
+
out = []
|
|
255
|
+
for part in content:
|
|
256
|
+
if isinstance(part, dict) and part.get("type") in ("text", "output_text") and isinstance(part.get("text"), str):
|
|
257
|
+
out.append(part["text"])
|
|
258
|
+
elif isinstance(part, str):
|
|
259
|
+
out.append(part)
|
|
260
|
+
return "".join(out)
|
|
261
|
+
return ""
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _openai_chat_text(resp: dict) -> str:
|
|
265
|
+
choices = resp.get("choices")
|
|
266
|
+
if not isinstance(choices, list) or not choices:
|
|
267
|
+
return ""
|
|
268
|
+
first = choices[0]
|
|
269
|
+
if not isinstance(first, dict):
|
|
270
|
+
return ""
|
|
271
|
+
msg = first.get("message")
|
|
272
|
+
if isinstance(msg, dict):
|
|
273
|
+
return _openai_message_content(msg.get("content"))
|
|
274
|
+
if isinstance(first.get("text"), str): # legacy /v1/completions
|
|
275
|
+
return first["text"]
|
|
276
|
+
return ""
|
|
277
|
+
|
|
278
|
+
def _gemini_text(resp:dict)-> str:
|
|
279
|
+
candidates = resp.get("candidates")
|
|
280
|
+
if not isinstance(candidates,list) or not candidates:
|
|
281
|
+
return""
|
|
282
|
+
|
|
283
|
+
first = candidates[0]
|
|
284
|
+
if not isinstance(first, dict):
|
|
285
|
+
return ""
|
|
286
|
+
content = first.get("content") or {}
|
|
287
|
+
parts = content.get("parts") or []
|
|
288
|
+
return "".join(p.get("text", "") for p in parts if isinstance(p, dict) and isinstance(p.get("text"), str))
|
|
289
|
+
|
|
290
|
+
def _responses_text(resp: dict) -> str:
|
|
291
|
+
top = resp.get("output_text")
|
|
292
|
+
if isinstance(top, str) and top:
|
|
293
|
+
return top
|
|
294
|
+
output = resp.get("output")
|
|
295
|
+
if not isinstance(output, list):
|
|
296
|
+
return ""
|
|
297
|
+
out = []
|
|
298
|
+
for item in output:
|
|
299
|
+
if not isinstance(item, dict):
|
|
300
|
+
continue
|
|
301
|
+
content = item.get("content")
|
|
302
|
+
if not isinstance(content, list):
|
|
303
|
+
continue
|
|
304
|
+
for part in content:
|
|
305
|
+
if isinstance(part, dict) and part.get("type") == "output_text" and isinstance(part.get("text"), str):
|
|
306
|
+
out.append(part["text"])
|
|
307
|
+
return "".join(out)
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def extract_text(style: str, resp: dict) -> str:
|
|
311
|
+
"""Assistant text of a non-streaming response ("" for embeddings, errors, tool-only replies)."""
|
|
312
|
+
if not isinstance(resp, dict):
|
|
313
|
+
return ""
|
|
314
|
+
try:
|
|
315
|
+
if "choices" in resp:
|
|
316
|
+
return _openai_chat_text(resp)
|
|
317
|
+
if "output" in resp or "output_text" in resp:
|
|
318
|
+
return _responses_text(resp)
|
|
319
|
+
if "content" in resp:
|
|
320
|
+
return _anthropic_text(resp.get("content"))
|
|
321
|
+
if style == "gemini" or "candidates" in resp:
|
|
322
|
+
return _gemini_text(resp)
|
|
323
|
+
if isinstance(resp.get("response"), dict):
|
|
324
|
+
return _responses_text(resp["response"])
|
|
325
|
+
except Exception:
|
|
326
|
+
return ""
|
|
327
|
+
return ""
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def response_model(style: str, resp: dict) -> str | None:
|
|
331
|
+
"""Model name reported by the upstream response."""
|
|
332
|
+
if not isinstance(resp, dict):
|
|
333
|
+
return None
|
|
334
|
+
m = resp.get("model")
|
|
335
|
+
if not (isinstance(m, str) and m) and isinstance(resp.get("response"), dict):
|
|
336
|
+
m = resp["response"].get("model")
|
|
337
|
+
return m if isinstance(m, str) and m else None
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
# --------------------------------------------------------------------------------------------
|
|
341
|
+
# SSE
|
|
342
|
+
# --------------------------------------------------------------------------------------------
|
|
343
|
+
def _sse_events(raw: str) -> list[tuple[str, str]]:
|
|
344
|
+
"""Split an SSE transcript into (event_name, data) pairs. Multi-line data joined with \\n."""
|
|
345
|
+
text = raw.replace("\r\n", "\n").replace("\r", "\n")
|
|
346
|
+
events: list[tuple[str, str]] = []
|
|
347
|
+
name = ""
|
|
348
|
+
data: list[str] = []
|
|
349
|
+
for line in text.split("\n") + [""]:
|
|
350
|
+
if line == "":
|
|
351
|
+
if data:
|
|
352
|
+
events.append((name, "\n".join(data)))
|
|
353
|
+
name, data = "", []
|
|
354
|
+
continue
|
|
355
|
+
if line.startswith(":"):
|
|
356
|
+
continue
|
|
357
|
+
field, colon, value = line.partition(":")
|
|
358
|
+
if not colon:
|
|
359
|
+
field, value = line, ""
|
|
360
|
+
elif value.startswith(" "):
|
|
361
|
+
value = value[1:]
|
|
362
|
+
if field == "event":
|
|
363
|
+
name = value.strip()
|
|
364
|
+
elif field == "data":
|
|
365
|
+
data.append(value)
|
|
366
|
+
return events
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _merge_usage(dst: Usage, src: dict, override_output: bool = True) -> None:
|
|
370
|
+
"""Merge a (possibly partial) usage dict into dst; only fields present and > 0 overwrite."""
|
|
371
|
+
u = _usage_from_dict(src)
|
|
372
|
+
if u.input_tokens:
|
|
373
|
+
dst.input_tokens = u.input_tokens
|
|
374
|
+
if u.output_tokens and override_output:
|
|
375
|
+
dst.output_tokens = u.output_tokens
|
|
376
|
+
if u.cache_read_tokens:
|
|
377
|
+
dst.cache_read_tokens = u.cache_read_tokens
|
|
378
|
+
if u.cache_write_tokens:
|
|
379
|
+
dst.cache_write_tokens = u.cache_write_tokens
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def parse_sse(style: str, raw: str) -> tuple[Usage, str, str | None]:
|
|
383
|
+
"""Parse a full raw SSE transcript into (usage, assembled_text, model).
|
|
384
|
+
|
|
385
|
+
Handles anthropic messages streams, openai chat-completion chunks (incl. final usage chunk),
|
|
386
|
+
legacy completions chunks, and the openai Responses API event stream. Bad lines are skipped.
|
|
387
|
+
"""
|
|
388
|
+
usage = Usage()
|
|
389
|
+
text_parts: list[str] = []
|
|
390
|
+
model: str | None = None
|
|
391
|
+
completed_text = ""
|
|
392
|
+
if not isinstance(raw, str):
|
|
393
|
+
try:
|
|
394
|
+
raw = bytes(raw).decode("utf-8", "replace")
|
|
395
|
+
except Exception:
|
|
396
|
+
return usage, "", None
|
|
397
|
+
|
|
398
|
+
for name, data in _sse_events(raw):
|
|
399
|
+
payload = data.strip()
|
|
400
|
+
if not payload or payload == "[DONE]":
|
|
401
|
+
continue
|
|
402
|
+
try:
|
|
403
|
+
obj = json.loads(payload)
|
|
404
|
+
except Exception:
|
|
405
|
+
continue
|
|
406
|
+
if not isinstance(obj, dict):
|
|
407
|
+
continue
|
|
408
|
+
etype = obj.get("type") if isinstance(obj.get("type"), str) else name
|
|
409
|
+
|
|
410
|
+
try:
|
|
411
|
+
# ---- anthropic ----
|
|
412
|
+
if etype == "message_start":
|
|
413
|
+
msg = _dict(obj.get("message"))
|
|
414
|
+
if not model and isinstance(msg.get("model"), str) and msg["model"]:
|
|
415
|
+
model = msg["model"]
|
|
416
|
+
_merge_usage(usage, _dict(msg.get("usage")))
|
|
417
|
+
continue
|
|
418
|
+
if etype == "message_delta":
|
|
419
|
+
_merge_usage(usage, _dict(obj.get("usage")))
|
|
420
|
+
continue
|
|
421
|
+
if etype == "content_block_delta":
|
|
422
|
+
delta = _dict(obj.get("delta"))
|
|
423
|
+
if delta.get("type", "text_delta") == "text_delta" and isinstance(delta.get("text"), str):
|
|
424
|
+
text_parts.append(delta["text"])
|
|
425
|
+
continue
|
|
426
|
+
if etype in ("content_block_start", "content_block_stop", "message_stop", "ping", "error"):
|
|
427
|
+
continue
|
|
428
|
+
|
|
429
|
+
# ---- openai responses API ----
|
|
430
|
+
if etype == "response.output_text.delta":
|
|
431
|
+
if isinstance(obj.get("delta"), str):
|
|
432
|
+
text_parts.append(obj["delta"])
|
|
433
|
+
continue
|
|
434
|
+
if etype in ("response.created", "response.in_progress", "response.completed",
|
|
435
|
+
"response.incomplete", "response.failed"):
|
|
436
|
+
resp = _dict(obj.get("response"))
|
|
437
|
+
if isinstance(resp.get("model"), str) and resp["model"]:
|
|
438
|
+
model = model or resp["model"]
|
|
439
|
+
if isinstance(resp.get("usage"), dict):
|
|
440
|
+
_merge_usage(usage, resp["usage"])
|
|
441
|
+
if etype != "response.created" and etype != "response.in_progress":
|
|
442
|
+
completed_text = _responses_text(resp) or completed_text
|
|
443
|
+
continue
|
|
444
|
+
if isinstance(etype, str) and etype.startswith("response."):
|
|
445
|
+
continue
|
|
446
|
+
|
|
447
|
+
#Google Gemini API Responses
|
|
448
|
+
if style == "gemini" or "candidates" in obj or "usageMetadata" in obj:
|
|
449
|
+
# 1. Extract text delta from candidate parts
|
|
450
|
+
for cand in obj.get("candidates") or []:
|
|
451
|
+
for part in (cand.get("content") or {}).get("parts") or []:
|
|
452
|
+
if isinstance(part, dict) and isinstance(part.get("text"), str):
|
|
453
|
+
text_parts.append(part["text"])
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
# 2. Extract usage if present in this chunk
|
|
457
|
+
if isinstance(obj.get("usageMetadata"), dict):
|
|
458
|
+
g = _gemini_usage(obj["usageMetadata"])
|
|
459
|
+
# usageMetadata is cumulative per chunk: keep the latest non-empty totals
|
|
460
|
+
if g.input_tokens:
|
|
461
|
+
usage.input_tokens = g.input_tokens
|
|
462
|
+
if g.output_tokens:
|
|
463
|
+
usage.output_tokens = g.output_tokens
|
|
464
|
+
if g.cache_read_tokens:
|
|
465
|
+
usage.cache_read_tokens = g.cache_read_tokens
|
|
466
|
+
if not model and isinstance(obj.get("modelVersion"), str):
|
|
467
|
+
model = obj["modelVersion"]
|
|
468
|
+
continue
|
|
469
|
+
# ---- openai chat / legacy completions chunks ----
|
|
470
|
+
if not model and isinstance(obj.get("model"), str) and obj["model"]:
|
|
471
|
+
model = obj["model"]
|
|
472
|
+
choices = obj.get("choices")
|
|
473
|
+
if isinstance(choices, list):
|
|
474
|
+
for ch in choices:
|
|
475
|
+
if not isinstance(ch, dict) or _int(ch.get("index", 0)) != 0:
|
|
476
|
+
continue
|
|
477
|
+
delta = ch.get("delta")
|
|
478
|
+
if isinstance(delta, dict):
|
|
479
|
+
piece = _openai_message_content(delta.get("content"))
|
|
480
|
+
if piece:
|
|
481
|
+
text_parts.append(piece)
|
|
482
|
+
elif isinstance(ch.get("text"), str):
|
|
483
|
+
text_parts.append(ch["text"])
|
|
484
|
+
if isinstance(obj.get("usage"), dict):
|
|
485
|
+
_merge_usage(usage, obj["usage"])
|
|
486
|
+
except Exception:
|
|
487
|
+
continue
|
|
488
|
+
|
|
489
|
+
text = "".join(text_parts) or completed_text
|
|
490
|
+
return usage, text, model
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def _error_message(err: Any) -> str:
|
|
494
|
+
if isinstance(err, dict):
|
|
495
|
+
m = err.get("message")
|
|
496
|
+
t = err.get("type") or err.get("code")
|
|
497
|
+
if isinstance(m, str) and m:
|
|
498
|
+
return m[:500]
|
|
499
|
+
if isinstance(t, str) and t:
|
|
500
|
+
return t[:500]
|
|
501
|
+
return "upstream error"
|
|
502
|
+
if isinstance(err, str) and err:
|
|
503
|
+
return err[:500]
|
|
504
|
+
return ""
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def body_error(resp: Any) -> str | None:
|
|
508
|
+
"""Message of a top-level ``error`` in a (successful-status) JSON response, e.g. gateways that answer
|
|
509
|
+
HTTP 200 with ``{"error": {...}}``; None otherwise."""
|
|
510
|
+
if isinstance(resp, dict) and resp.get("error"):
|
|
511
|
+
return _error_message(resp["error"]) or "upstream error"
|
|
512
|
+
return None
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def sse_error(raw: str) -> str | None:
|
|
516
|
+
"""Message of the first in-band error in an SSE transcript (anthropic ``event: error``, openai
|
|
517
|
+
``data: {"error": ...}``, responses ``response.failed``); None when the stream carried no error."""
|
|
518
|
+
if not isinstance(raw, str) or "error" not in raw and "response.failed" not in raw:
|
|
519
|
+
return None
|
|
520
|
+
for name, data in _sse_events(raw):
|
|
521
|
+
payload = data.strip()
|
|
522
|
+
if not payload.startswith("{"):
|
|
523
|
+
continue
|
|
524
|
+
try:
|
|
525
|
+
obj = json.loads(payload)
|
|
526
|
+
except Exception:
|
|
527
|
+
continue
|
|
528
|
+
if not isinstance(obj, dict):
|
|
529
|
+
continue
|
|
530
|
+
etype = obj.get("type") if isinstance(obj.get("type"), str) else name
|
|
531
|
+
if etype == "error" or (obj.get("error") and "choices" not in obj and etype != "message_start"):
|
|
532
|
+
return _error_message(obj.get("error")) or _error_message(obj.get("message")) or "upstream stream error"
|
|
533
|
+
if etype == "response.failed":
|
|
534
|
+
return _error_message(_dict(obj.get("response")).get("error")) or "response failed"
|
|
535
|
+
return None
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
# --------------------------------------------------------------------------------------------
|
|
539
|
+
# estimation
|
|
540
|
+
# --------------------------------------------------------------------------------------------
|
|
541
|
+
def estimate_tokens(text: str) -> int:
|
|
542
|
+
"""Crude chars/4 token estimate: max(1, len//4) for non-empty text, else 0."""
|
|
543
|
+
if not text or not isinstance(text, str):
|
|
544
|
+
return 0
|
|
545
|
+
return max(1, len(text) // 4)
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
_B64_RE = re.compile(r"[A-Za-z0-9+/=_-]+")
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def _strings(obj: Any, out: list[str], depth: int = 0) -> None:
|
|
552
|
+
if depth > 64:
|
|
553
|
+
return
|
|
554
|
+
if isinstance(obj, str):
|
|
555
|
+
# skip inline binary payloads (base64 images/audio, data: URLs) — they aren't text tokens
|
|
556
|
+
if obj.startswith("data:") or (len(obj) > 512 and _B64_RE.fullmatch(obj[:512])):
|
|
557
|
+
return
|
|
558
|
+
out.append(obj)
|
|
559
|
+
elif isinstance(obj, dict):
|
|
560
|
+
for v in obj.values():
|
|
561
|
+
_strings(v, out, depth + 1)
|
|
562
|
+
elif isinstance(obj, (list, tuple)):
|
|
563
|
+
for v in obj:
|
|
564
|
+
_strings(v, out, depth + 1)
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def estimate_usage(req: dict | None, output_text: str) -> Usage:
|
|
568
|
+
"""Estimated usage: input from every string value in the request (keys excluded), output from text."""
|
|
569
|
+
strings: list[str] = []
|
|
570
|
+
if isinstance(req, (dict, list)):
|
|
571
|
+
_strings(req, strings)
|
|
572
|
+
return Usage(
|
|
573
|
+
input_tokens=estimate_tokens("\n".join(strings)),
|
|
574
|
+
output_tokens=estimate_tokens(output_text if isinstance(output_text, str) else ""),
|
|
575
|
+
estimated=True,
|
|
576
|
+
)
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
# --------------------------------------------------------------------------------------------
|
|
580
|
+
# errors + headers
|
|
581
|
+
# --------------------------------------------------------------------------------------------
|
|
582
|
+
def error_body(style: str, status: int, message: str, err_type: str = "llmtap_error") -> bytes:
|
|
583
|
+
"""Provider-shaped JSON error body so SDKs surface llmtap errors nicely."""
|
|
584
|
+
if style == "anthropic":
|
|
585
|
+
obj: dict[str, Any] = {"type": "error", "error": {"type": err_type, "message": message}}
|
|
586
|
+
elif style == "gemini":
|
|
587
|
+
obj = {"error": {"code": status, "message": message, "status": err_type.upper()}}
|
|
588
|
+
else:
|
|
589
|
+
obj = {"error": {"message": message, "type": err_type, "code": err_type}}
|
|
590
|
+
return json.dumps(obj).encode("utf-8")
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
HOP_BY_HOP: frozenset[str] = frozenset({
|
|
594
|
+
"connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailers",
|
|
595
|
+
"transfer-encoding", "upgrade", "host", "content-length", "accept-encoding",
|
|
596
|
+
})
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
def _items(headers: Mapping[str, str] | None) -> list[tuple[str, str]]:
|
|
600
|
+
try:
|
|
601
|
+
return [(str(k), str(v)) for k, v in (headers or {}).items()]
|
|
602
|
+
except Exception:
|
|
603
|
+
return []
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
def forward_headers(headers: Mapping[str, str]) -> dict[str, str]:
|
|
607
|
+
"""Headers to send upstream: drop hop-by-hop (+ those named in Connection) and x-llmtap-*."""
|
|
608
|
+
items = _items(headers)
|
|
609
|
+
extra: set[str] = set()
|
|
610
|
+
for k, v in items:
|
|
611
|
+
if k.lower() == "connection":
|
|
612
|
+
extra |= {t.strip().lower() for t in v.split(",") if t.strip()}
|
|
613
|
+
out: dict[str, str] = {}
|
|
614
|
+
for k, v in items:
|
|
615
|
+
lk = k.lower()
|
|
616
|
+
# `expect` is answered by llmtap itself (100-continue); the body is re-sent with a Content-Length
|
|
617
|
+
if lk in HOP_BY_HOP or lk in extra or lk == "expect" or lk.startswith("x-llmtap-"):
|
|
618
|
+
continue
|
|
619
|
+
out[k] = v
|
|
620
|
+
out["accept-encoding"] = "identity"
|
|
621
|
+
return out
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
def tags_from_headers(headers: Mapping[str, str]) -> dict[str, str]:
|
|
625
|
+
"""``X-LLMTap-Tag-Foo: bar`` -> {"foo": "bar"}; ``X-LLMTap-Session: s`` -> {"session": "s"}."""
|
|
626
|
+
tags: dict[str, str] = {}
|
|
627
|
+
for k, v in _items(headers):
|
|
628
|
+
lk = k.lower()
|
|
629
|
+
if lk == "x-llmtap-session":
|
|
630
|
+
tags["session"] = v.strip()
|
|
631
|
+
elif lk.startswith("x-llmtap-tag-"):
|
|
632
|
+
name = lk[len("x-llmtap-tag-"):].strip()
|
|
633
|
+
if name:
|
|
634
|
+
tags[name] = v.strip()
|
|
635
|
+
return tags
|