llmtap-proxy 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llmtap/__init__.py +2 -0
- llmtap/__main__.py +3 -0
- llmtap/budget.py +166 -0
- llmtap/cache.py +129 -0
- llmtap/cassette.py +172 -0
- llmtap/cli.py +1066 -0
- llmtap/config.py +112 -0
- llmtap/dashboard.py +445 -0
- llmtap/demo.py +244 -0
- llmtap/doctor.py +275 -0
- llmtap/guard.py +391 -0
- llmtap/mock.py +478 -0
- llmtap/models.py +75 -0
- llmtap/pricing.py +379 -0
- llmtap/providers.py +635 -0
- llmtap/proxy.py +1153 -0
- llmtap/pytest_plugin.py +444 -0
- llmtap/report.py +893 -0
- llmtap/store.py +319 -0
- llmtap/term.py +477 -0
- llmtap/ui/index.html +893 -0
- llmtap_proxy-0.2.0.dist-info/METADATA +609 -0
- llmtap_proxy-0.2.0.dist-info/RECORD +27 -0
- llmtap_proxy-0.2.0.dist-info/WHEEL +5 -0
- llmtap_proxy-0.2.0.dist-info/entry_points.txt +5 -0
- llmtap_proxy-0.2.0.dist-info/licenses/LICENSE +21 -0
- llmtap_proxy-0.2.0.dist-info/top_level.txt +1 -0
llmtap/demo.py
ADDED
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
"""Realistic synthetic traffic for `llmtap demo` (and the e2e tests).
|
|
2
|
+
|
|
3
|
+
Pure urllib client that talks to a running llmtap proxy: a mix of Anthropic / OpenAI calls,
|
|
4
|
+
streaming and non-streaming, several models, repeated prompts (cache hits), a prompt with a
|
|
5
|
+
leaked (fake) AWS key for the guard, and a burst of identical calls for the loop breaker.
|
|
6
|
+
Errors (429 / 409 / 400 / 5xx) are expected and swallowed.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import random
|
|
12
|
+
import threading
|
|
13
|
+
import urllib.error
|
|
14
|
+
import urllib.request
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
FAKE_AWS_KEY = "AKIAIOSFODNN7EXAMPLE"
|
|
19
|
+
|
|
20
|
+
_PY_SNIPPET = '''def merge_intervals(intervals):
|
|
21
|
+
intervals.sort()
|
|
22
|
+
merged = [intervals[0]]
|
|
23
|
+
for start, end in intervals[1:]:
|
|
24
|
+
if start <= merged[-1][1]:
|
|
25
|
+
merged[-1][1] = end
|
|
26
|
+
else:
|
|
27
|
+
merged.append([start, end])
|
|
28
|
+
return merged'''
|
|
29
|
+
|
|
30
|
+
_INCIDENT = (
|
|
31
|
+
"INC-4821 — 2026-10-28 14:02 UTC. Checkout API p99 latency rose from 180ms to 4.2s after deploy "
|
|
32
|
+
"v2.31.0. Root cause: a new ORM eager-load on Order.line_items produced N+1 queries against the "
|
|
33
|
+
"replica, saturating its connection pool (max 200). Mitigation: rollback at 14:37, pool size raised "
|
|
34
|
+
"to 400. Impact: 35 minutes, ~3.1% of checkouts failed, est. revenue impact $48k. Follow-ups: add "
|
|
35
|
+
"query-count assertions to CI, alert on replica pool saturation > 80%."
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
_WEATHER_TOOL = {
|
|
39
|
+
"type": "function",
|
|
40
|
+
"function": {
|
|
41
|
+
"name": "get_weather",
|
|
42
|
+
"description": "Get the current weather for a city",
|
|
43
|
+
"parameters": {
|
|
44
|
+
"type": "object",
|
|
45
|
+
"properties": {"city": {"type": "string"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}},
|
|
46
|
+
"required": ["city"],
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
_SEARCH_TOOL_ANTHROPIC = {
|
|
52
|
+
"name": "search_docs",
|
|
53
|
+
"description": "Full-text search over the internal engineering wiki",
|
|
54
|
+
"input_schema": {"type": "object", "properties": {"query": {"type": "string"}, "limit": {"type": "integer"}},
|
|
55
|
+
"required": ["query"]},
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class Call:
|
|
61
|
+
"""One scripted request."""
|
|
62
|
+
kind: str # "anthropic" | "chat" | "responses" | "embeddings"
|
|
63
|
+
model: str
|
|
64
|
+
feature: str
|
|
65
|
+
messages: list[dict[str, Any]] = field(default_factory=list)
|
|
66
|
+
system: str | None = None
|
|
67
|
+
stream: bool = False
|
|
68
|
+
max_tokens: int = 1024
|
|
69
|
+
extra: dict[str, Any] = field(default_factory=dict)
|
|
70
|
+
input: Any = None # responses / embeddings input
|
|
71
|
+
session: str = ""
|
|
72
|
+
|
|
73
|
+
def path(self) -> str:
|
|
74
|
+
return {"anthropic": "/v1/messages", "chat": "/v1/chat/completions",
|
|
75
|
+
"responses": "/v1/responses", "embeddings": "/v1/embeddings"}[self.kind]
|
|
76
|
+
|
|
77
|
+
def body(self) -> dict[str, Any]:
|
|
78
|
+
if self.kind == "anthropic":
|
|
79
|
+
b: dict[str, Any] = {"model": self.model, "max_tokens": self.max_tokens, "messages": self.messages}
|
|
80
|
+
if self.system:
|
|
81
|
+
b["system"] = self.system
|
|
82
|
+
elif self.kind == "chat":
|
|
83
|
+
msgs = ([{"role": "system", "content": self.system}] if self.system else []) + self.messages
|
|
84
|
+
b = {"model": self.model, "messages": msgs}
|
|
85
|
+
if self.stream:
|
|
86
|
+
b["stream_options"] = {"include_usage": True}
|
|
87
|
+
elif self.kind == "responses":
|
|
88
|
+
b = {"model": self.model, "input": self.input}
|
|
89
|
+
if self.system:
|
|
90
|
+
b["instructions"] = self.system
|
|
91
|
+
else:
|
|
92
|
+
return {"model": self.model, "input": self.input}
|
|
93
|
+
if self.stream:
|
|
94
|
+
b["stream"] = True
|
|
95
|
+
b.update(self.extra)
|
|
96
|
+
return b
|
|
97
|
+
|
|
98
|
+
def headers(self) -> dict[str, str]:
|
|
99
|
+
h = {"content-type": "application/json", "user-agent": "llmtap-demo/0.1",
|
|
100
|
+
"x-llmtap-tag-feature": self.feature}
|
|
101
|
+
if self.session:
|
|
102
|
+
h["x-llmtap-session"] = self.session
|
|
103
|
+
if self.kind == "anthropic":
|
|
104
|
+
h["x-api-key"] = "llmtap-demo-key"
|
|
105
|
+
h["anthropic-version"] = "2023-06-01"
|
|
106
|
+
else:
|
|
107
|
+
h["authorization"] = "Bearer llmtap-demo-key"
|
|
108
|
+
return h
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _u(text: str) -> dict[str, Any]:
|
|
112
|
+
return {"role": "user", "content": text}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _a(text: str) -> dict[str, Any]:
|
|
116
|
+
return {"role": "assistant", "content": text}
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _script() -> tuple[list[Call], list[Call]]:
|
|
120
|
+
"""Return (ordered first pass, pool for repeats)."""
|
|
121
|
+
code_review = Call("anthropic", "claude-sonnet-4-5", "code-review", stream=True,
|
|
122
|
+
system="You are a meticulous senior Python reviewer. Point out bugs first, then style.",
|
|
123
|
+
messages=[_u("Review this Python function for bugs and edge cases:\n\n" + _PY_SNIPPET)],
|
|
124
|
+
session="agent-run-17")
|
|
125
|
+
incident = Call("chat", "gpt-4o", "summarize",
|
|
126
|
+
system="You write crisp executive summaries for incident reports.",
|
|
127
|
+
messages=[_u("Summarize this incident report in 3 bullet points for the VP of Eng:\n\n" + _INCIDENT)])
|
|
128
|
+
sql = Call("anthropic", "claude-haiku-4-5", "sql-assistant",
|
|
129
|
+
system="You are a PostgreSQL expert. Return only SQL in a code block.",
|
|
130
|
+
messages=[_u("Write a SQL query that returns the top 10 customers by lifetime revenue in 2026, "
|
|
131
|
+
"joining orders(id, customer_id, total_cents, created_at) with customers(id, name, "
|
|
132
|
+
"email), excluding refunded orders.")])
|
|
133
|
+
translate = Call("chat", "gpt-4o-mini", "i18n", stream=True,
|
|
134
|
+
system="Translate faithfully. Keep placeholders like {name} untouched.",
|
|
135
|
+
messages=[_u("Translate to Japanese: \"Hi {name}, your order has shipped and should arrive "
|
|
136
|
+
"by Friday. Thanks for shopping with us!\"")])
|
|
137
|
+
architecture = Call("anthropic", "claude-opus-4-1", "architecture", stream=True,
|
|
138
|
+
system="You are a staff engineer. Be concrete about trade-offs.",
|
|
139
|
+
messages=[_u("We're outgrowing a single Postgres instance at ~12k writes/sec. Options?"),
|
|
140
|
+
_a("Main options: vertical scaling, read replicas + CQRS, partitioning, or "
|
|
141
|
+
"sharding with Citus. What dominates your write load?"),
|
|
142
|
+
_u("Mostly append-only event rows, 90% of reads hit the last 7 days. "
|
|
143
|
+
"Should we partition by time or shard by tenant?")],
|
|
144
|
+
session="design-review")
|
|
145
|
+
weather_tool = Call("chat", "gpt-5", "agent-tools",
|
|
146
|
+
system="You are a travel assistant. Use tools when you need live data.",
|
|
147
|
+
messages=[_u("Should I pack an umbrella for Lisbon tomorrow?")],
|
|
148
|
+
extra={"tools": [_WEATHER_TOOL], "tool_choice": "auto"}, session="agent-run-17")
|
|
149
|
+
leaked = Call("anthropic", "claude-sonnet-4-5", "devops",
|
|
150
|
+
messages=[_u("Why does this deploy script fail with AccessDenied?\n\n"
|
|
151
|
+
f"export AWS_ACCESS_KEY_ID={FAKE_AWS_KEY}\n"
|
|
152
|
+
"aws s3 sync ./build s3://prod-web-assets --delete")])
|
|
153
|
+
embed = Call("embeddings", "text-embedding-3-small", "rag-index",
|
|
154
|
+
input=["How do I rotate API keys?", "Rotating credentials without downtime",
|
|
155
|
+
"Quarterly OKR planning template"])
|
|
156
|
+
unit_tests = Call("chat", "gpt-4o-mini", "test-gen",
|
|
157
|
+
messages=[_u("Write pytest unit tests for merge_intervals covering empty input, "
|
|
158
|
+
"touching intervals and unsorted input.\n\n" + _PY_SNIPPET)])
|
|
159
|
+
commit_msg = Call("anthropic", "claude-haiku-4-5", "git-helper", stream=True,
|
|
160
|
+
system="Write conventional-commit messages. Max 72 chars in the subject.",
|
|
161
|
+
messages=[_u("Write a commit message for: switched session store from Redis to Postgres "
|
|
162
|
+
"advisory locks, removed redis dependency, added migration 0042.")])
|
|
163
|
+
wiki_search = Call("anthropic", "claude-sonnet-4-5", "support-bot",
|
|
164
|
+
system="You answer engineering questions using the internal wiki. Cite pages.",
|
|
165
|
+
messages=[_u("What's our on-call escalation policy for SEV-2 incidents?")],
|
|
166
|
+
extra={"tools": [_SEARCH_TOOL_ANTHROPIC]}, session="support-4412")
|
|
167
|
+
responses_api = Call("responses", "gpt-5", "release-notes", stream=True,
|
|
168
|
+
system="You write friendly, scannable release notes.",
|
|
169
|
+
input="Draft release notes for v2.32: dark mode, 40% faster search, "
|
|
170
|
+
"fixed CSV export timezone bug, deprecated the v1 webhooks API.")
|
|
171
|
+
regex = Call("chat", "gpt-4o", "dev-tools",
|
|
172
|
+
messages=[_u("Give me a regex that matches ISO-8601 dates with optional time and timezone, "
|
|
173
|
+
"and explain each group.")])
|
|
174
|
+
email = Call("anthropic", "claude-sonnet-4-5", "email-drafts",
|
|
175
|
+
messages=[_u("Draft a polite email to a customer explaining that their feature request "
|
|
176
|
+
"(SAML SSO for the Starter plan) is on our Q1 roadmap.")])
|
|
177
|
+
classify = Call("chat", "gpt-4o-mini", "triage", stream=True,
|
|
178
|
+
system="Classify support tickets into: billing, bug, feature_request, account, other. "
|
|
179
|
+
"Reply with JSON {\"label\":..., \"confidence\":...}.",
|
|
180
|
+
messages=[_u("Ticket #88123: 'I was charged twice for October and the invoice page "
|
|
181
|
+
"shows a 500 error.'")])
|
|
182
|
+
docstring = Call("anthropic", "claude-haiku-4-5", "docs",
|
|
183
|
+
messages=[_u("Write a Google-style docstring for merge_intervals:\n\n" + _PY_SNIPPET)])
|
|
184
|
+
explain_k8s = Call("chat", "gpt-5", "dev-tools",
|
|
185
|
+
messages=[_u("Explain why my Kubernetes pod is in CrashLoopBackOff with exit code 137 "
|
|
186
|
+
"and how to confirm it's an OOM kill.")])
|
|
187
|
+
story_points = Call("anthropic", "claude-opus-4-1", "planning",
|
|
188
|
+
system="You are a pragmatic engineering manager.",
|
|
189
|
+
messages=[_u("Break the epic 'Usage-based billing' into 6-8 tickets with rough "
|
|
190
|
+
"story-point estimates and dependencies.")])
|
|
191
|
+
outage = Call("chat", "gpt-4o", "summarize",
|
|
192
|
+
messages=[_u("Summarize the status page update (simulated upstream outage MOCK_ERROR_500)")])
|
|
193
|
+
retry_loop = Call("chat", "gpt-4o-mini", "agent-loop",
|
|
194
|
+
system="You are an autonomous coding agent.",
|
|
195
|
+
messages=[_u("The test tests/test_billing.py::test_proration still fails. "
|
|
196
|
+
"Fix it and try again.")], session="agent-run-17")
|
|
197
|
+
|
|
198
|
+
first = [code_review, incident, sql, translate, architecture, weather_tool, leaked, embed,
|
|
199
|
+
code_review, incident, # repeats -> cache hits
|
|
200
|
+
retry_loop, retry_loop, retry_loop, retry_loop, retry_loop, retry_loop, # runaway agent burst
|
|
201
|
+
unit_tests, commit_msg, wiki_search, responses_api, regex, email, classify, docstring,
|
|
202
|
+
explain_k8s, story_points, outage, sql, translate]
|
|
203
|
+
pool = [code_review, incident, sql, translate, architecture, weather_tool, embed, unit_tests, commit_msg,
|
|
204
|
+
wiki_search, responses_api, regex, email, classify, docstring, explain_k8s, story_points]
|
|
205
|
+
return first, pool
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def send(base_url: str, call: Call, timeout: float = 30.0) -> tuple[int, bytes]:
|
|
209
|
+
"""Send one call through the proxy. Never raises; returns (status, body) — status 0 on connection error."""
|
|
210
|
+
data = json.dumps(call.body()).encode("utf-8")
|
|
211
|
+
req = urllib.request.Request(base_url.rstrip("/") + call.path(), data=data, method="POST",
|
|
212
|
+
headers=call.headers())
|
|
213
|
+
try:
|
|
214
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
215
|
+
return resp.status, resp.read()
|
|
216
|
+
except urllib.error.HTTPError as e:
|
|
217
|
+
try:
|
|
218
|
+
body = e.read()
|
|
219
|
+
except Exception:
|
|
220
|
+
body = b""
|
|
221
|
+
return e.code, body
|
|
222
|
+
except Exception:
|
|
223
|
+
return 0, b""
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def generate_traffic(base_url: str, count: int = 40, delay_s: float = 0.3,
|
|
227
|
+
stop: threading.Event | None = None) -> None:
|
|
228
|
+
"""Send `count` realistic requests to the llmtap proxy at `base_url`, pausing ~delay_s between them."""
|
|
229
|
+
first, pool = _script()
|
|
230
|
+
rng = random.Random(4141)
|
|
231
|
+
for i in range(count):
|
|
232
|
+
if stop is not None and stop.is_set():
|
|
233
|
+
return
|
|
234
|
+
call = first[i] if i < len(first) else rng.choice(pool)
|
|
235
|
+
send(base_url, call)
|
|
236
|
+
if delay_s > 0 and i < count - 1:
|
|
237
|
+
pause = delay_s * rng.uniform(0.4, 1.6)
|
|
238
|
+
if call.feature == "agent-loop":
|
|
239
|
+
pause = delay_s * 0.15 # the runaway agent hammers quickly
|
|
240
|
+
if stop is not None:
|
|
241
|
+
if stop.wait(pause):
|
|
242
|
+
return
|
|
243
|
+
else:
|
|
244
|
+
threading.Event().wait(pause)
|
llmtap/doctor.py
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"""`llmtap doctor`: environment checks with fix hints. Never prints secret values."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import os
|
|
5
|
+
import shutil
|
|
6
|
+
import socket
|
|
7
|
+
import sqlite3
|
|
8
|
+
import subprocess
|
|
9
|
+
import sys
|
|
10
|
+
import tomllib
|
|
11
|
+
import urllib.error
|
|
12
|
+
import urllib.request
|
|
13
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
from urllib.parse import urlsplit
|
|
18
|
+
|
|
19
|
+
from llmtap import __version__
|
|
20
|
+
|
|
21
|
+
OK, FAIL, WARN, INFO = "ok", "fail", "warn", "info"
|
|
22
|
+
_GLYPH = {OK: "✓", FAIL: "✗", WARN: "!", INFO: "·"}
|
|
23
|
+
_COLOR = {OK: "green", FAIL: "red", WARN: "yellow", INFO: "gray"}
|
|
24
|
+
LOCAL_HOSTS = {"localhost", "127.0.0.1", "::1", "0.0.0.0"}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class Check:
|
|
29
|
+
level: str # ok | fail | warn | info
|
|
30
|
+
title: str
|
|
31
|
+
detail: str = ""
|
|
32
|
+
hint: str = ""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# --------------------------------------------------------------------------------------------
|
|
36
|
+
# individual checks (small, monkeypatch-friendly)
|
|
37
|
+
|
|
38
|
+
def check_python() -> Check:
|
|
39
|
+
v = sys.version_info
|
|
40
|
+
ver = f"{v.major}.{v.minor}.{v.micro}"
|
|
41
|
+
if (v.major, v.minor) >= (3, 11):
|
|
42
|
+
return Check(OK, f"Python {ver}")
|
|
43
|
+
return Check(FAIL, f"Python {ver}", "llmtap needs Python 3.11 or newer",
|
|
44
|
+
"install Python 3.11+ (e.g. `pyenv install 3.12`) and reinstall llmtap")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _listener_info(port: int) -> str:
|
|
48
|
+
"""Best-effort 'who is listening' via ss/lsof; empty string when unknown."""
|
|
49
|
+
for cmd in (["ss", "-ltnpH", f"sport = :{port}"], ["lsof", "-nP", f"-iTCP:{port}", "-sTCP:LISTEN"]):
|
|
50
|
+
if not shutil.which(cmd[0]):
|
|
51
|
+
continue
|
|
52
|
+
try:
|
|
53
|
+
out = subprocess.run(cmd, capture_output=True, text=True, timeout=3).stdout.strip()
|
|
54
|
+
except Exception:
|
|
55
|
+
continue
|
|
56
|
+
if out:
|
|
57
|
+
line = out.splitlines()[-1]
|
|
58
|
+
if "users:" in line:
|
|
59
|
+
return line[line.index("users:"):][:80]
|
|
60
|
+
return " ".join(line.split())[:80]
|
|
61
|
+
return ""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _port_in_use(host: str, port: int) -> bool:
|
|
65
|
+
"""True when something is already bound to host:port."""
|
|
66
|
+
fam = socket.AF_INET6 if ":" in host else socket.AF_INET
|
|
67
|
+
s = socket.socket(fam, socket.SOCK_STREAM)
|
|
68
|
+
try:
|
|
69
|
+
s.settimeout(1.0)
|
|
70
|
+
s.bind((host, port))
|
|
71
|
+
return False
|
|
72
|
+
except OSError:
|
|
73
|
+
return True
|
|
74
|
+
finally:
|
|
75
|
+
s.close()
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _is_llmtap(host: str, port: int) -> str | None:
|
|
79
|
+
"""Return the llmtap version if /_llmtap/health answers on host:port."""
|
|
80
|
+
import json
|
|
81
|
+
try:
|
|
82
|
+
h = "127.0.0.1" if host in ("0.0.0.0", "") else host
|
|
83
|
+
with urllib.request.urlopen(f"http://{h}:{port}/_llmtap/health", timeout=1.5) as r:
|
|
84
|
+
data = json.loads(r.read().decode("utf-8", "replace"))
|
|
85
|
+
if isinstance(data, dict) and data.get("ok"):
|
|
86
|
+
return str(data.get("version") or "?")
|
|
87
|
+
except Exception:
|
|
88
|
+
pass
|
|
89
|
+
return None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def check_port(host: str, port: int) -> Check:
|
|
93
|
+
if port == 0:
|
|
94
|
+
return Check(OK, "Port 0 (any free port)")
|
|
95
|
+
if not _port_in_use(host, port):
|
|
96
|
+
return Check(OK, f"Port {port} is free")
|
|
97
|
+
ver = _is_llmtap(host, port)
|
|
98
|
+
if ver is not None:
|
|
99
|
+
return Check(OK, f"Port {port}: llmtap v{ver} is already running", f"dashboard: http://{host}:{port}/_llmtap/")
|
|
100
|
+
who = _listener_info(port)
|
|
101
|
+
return Check(WARN, f"Port {port} is in use" + (f" ({who})" if who else ""), "",
|
|
102
|
+
f"stop that process, or run `llmtap serve --port {port + 1}`")
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _db_path(cfg: Any) -> Path:
|
|
106
|
+
if cfg.db_path:
|
|
107
|
+
return Path(cfg.db_path).expanduser()
|
|
108
|
+
from llmtap.config import llmtap_home
|
|
109
|
+
return llmtap_home() / "llmtap.db"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def check_db(cfg: Any) -> Check:
|
|
113
|
+
if cfg.db_path == ":memory:":
|
|
114
|
+
return Check(OK, "Database: in-memory (nothing persisted)")
|
|
115
|
+
p = _db_path(cfg)
|
|
116
|
+
probe = p if p.exists() else p.parent
|
|
117
|
+
while not probe.exists() and probe != probe.parent:
|
|
118
|
+
probe = probe.parent
|
|
119
|
+
if not os.access(probe, os.W_OK):
|
|
120
|
+
return Check(FAIL, f"Database path not writable: {p}", "", f"fix permissions (`chmod u+w {probe}`) or use `--db PATH`")
|
|
121
|
+
detail = ""
|
|
122
|
+
if p.is_file():
|
|
123
|
+
try:
|
|
124
|
+
c = sqlite3.connect(f"file:{p}?mode=ro", uri=True, timeout=2)
|
|
125
|
+
try:
|
|
126
|
+
n = c.execute("SELECT COUNT(*) FROM exchanges").fetchone()[0]
|
|
127
|
+
detail = f"{n:,} exchanges recorded"
|
|
128
|
+
finally:
|
|
129
|
+
c.close()
|
|
130
|
+
except sqlite3.Error as e:
|
|
131
|
+
return Check(FAIL, f"Database unreadable: {p}", str(e), "move the file aside or point `--db` elsewhere")
|
|
132
|
+
return Check(OK, f"Database writable: {p}", detail)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def check_llmtap_env() -> Check:
|
|
136
|
+
names = sorted(k for k in os.environ if k.startswith("LLMTAP_"))
|
|
137
|
+
if not names:
|
|
138
|
+
return Check(INFO, "No LLMTAP_* env vars set", "using defaults / config file")
|
|
139
|
+
return Check(OK, "LLMTAP_* env vars set", ", ".join(names))
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _base_url_check(var: str, cfg: Any) -> Check:
|
|
143
|
+
val = os.environ.get(var)
|
|
144
|
+
if not val:
|
|
145
|
+
return Check(WARN, f"{var} not set", "this SDK talks to the provider directly",
|
|
146
|
+
'run `eval "$(llmtap env)"` or `llmtap wrap -- <your command>`')
|
|
147
|
+
try:
|
|
148
|
+
u = urlsplit(val if "://" in val else "http://" + val)
|
|
149
|
+
host, port = (u.hostname or ""), (u.port or (443 if u.scheme == "https" else 80))
|
|
150
|
+
except ValueError:
|
|
151
|
+
return Check(WARN, f"{var} is not a valid URL")
|
|
152
|
+
shown = f"{u.scheme}://{host}" + (f":{u.port}" if u.port else "")
|
|
153
|
+
if host in LOCAL_HOSTS and port in (cfg.port, 4141):
|
|
154
|
+
return Check(OK, f"{var} points at llmtap", shown)
|
|
155
|
+
if host in LOCAL_HOSTS:
|
|
156
|
+
return Check(WARN, f"{var} points at a local server on port {port}, not llmtap's {cfg.port}", shown,
|
|
157
|
+
'run `eval "$(llmtap env)"` to point it at llmtap')
|
|
158
|
+
return Check(WARN, f"{var} is set to {host}: traffic bypasses llmtap", shown,
|
|
159
|
+
'run `eval "$(llmtap env)"` (or `llmtap wrap -- <cmd>`)')
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def check_key(var: str) -> Check:
|
|
163
|
+
if os.environ.get(var):
|
|
164
|
+
return Check(OK, f"{var} set")
|
|
165
|
+
return Check(INFO, f"{var} not set", "", "export it if you call that provider (or use `llmtap demo` with no keys)")
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def check_sqlite() -> Check:
|
|
169
|
+
v = sqlite3.sqlite_version
|
|
170
|
+
nums = tuple(int(x) for x in v.split(".")[:3] if x.isdigit())
|
|
171
|
+
try:
|
|
172
|
+
c = sqlite3.connect(":memory:")
|
|
173
|
+
c.execute("SELECT json_valid('{}'), json_array_length('[1]')").fetchone()
|
|
174
|
+
c.close()
|
|
175
|
+
except sqlite3.Error:
|
|
176
|
+
return Check(FAIL, f"SQLite {v} lacks JSON1 functions", "llmtap stats/report need json_* support",
|
|
177
|
+
"use a Python build linked against SQLite >= 3.38")
|
|
178
|
+
if nums < (3, 45):
|
|
179
|
+
return Check(OK, f"SQLite {v}", "works, but 3.45+ is recommended (newer JSON handling)")
|
|
180
|
+
return Check(OK, f"SQLite {v}")
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def check_config(path: str | None) -> Check:
|
|
184
|
+
cands = [Path(path)] if path else [Path("llmtap.toml"), Path.home() / ".config" / "llmtap" / "config.toml"]
|
|
185
|
+
for p in cands:
|
|
186
|
+
if p.is_file():
|
|
187
|
+
try:
|
|
188
|
+
tomllib.loads(p.read_text())
|
|
189
|
+
except (tomllib.TOMLDecodeError, OSError, UnicodeDecodeError) as e:
|
|
190
|
+
return Check(FAIL, f"Config file is invalid: {p}", str(e), "fix the TOML syntax")
|
|
191
|
+
return Check(OK, f"Config file: {p}")
|
|
192
|
+
if path:
|
|
193
|
+
return Check(FAIL, f"Config file not found: {path}", "", "check the --config path")
|
|
194
|
+
return Check(INFO, "No config file found", "using defaults", "copy llmtap.example.toml to llmtap.toml to customize")
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def probe_host(url: str, timeout: float = 5.0) -> tuple[bool, str]:
|
|
198
|
+
"""HEAD (fallback GET) the host. ANY HTTP response, even 401/404/405, means reachable."""
|
|
199
|
+
last = ""
|
|
200
|
+
for method in ("HEAD", "GET"):
|
|
201
|
+
try:
|
|
202
|
+
req = urllib.request.Request(url, method=method, headers={"User-Agent": f"llmtap-doctor/{__version__}"})
|
|
203
|
+
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
204
|
+
return True, f"HTTP {r.status}"
|
|
205
|
+
except urllib.error.HTTPError as e:
|
|
206
|
+
return True, f"HTTP {e.code}"
|
|
207
|
+
except Exception as e: # URLError, timeout, ssl ...
|
|
208
|
+
last = getattr(e, "reason", None) or e
|
|
209
|
+
return False, str(last)[:80]
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def check_online(cfg: Any, timeout: float = 5.0) -> list[Check]:
|
|
213
|
+
ups = {k: v for k, v in cfg.upstreams.items()}
|
|
214
|
+
items = sorted(ups.items(), key=lambda kv: (urlsplit(kv[1]["base"]).hostname in LOCAL_HOSTS, kv[0]))
|
|
215
|
+
|
|
216
|
+
def one(kv: tuple[str, dict[str, str]]) -> Check:
|
|
217
|
+
name, spec = kv
|
|
218
|
+
host = urlsplit(spec["base"]).hostname or spec["base"]
|
|
219
|
+
ok, detail = probe_host(spec["base"], timeout)
|
|
220
|
+
if ok:
|
|
221
|
+
return Check(OK, f"{name} reachable ({host})", detail)
|
|
222
|
+
local = host in LOCAL_HOSTS
|
|
223
|
+
return Check(INFO if local else WARN, f"{name} unreachable ({host})", detail,
|
|
224
|
+
"start it locally if you use it" if local else "check network/proxy/firewall settings")
|
|
225
|
+
|
|
226
|
+
with ThreadPoolExecutor(max_workers=min(8, max(1, len(items)))) as ex:
|
|
227
|
+
return list(ex.map(one, items))
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
# --------------------------------------------------------------------------------------------
|
|
231
|
+
|
|
232
|
+
def run_checks(cfg: Any, config_path: str | None = None, online: bool = False) -> list[Check]:
|
|
233
|
+
checks = [
|
|
234
|
+
check_python(),
|
|
235
|
+
check_port(cfg.host, cfg.port),
|
|
236
|
+
check_db(cfg),
|
|
237
|
+
check_sqlite(),
|
|
238
|
+
check_config(config_path),
|
|
239
|
+
check_llmtap_env(),
|
|
240
|
+
_base_url_check("ANTHROPIC_BASE_URL", cfg),
|
|
241
|
+
_base_url_check("OPENAI_BASE_URL", cfg),
|
|
242
|
+
check_key("ANTHROPIC_API_KEY"),
|
|
243
|
+
check_key("OPENAI_API_KEY"),
|
|
244
|
+
]
|
|
245
|
+
if online:
|
|
246
|
+
checks.extend(check_online(cfg))
|
|
247
|
+
return checks
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def render(checks: list[Check], color: bool = False) -> str:
|
|
251
|
+
from llmtap import term
|
|
252
|
+
lines = [term.paint(f"llmtap doctor · v{__version__}", "bold", color=color), ""]
|
|
253
|
+
for c in checks:
|
|
254
|
+
g = term.paint(_GLYPH[c.level], "bold", _COLOR[c.level], color=color)
|
|
255
|
+
line = f" {g} {c.title}"
|
|
256
|
+
if c.detail:
|
|
257
|
+
line += term.paint(f" {c.detail}", "dim", color=color)
|
|
258
|
+
lines.append(line)
|
|
259
|
+
if c.hint and c.level != OK:
|
|
260
|
+
lines.append(" " + term.paint("→ " + c.hint, "cyan", color=color))
|
|
261
|
+
fails = sum(c.level == FAIL for c in checks)
|
|
262
|
+
warns = sum(c.level == WARN for c in checks)
|
|
263
|
+
lines.append("")
|
|
264
|
+
if fails:
|
|
265
|
+
lines.append(term.paint(f"{fails} problem{'s' if fails != 1 else ''} to fix", "bold", "red", color=color)
|
|
266
|
+
+ (f", {warns} warning{'s' if warns != 1 else ''}" if warns else ""))
|
|
267
|
+
elif warns:
|
|
268
|
+
lines.append(term.paint(f"All good, with {warns} warning{'s' if warns != 1 else ''}", "bold", "yellow", color=color))
|
|
269
|
+
else:
|
|
270
|
+
lines.append(term.paint("All good — you're ready to tap.", "bold", "green", color=color))
|
|
271
|
+
return "\n".join(lines)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def exit_code(checks: list[Check]) -> int:
|
|
275
|
+
return 1 if any(c.level == FAIL for c in checks) else 0
|