toolbloat 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toolbloat/__init__.py +3 -0
- toolbloat/__main__.py +3 -0
- toolbloat/cli.py +439 -0
- toolbloat/config.py +175 -0
- toolbloat/data/demo/results.json +3677 -0
- toolbloat/discover.py +100 -0
- toolbloat/errors.py +25 -0
- toolbloat/evaluate.py +245 -0
- toolbloat/intents.py +167 -0
- toolbloat/llm.py +261 -0
- toolbloat/metrics.py +197 -0
- toolbloat/py.typed +0 -0
- toolbloat/report.py +67 -0
- toolbloat/templates/report.html.j2 +376 -0
- toolbloat/terminal.py +132 -0
- toolbloat/tokens.py +55 -0
- toolbloat-0.2.0.dist-info/METADATA +296 -0
- toolbloat-0.2.0.dist-info/RECORD +22 -0
- toolbloat-0.2.0.dist-info/WHEEL +4 -0
- toolbloat-0.2.0.dist-info/entry_points.txt +2 -0
- toolbloat-0.2.0.dist-info/licenses/LICENSE +21 -0
- toolbloat-0.2.0.dist-info/licenses/NOTICE +232 -0
toolbloat/llm.py
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
"""One OpenAI-compatible client with provider presets, a budget and backoff."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import random
|
|
7
|
+
import threading
|
|
8
|
+
import time
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
|
|
11
|
+
from .errors import (
|
|
12
|
+
BudgetExceeded,
|
|
13
|
+
MissingApiKey,
|
|
14
|
+
ModelNotFound,
|
|
15
|
+
ProviderUnreachable,
|
|
16
|
+
ToolbloatError,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
# base_url, env var holding the key (None = no key needed)
|
|
20
|
+
PROVIDERS: dict[str, tuple[str, str | None]] = {
|
|
21
|
+
"openrouter": ("https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"),
|
|
22
|
+
"openai": ("https://api.openai.com/v1", "OPENAI_API_KEY"),
|
|
23
|
+
"mistral": ("https://api.mistral.ai/v1", "MISTRAL_API_KEY"),
|
|
24
|
+
"ollama": ("http://localhost:11434/v1", None),
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
# USD per million tokens (input, output). Used for --max-cost and the monthly
|
|
28
|
+
# projection. Unknown models fall back to --price-in/--price-out or 0 with a warning.
|
|
29
|
+
# Mistral list prices from https://docs.mistral.ai/inference/pricing (checked 2026-11-03):
|
|
30
|
+
# mistral-small-latest = Mistral Small 4, ministral-8b-latest = Ministral 3 8B.
|
|
31
|
+
# Mistral Medium 3.5 is not listed there; we budget it at the Mistral Large 4 price
|
|
32
|
+
# as a conservative upper bound.
|
|
33
|
+
PRICES: dict[str, tuple[float, float]] = {
|
|
34
|
+
"mistral:mistral-small-latest": (0.15, 0.60),
|
|
35
|
+
"mistral:ministral-8b-latest": (0.15, 0.15),
|
|
36
|
+
"mistral:ministral-14b-latest": (0.20, 0.20),
|
|
37
|
+
"mistral:ministral-3b-latest": (0.10, 0.10),
|
|
38
|
+
"mistral:codestral-latest": (0.30, 0.90),
|
|
39
|
+
"mistral:mistral-medium-latest": (0.70, 2.10),
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def ollama_base_url() -> str:
|
|
44
|
+
host = os.environ.get("OLLAMA_HOST", "").strip()
|
|
45
|
+
if not host:
|
|
46
|
+
return PROVIDERS["ollama"][0]
|
|
47
|
+
if not host.startswith("http"):
|
|
48
|
+
host = "http://" + host
|
|
49
|
+
host = host.rstrip("/")
|
|
50
|
+
return host if host.endswith("/v1") else host + "/v1"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class ModelSpec:
|
|
55
|
+
provider: str
|
|
56
|
+
model: str
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def id(self) -> str:
|
|
60
|
+
return f"{self.provider}:{self.model}"
|
|
61
|
+
|
|
62
|
+
@classmethod
|
|
63
|
+
def parse(cls, text: str) -> ModelSpec:
|
|
64
|
+
provider, sep, model = text.partition(":")
|
|
65
|
+
if not sep or not model:
|
|
66
|
+
raise ToolbloatError(
|
|
67
|
+
f"Model '{text}' must look like provider:model, e.g. ollama:qwen3:4b "
|
|
68
|
+
f"or mistral:mistral-small-latest. Providers: {', '.join(PROVIDERS)}."
|
|
69
|
+
)
|
|
70
|
+
if provider not in PROVIDERS:
|
|
71
|
+
raise ToolbloatError(f"Unknown provider '{provider}'. Use one of: {', '.join(PROVIDERS)}.")
|
|
72
|
+
return cls(provider, model)
|
|
73
|
+
|
|
74
|
+
def price(self) -> tuple[float, float]:
|
|
75
|
+
if self.provider == "ollama":
|
|
76
|
+
return (0.0, 0.0)
|
|
77
|
+
return PRICES.get(self.id, (0.0, 0.0))
|
|
78
|
+
|
|
79
|
+
def has_known_price(self) -> bool:
|
|
80
|
+
return self.provider == "ollama" or self.id in PRICES
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass
|
|
84
|
+
class Budget:
|
|
85
|
+
"""Tracks spend and refuses a call once the cap would be passed."""
|
|
86
|
+
|
|
87
|
+
max_cost: float
|
|
88
|
+
spent: dict[str, float] = field(default_factory=dict)
|
|
89
|
+
_lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def total(self) -> float:
|
|
93
|
+
return sum(self.spent.values())
|
|
94
|
+
|
|
95
|
+
def check(self, estimate: float = 0.0) -> None:
|
|
96
|
+
if self.total + estimate > self.max_cost:
|
|
97
|
+
raise BudgetExceeded(
|
|
98
|
+
f"Budget reached: spent ${self.total:.4f} of --max-cost ${self.max_cost:.2f}"
|
|
99
|
+
f" (next call estimated at ${estimate:.4f}). Partial results were kept;"
|
|
100
|
+
" raise --max-cost to continue."
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
def add(self, provider: str, usd: float) -> None:
|
|
104
|
+
with self._lock:
|
|
105
|
+
self.spent[provider] = self.spent.get(provider, 0.0) + usd
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@dataclass
|
|
109
|
+
class Completion:
|
|
110
|
+
tool_name: str | None
|
|
111
|
+
arguments: str | None
|
|
112
|
+
text: str
|
|
113
|
+
prompt_tokens: int
|
|
114
|
+
completion_tokens: int
|
|
115
|
+
cost: float
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class LLM:
|
|
119
|
+
"""Thin wrapper around openai.OpenAI for one model."""
|
|
120
|
+
|
|
121
|
+
def __init__(self, spec: ModelSpec, budget: Budget, *, max_retries: int = 9, timeout: float | None = None):
|
|
122
|
+
from openai import OpenAI
|
|
123
|
+
|
|
124
|
+
base_url, key_var = PROVIDERS[spec.provider]
|
|
125
|
+
if spec.provider == "ollama":
|
|
126
|
+
base_url = ollama_base_url()
|
|
127
|
+
api_key = "ollama"
|
|
128
|
+
else:
|
|
129
|
+
api_key = os.environ.get(key_var or "", "").strip()
|
|
130
|
+
if not api_key:
|
|
131
|
+
raise MissingApiKey(
|
|
132
|
+
f"{key_var} is not set. Export it to use {spec.id}, or pick a local model such as ollama:qwen3:4b."
|
|
133
|
+
)
|
|
134
|
+
self.spec = spec
|
|
135
|
+
self.budget = budget
|
|
136
|
+
self.max_retries = max_retries
|
|
137
|
+
self.base_url = base_url
|
|
138
|
+
# Local models can be slow (long prompts, thinking, a busy GPU): give them more time.
|
|
139
|
+
if timeout is None:
|
|
140
|
+
timeout = 1200.0 if spec.provider == "ollama" else 180.0
|
|
141
|
+
self.client = OpenAI(base_url=base_url, api_key=api_key, timeout=timeout, max_retries=0)
|
|
142
|
+
|
|
143
|
+
def preflight(self) -> None:
|
|
144
|
+
"""For Ollama, fail fast if the server is down or the model is not pulled (free, no generation)."""
|
|
145
|
+
if self.spec.provider != "ollama":
|
|
146
|
+
return
|
|
147
|
+
import openai
|
|
148
|
+
|
|
149
|
+
try:
|
|
150
|
+
names = {m.id for m in self.client.models.list()}
|
|
151
|
+
except openai.APIConnectionError as e:
|
|
152
|
+
raise ProviderUnreachable(
|
|
153
|
+
f"Cannot reach Ollama at {self.base_url}. Start it with `ollama serve` or set OLLAMA_HOST."
|
|
154
|
+
) from e
|
|
155
|
+
except openai.APIError:
|
|
156
|
+
return # an older Ollama without /v1/models: let the first call tell
|
|
157
|
+
wanted = self.spec.model
|
|
158
|
+
if wanted not in names and f"{wanted}:latest" not in names:
|
|
159
|
+
raise ModelNotFound(f"Model '{wanted}' is not available in Ollama. Run `ollama pull {wanted}` first.")
|
|
160
|
+
|
|
161
|
+
def estimate_cost(self, prompt_tokens: int, max_out: int) -> float:
|
|
162
|
+
pin, pout = self.spec.price()
|
|
163
|
+
return (prompt_tokens * pin + max_out * pout) / 1_000_000
|
|
164
|
+
|
|
165
|
+
def chat(
|
|
166
|
+
self,
|
|
167
|
+
messages: list[dict],
|
|
168
|
+
*,
|
|
169
|
+
tools: list[dict] | None = None,
|
|
170
|
+
max_tokens: int = 1024,
|
|
171
|
+
temperature: float = 0.0,
|
|
172
|
+
est_prompt_tokens: int = 0,
|
|
173
|
+
json_mode: bool = False,
|
|
174
|
+
) -> Completion:
|
|
175
|
+
import openai
|
|
176
|
+
|
|
177
|
+
self.budget.check(self.estimate_cost(est_prompt_tokens, max_tokens))
|
|
178
|
+
kwargs: dict = dict(
|
|
179
|
+
model=self.spec.model,
|
|
180
|
+
messages=messages,
|
|
181
|
+
max_tokens=max_tokens,
|
|
182
|
+
temperature=temperature,
|
|
183
|
+
)
|
|
184
|
+
if tools:
|
|
185
|
+
kwargs["tools"] = tools
|
|
186
|
+
kwargs["tool_choice"] = "auto"
|
|
187
|
+
if json_mode:
|
|
188
|
+
kwargs["response_format"] = {"type": "json_object"}
|
|
189
|
+
|
|
190
|
+
delay = 2.0
|
|
191
|
+
last_err: Exception | None = None
|
|
192
|
+
for attempt in range(self.max_retries + 1):
|
|
193
|
+
try:
|
|
194
|
+
resp = self.client.chat.completions.create(**kwargs)
|
|
195
|
+
break
|
|
196
|
+
except openai.AuthenticationError as e:
|
|
197
|
+
raise MissingApiKey(
|
|
198
|
+
f"{self.spec.provider} rejected the API key (401). Check {PROVIDERS[self.spec.provider][1]}."
|
|
199
|
+
) from e
|
|
200
|
+
except openai.NotFoundError as e:
|
|
201
|
+
hint = f" Run `ollama pull {self.spec.model}` first." if self.spec.provider == "ollama" else ""
|
|
202
|
+
raise ModelNotFound(f"Model '{self.spec.model}' not found at {self.spec.provider}.{hint}") from e
|
|
203
|
+
except openai.APIConnectionError as e:
|
|
204
|
+
last_err = e
|
|
205
|
+
if self.spec.provider == "ollama":
|
|
206
|
+
raise ProviderUnreachable(
|
|
207
|
+
f"Cannot reach Ollama at {self.base_url}. Start it with `ollama serve` or set OLLAMA_HOST."
|
|
208
|
+
) from e
|
|
209
|
+
if attempt >= self.max_retries:
|
|
210
|
+
raise ProviderUnreachable(f"Cannot reach {self.spec.provider} at {self.base_url}.") from e
|
|
211
|
+
except openai.PermissionDeniedError as e:
|
|
212
|
+
raise ToolbloatError(f"{self.spec.provider} refused the request (403): {_short(e)}") from e
|
|
213
|
+
except openai.BadRequestError as e:
|
|
214
|
+
msg = _short(e)
|
|
215
|
+
if "model" in msg.lower() and ("not" in msg.lower() or "invalid" in msg.lower()):
|
|
216
|
+
raise ModelNotFound(f"Model '{self.spec.model}' rejected by {self.spec.provider}: {msg}") from e
|
|
217
|
+
raise ToolbloatError(f"{self.spec.id} rejected the request: {msg}") from e
|
|
218
|
+
except (openai.RateLimitError, openai.InternalServerError, openai.APITimeoutError) as e:
|
|
219
|
+
last_err = e
|
|
220
|
+
if isinstance(e, openai.RateLimitError) and _is_quota(e):
|
|
221
|
+
raise BudgetExceeded(
|
|
222
|
+
f"{self.spec.provider} says the account has no credit left: {_short(e)}"
|
|
223
|
+
) from e
|
|
224
|
+
if attempt >= self.max_retries:
|
|
225
|
+
raise ProviderUnreachable(
|
|
226
|
+
f"{self.spec.id} kept failing after {self.max_retries} retries: {_short(e)}"
|
|
227
|
+
) from e
|
|
228
|
+
wait = delay + random.uniform(0, delay / 2)
|
|
229
|
+
headers = getattr(getattr(last_err, "response", None), "headers", None) or {}
|
|
230
|
+
retry_after = headers.get("retry-after")
|
|
231
|
+
if retry_after:
|
|
232
|
+
try:
|
|
233
|
+
wait = max(wait, min(float(retry_after), 120))
|
|
234
|
+
except ValueError:
|
|
235
|
+
pass
|
|
236
|
+
time.sleep(wait)
|
|
237
|
+
delay = min(delay * 2, 60)
|
|
238
|
+
|
|
239
|
+
usage = resp.usage
|
|
240
|
+
pt = getattr(usage, "prompt_tokens", 0) or 0
|
|
241
|
+
ct = getattr(usage, "completion_tokens", 0) or 0
|
|
242
|
+
pin, pout = self.spec.price()
|
|
243
|
+
cost = (pt * pin + ct * pout) / 1_000_000
|
|
244
|
+
self.budget.add(self.spec.provider, cost)
|
|
245
|
+
|
|
246
|
+
msg = resp.choices[0].message
|
|
247
|
+
tool_name = arguments = None
|
|
248
|
+
if msg.tool_calls:
|
|
249
|
+
fn = msg.tool_calls[0].function
|
|
250
|
+
tool_name, arguments = fn.name, fn.arguments
|
|
251
|
+
return Completion(tool_name, arguments, msg.content or "", pt, ct, cost)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _short(e: Exception) -> str:
|
|
255
|
+
text = str(e)
|
|
256
|
+
return text if len(text) < 300 else text[:300] + "..."
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _is_quota(e: Exception) -> bool:
|
|
260
|
+
text = str(e).lower()
|
|
261
|
+
return any(w in text for w in ("insufficient", "quota", "credit", "spending limit", "payment"))
|
toolbloat/metrics.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"""Turn a results file into the numbers the report shows."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections import Counter, defaultdict
|
|
6
|
+
from statistics import mean
|
|
7
|
+
|
|
8
|
+
NONE = "(no call)"
|
|
9
|
+
UNKNOWN = "(unknown tool)"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _server_of(chosen: str | None) -> str:
|
|
13
|
+
if chosen is None:
|
|
14
|
+
return NONE
|
|
15
|
+
if chosen.startswith("?:"):
|
|
16
|
+
return UNKNOWN
|
|
17
|
+
return chosen.split("/", 1)[0]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _rate(num: int, den: int) -> float | None:
|
|
21
|
+
return num / den if den else None
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def compute(results: dict) -> dict:
|
|
25
|
+
servers = [s for s in results["servers"] if not s.get("error")]
|
|
26
|
+
tools = [(s["name"], t) for s in servers for t in s["tools"]]
|
|
27
|
+
tool_tokens = {f"{srv}/{t['name']}": t["tokens"] for srv, t in tools}
|
|
28
|
+
total_tokens = sum(tool_tokens.values())
|
|
29
|
+
intents = {i["id"]: i for i in results["intents"]}
|
|
30
|
+
calls = [c for c in results["calls"] if c["intent_id"] in intents]
|
|
31
|
+
turns = results.get("settings", {}).get("turns_per_day", 100)
|
|
32
|
+
days = 30
|
|
33
|
+
|
|
34
|
+
# Per model, per condition
|
|
35
|
+
models = []
|
|
36
|
+
by_model = defaultdict(list)
|
|
37
|
+
for c in calls:
|
|
38
|
+
by_model[c["model"]].append(c)
|
|
39
|
+
prices = {m["id"]: (m.get("price_in", 0.0), m.get("price_out", 0.0)) for m in results["models"]}
|
|
40
|
+
for m in results["models"]:
|
|
41
|
+
mid = m["id"]
|
|
42
|
+
cs_all = by_model.get(mid, [])
|
|
43
|
+
# Compare like with like: keep only requests answered in both conditions.
|
|
44
|
+
answered = defaultdict(set)
|
|
45
|
+
for c in cs_all:
|
|
46
|
+
if not c.get("error"):
|
|
47
|
+
answered[c["intent_id"]].add(c["condition"])
|
|
48
|
+
paired = {i for i, conds in answered.items() if conds >= {"solo", "stack"}}
|
|
49
|
+
cs = [c for c in cs_all if c["intent_id"] in paired or c.get("error")]
|
|
50
|
+
row = {
|
|
51
|
+
"id": mid,
|
|
52
|
+
"price_in": prices[mid][0],
|
|
53
|
+
"calls": len(cs_all),
|
|
54
|
+
"paired": len(paired),
|
|
55
|
+
"cost": sum(c["cost"] for c in cs_all),
|
|
56
|
+
}
|
|
57
|
+
for cond in ("solo", "stack"):
|
|
58
|
+
cc = [c for c in cs if c["condition"] == cond and not c.get("error")]
|
|
59
|
+
row[f"{cond}_n"] = len(cc)
|
|
60
|
+
row[f"{cond}_acc"] = _rate(sum(c["correct"] for c in cc), len(cc))
|
|
61
|
+
row[f"{cond}_errors"] = sum(1 for c in cs if c["condition"] == cond and c.get("error"))
|
|
62
|
+
row[f"{cond}_prompt_tokens"] = mean([c["prompt_tokens"] for c in cc]) if cc else None
|
|
63
|
+
row[f"{cond}_no_call"] = _rate(sum(c["chosen"] is None for c in cc), len(cc))
|
|
64
|
+
tc = [c for c in cc if c["args_valid"] is not None and c["correct"]]
|
|
65
|
+
row[f"{cond}_args_valid"] = _rate(sum(c["args_valid"] for c in tc), len(tc))
|
|
66
|
+
st = [c for c in cs if c["condition"] == "stack" and not c.get("error")]
|
|
67
|
+
row["stack_server_acc"] = _rate(
|
|
68
|
+
sum(_server_of(c["chosen"]) == intents[c["intent_id"]]["tool"].split("/")[0] for c in st), len(st)
|
|
69
|
+
)
|
|
70
|
+
row["delta"] = (
|
|
71
|
+
row["stack_acc"] - row["solo_acc"] if row["stack_acc"] is not None and row["solo_acc"] is not None else None
|
|
72
|
+
)
|
|
73
|
+
# Intents the model got right alone but wrong once every server is loaded.
|
|
74
|
+
solo_ok = {c["intent_id"] for c in cs if c["condition"] == "solo" and c["correct"]}
|
|
75
|
+
row["broken_by_stack"] = sum(1 for c in st if c["intent_id"] in solo_ok and not c["correct"])
|
|
76
|
+
row["cross_server_errors"] = sum(
|
|
77
|
+
1
|
|
78
|
+
for c in st
|
|
79
|
+
if not c["correct"]
|
|
80
|
+
and _server_of(c["chosen"]) not in (NONE, UNKNOWN, intents[c["intent_id"]]["tool"].split("/")[0])
|
|
81
|
+
)
|
|
82
|
+
# A provider that bills fewer prompt tokens than the definitions alone
|
|
83
|
+
# probably truncated the context (Ollama's default window is small).
|
|
84
|
+
spt = row["stack_prompt_tokens"]
|
|
85
|
+
row["truncated"] = bool(spt is not None and total_tokens and spt < 0.7 * total_tokens)
|
|
86
|
+
monthly = total_tokens * turns * days
|
|
87
|
+
row["monthly_tokens"] = monthly
|
|
88
|
+
row["monthly_usd"] = monthly * prices[mid][0] / 1_000_000
|
|
89
|
+
models.append(row)
|
|
90
|
+
|
|
91
|
+
# Cross-server confusion matrix, stack condition, all models
|
|
92
|
+
names = [s["name"] for s in servers]
|
|
93
|
+
cols = names + [NONE, UNKNOWN]
|
|
94
|
+
matrix = {r: Counter() for r in names}
|
|
95
|
+
per_model_matrix = {}
|
|
96
|
+
for c in calls:
|
|
97
|
+
if c["condition"] != "stack" or c.get("error"):
|
|
98
|
+
continue
|
|
99
|
+
exp = intents[c["intent_id"]]["tool"].split("/")[0]
|
|
100
|
+
if exp not in matrix:
|
|
101
|
+
continue
|
|
102
|
+
got = _server_of(c["chosen"])
|
|
103
|
+
matrix[exp][got] += 1
|
|
104
|
+
per_model_matrix.setdefault(c["model"], {r: Counter() for r in names})[exp][got] += 1
|
|
105
|
+
used_cols = [col for col in cols if col in names or any(matrix[r][col] for r in names)]
|
|
106
|
+
|
|
107
|
+
# Wrong picks in the stack condition, grouped by (expected tool, chosen tool)
|
|
108
|
+
pairs = Counter()
|
|
109
|
+
pair_models = defaultdict(set)
|
|
110
|
+
for c in calls:
|
|
111
|
+
if c["condition"] == "stack" and not c["correct"] and not c.get("error"):
|
|
112
|
+
exp = intents[c["intent_id"]]["tool"]
|
|
113
|
+
got = c["chosen"] or NONE
|
|
114
|
+
pairs[(exp, got)] += 1
|
|
115
|
+
pair_models[(exp, got)].add(c["model"])
|
|
116
|
+
confusions = [
|
|
117
|
+
{
|
|
118
|
+
"expected": e,
|
|
119
|
+
"chosen": g,
|
|
120
|
+
"count": n,
|
|
121
|
+
"models": sorted(pair_models[(e, g)]),
|
|
122
|
+
"cross_server": _server_of(g) not in (NONE, UNKNOWN, e.split("/")[0]),
|
|
123
|
+
}
|
|
124
|
+
for (e, g), n in pairs.most_common()
|
|
125
|
+
]
|
|
126
|
+
|
|
127
|
+
# Tools never chosen by any model in the stack condition
|
|
128
|
+
chosen_stack = {c["chosen"] for c in calls if c["condition"] == "stack" and c["chosen"]}
|
|
129
|
+
stack_calls = sum(1 for c in calls if c["condition"] == "stack")
|
|
130
|
+
tested = {intents[c["intent_id"]]["tool"] for c in calls if c["condition"] == "stack"}
|
|
131
|
+
untested = sorted(k for k in tool_tokens if k not in tested)
|
|
132
|
+
never = (
|
|
133
|
+
[
|
|
134
|
+
{"tool": k, "tokens": v}
|
|
135
|
+
for k, v in sorted(tool_tokens.items(), key=lambda kv: -kv[1])
|
|
136
|
+
if k not in chosen_stack and k in tested
|
|
137
|
+
]
|
|
138
|
+
if stack_calls
|
|
139
|
+
else []
|
|
140
|
+
)
|
|
141
|
+
never_tokens = sum(n["tokens"] for n in never)
|
|
142
|
+
|
|
143
|
+
per_server = sorted(
|
|
144
|
+
(
|
|
145
|
+
{
|
|
146
|
+
"name": s["name"],
|
|
147
|
+
"command": s["command"],
|
|
148
|
+
"tools": len(s["tools"]),
|
|
149
|
+
"tokens": sum(t["tokens"] for t in s["tools"]),
|
|
150
|
+
"share": (sum(t["tokens"] for t in s["tools"]) / total_tokens) if total_tokens else 0,
|
|
151
|
+
}
|
|
152
|
+
for s in servers
|
|
153
|
+
),
|
|
154
|
+
key=lambda r: -r["tokens"],
|
|
155
|
+
)
|
|
156
|
+
heaviest_tools = sorted(({"tool": k, "tokens": v} for k, v in tool_tokens.items()), key=lambda r: -r["tokens"])
|
|
157
|
+
|
|
158
|
+
# Per-intent outcome table (compact): for each intent, per model solo/stack result
|
|
159
|
+
outcome = defaultdict(dict)
|
|
160
|
+
for c in calls:
|
|
161
|
+
outcome[c["intent_id"]][(c["model"], c["condition"])] = c
|
|
162
|
+
intent_rows = []
|
|
163
|
+
for iid, i in intents.items():
|
|
164
|
+
if i.get("skip"):
|
|
165
|
+
continue
|
|
166
|
+
cells = {}
|
|
167
|
+
for m in results["models"]:
|
|
168
|
+
for cond in ("solo", "stack"):
|
|
169
|
+
c = outcome[iid].get((m["id"], cond))
|
|
170
|
+
cells[f"{m['id']}|{cond}"] = (
|
|
171
|
+
None if c is None else {"ok": c["correct"], "chosen": c["chosen"], "error": bool(c.get("error"))}
|
|
172
|
+
)
|
|
173
|
+
intent_rows.append({"id": iid, "tool": i["tool"], "text": i["text"], "cells": cells})
|
|
174
|
+
|
|
175
|
+
return {
|
|
176
|
+
"total_tokens": total_tokens,
|
|
177
|
+
"n_tools": len(tools),
|
|
178
|
+
"n_servers": len(servers),
|
|
179
|
+
"failed_servers": [s for s in results["servers"] if s.get("error")],
|
|
180
|
+
"n_intents": sum(1 for i in intents.values() if not i.get("skip")),
|
|
181
|
+
"per_server": per_server,
|
|
182
|
+
"heaviest_tools": heaviest_tools,
|
|
183
|
+
"models": models,
|
|
184
|
+
"matrix": {"rows": names, "cols": used_cols, "cells": {r: dict(matrix[r]) for r in names}},
|
|
185
|
+
"per_model_matrix": {m: {r: dict(v) for r, v in mm.items()} for m, mm in per_model_matrix.items()},
|
|
186
|
+
"confusions": confusions,
|
|
187
|
+
"never_chosen": never,
|
|
188
|
+
"never_tokens": never_tokens,
|
|
189
|
+
"untested": untested if stack_calls else [],
|
|
190
|
+
"never_monthly_tokens": never_tokens * turns * days,
|
|
191
|
+
"turns_per_day": turns,
|
|
192
|
+
"monthly_tokens": total_tokens * turns * days,
|
|
193
|
+
"intent_rows": intent_rows,
|
|
194
|
+
"spend": results.get("spend", {}),
|
|
195
|
+
# Cost of every call in this report, including ones reused from the cache.
|
|
196
|
+
"total_spend": sum(c["cost"] for c in calls),
|
|
197
|
+
}
|
toolbloat/py.typed
ADDED
|
File without changes
|
toolbloat/report.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Static HTML report (single file, no external scripts)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from importlib import resources
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from jinja2 import Environment, select_autoescape
|
|
9
|
+
from markupsafe import Markup, escape
|
|
10
|
+
|
|
11
|
+
from .metrics import NONE, UNKNOWN, compute
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _env() -> Environment:
|
|
15
|
+
env = Environment(autoescape=select_autoescape(["html", "j2"]), trim_blocks=True, lstrip_blocks=True)
|
|
16
|
+
env.filters["wbr"] = lambda x: Markup("/<wbr>").join(escape(p) for p in str(x).split("/"))
|
|
17
|
+
# Numbers are rendered in English and re-formatted for the active language in the browser
|
|
18
|
+
# (data-n holds the raw value, data-f the format), so ES shows 6.064 and EN 6,064.
|
|
19
|
+
env.filters["pct"] = lambda x: _n(x, "pct", None if x is None else f"{x * 100:.0f}%")
|
|
20
|
+
env.filters["num"] = lambda x: _n(x, "int", None if x is None else f"{x:,.0f}")
|
|
21
|
+
env.filters["big"] = lambda x: _n(x, "big", _big(x))
|
|
22
|
+
env.filters["usd"] = lambda x: _n(x, "usd", f"${x:,.2f}" if x >= 0.01 or x == 0 else f"${x:,.4f}")
|
|
23
|
+
env.filters["pts"] = lambda x: _n(x, "pts", None if x is None else f"{x * 100:+.0f}")
|
|
24
|
+
return env
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _n(value, fmt: str, text: str | None) -> Markup:
|
|
28
|
+
if value is None or text is None:
|
|
29
|
+
return Markup("–")
|
|
30
|
+
return Markup('<span data-n="{}" data-f="{}">{}</span>').format(value, fmt, text)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _big(n: float) -> str:
|
|
34
|
+
for div, suf in ((1e9, "B"), (1e6, "M"), (1e3, "k")):
|
|
35
|
+
if n >= div:
|
|
36
|
+
return f"{n / div:.1f}{suf}"
|
|
37
|
+
return f"{n:.0f}"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def render(results: dict) -> str:
|
|
41
|
+
m = compute(results)
|
|
42
|
+
tpl = resources.files("toolbloat").joinpath("templates/report.html.j2").read_text(encoding="utf-8")
|
|
43
|
+
worst = None
|
|
44
|
+
rated = [r for r in m["models"] if r["delta"] is not None]
|
|
45
|
+
if rated:
|
|
46
|
+
worst = min(rated, key=lambda r: r["delta"])
|
|
47
|
+
max_cell = max([v for row in m["matrix"]["cells"].values() for v in row.values()] or [1])
|
|
48
|
+
return (
|
|
49
|
+
_env()
|
|
50
|
+
.from_string(tpl)
|
|
51
|
+
.render(
|
|
52
|
+
r=results,
|
|
53
|
+
m=m,
|
|
54
|
+
worst=worst,
|
|
55
|
+
max_cell=max_cell,
|
|
56
|
+
NONE=NONE,
|
|
57
|
+
UNKNOWN=UNKNOWN,
|
|
58
|
+
max_server_tokens=max([s["tokens"] for s in m["per_server"]] or [1]),
|
|
59
|
+
recorded=results.get("recorded", False),
|
|
60
|
+
)
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def write_report(results: dict, path: Path) -> Path:
|
|
65
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
66
|
+
path.write_text(render(results), encoding="utf-8")
|
|
67
|
+
return path
|