toolbloat 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
toolbloat/llm.py ADDED
@@ -0,0 +1,261 @@
1
+ """One OpenAI-compatible client with provider presets, a budget and backoff."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import random
7
+ import threading
8
+ import time
9
+ from dataclasses import dataclass, field
10
+
11
+ from .errors import (
12
+ BudgetExceeded,
13
+ MissingApiKey,
14
+ ModelNotFound,
15
+ ProviderUnreachable,
16
+ ToolbloatError,
17
+ )
18
+
19
+ # base_url, env var holding the key (None = no key needed)
20
+ PROVIDERS: dict[str, tuple[str, str | None]] = {
21
+ "openrouter": ("https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"),
22
+ "openai": ("https://api.openai.com/v1", "OPENAI_API_KEY"),
23
+ "mistral": ("https://api.mistral.ai/v1", "MISTRAL_API_KEY"),
24
+ "ollama": ("http://localhost:11434/v1", None),
25
+ }
26
+
27
+ # USD per million tokens (input, output). Used for --max-cost and the monthly
28
+ # projection. Unknown models fall back to --price-in/--price-out or 0 with a warning.
29
+ # Mistral list prices from https://docs.mistral.ai/inference/pricing (checked 2026-11-03):
30
+ # mistral-small-latest = Mistral Small 4, ministral-8b-latest = Ministral 3 8B.
31
+ # Mistral Medium 3.5 is not listed there; we budget it at the Mistral Large 4 price
32
+ # as a conservative upper bound.
33
+ PRICES: dict[str, tuple[float, float]] = {
34
+ "mistral:mistral-small-latest": (0.15, 0.60),
35
+ "mistral:ministral-8b-latest": (0.15, 0.15),
36
+ "mistral:ministral-14b-latest": (0.20, 0.20),
37
+ "mistral:ministral-3b-latest": (0.10, 0.10),
38
+ "mistral:codestral-latest": (0.30, 0.90),
39
+ "mistral:mistral-medium-latest": (0.70, 2.10),
40
+ }
41
+
42
+
43
+ def ollama_base_url() -> str:
44
+ host = os.environ.get("OLLAMA_HOST", "").strip()
45
+ if not host:
46
+ return PROVIDERS["ollama"][0]
47
+ if not host.startswith("http"):
48
+ host = "http://" + host
49
+ host = host.rstrip("/")
50
+ return host if host.endswith("/v1") else host + "/v1"
51
+
52
+
53
+ @dataclass(frozen=True)
54
+ class ModelSpec:
55
+ provider: str
56
+ model: str
57
+
58
+ @property
59
+ def id(self) -> str:
60
+ return f"{self.provider}:{self.model}"
61
+
62
+ @classmethod
63
+ def parse(cls, text: str) -> ModelSpec:
64
+ provider, sep, model = text.partition(":")
65
+ if not sep or not model:
66
+ raise ToolbloatError(
67
+ f"Model '{text}' must look like provider:model, e.g. ollama:qwen3:4b "
68
+ f"or mistral:mistral-small-latest. Providers: {', '.join(PROVIDERS)}."
69
+ )
70
+ if provider not in PROVIDERS:
71
+ raise ToolbloatError(f"Unknown provider '{provider}'. Use one of: {', '.join(PROVIDERS)}.")
72
+ return cls(provider, model)
73
+
74
+ def price(self) -> tuple[float, float]:
75
+ if self.provider == "ollama":
76
+ return (0.0, 0.0)
77
+ return PRICES.get(self.id, (0.0, 0.0))
78
+
79
+ def has_known_price(self) -> bool:
80
+ return self.provider == "ollama" or self.id in PRICES
81
+
82
+
83
+ @dataclass
84
+ class Budget:
85
+ """Tracks spend and refuses a call once the cap would be passed."""
86
+
87
+ max_cost: float
88
+ spent: dict[str, float] = field(default_factory=dict)
89
+ _lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
90
+
91
+ @property
92
+ def total(self) -> float:
93
+ return sum(self.spent.values())
94
+
95
+ def check(self, estimate: float = 0.0) -> None:
96
+ if self.total + estimate > self.max_cost:
97
+ raise BudgetExceeded(
98
+ f"Budget reached: spent ${self.total:.4f} of --max-cost ${self.max_cost:.2f}"
99
+ f" (next call estimated at ${estimate:.4f}). Partial results were kept;"
100
+ " raise --max-cost to continue."
101
+ )
102
+
103
+ def add(self, provider: str, usd: float) -> None:
104
+ with self._lock:
105
+ self.spent[provider] = self.spent.get(provider, 0.0) + usd
106
+
107
+
108
+ @dataclass
109
+ class Completion:
110
+ tool_name: str | None
111
+ arguments: str | None
112
+ text: str
113
+ prompt_tokens: int
114
+ completion_tokens: int
115
+ cost: float
116
+
117
+
118
+ class LLM:
119
+ """Thin wrapper around openai.OpenAI for one model."""
120
+
121
+ def __init__(self, spec: ModelSpec, budget: Budget, *, max_retries: int = 9, timeout: float | None = None):
122
+ from openai import OpenAI
123
+
124
+ base_url, key_var = PROVIDERS[spec.provider]
125
+ if spec.provider == "ollama":
126
+ base_url = ollama_base_url()
127
+ api_key = "ollama"
128
+ else:
129
+ api_key = os.environ.get(key_var or "", "").strip()
130
+ if not api_key:
131
+ raise MissingApiKey(
132
+ f"{key_var} is not set. Export it to use {spec.id}, or pick a local model such as ollama:qwen3:4b."
133
+ )
134
+ self.spec = spec
135
+ self.budget = budget
136
+ self.max_retries = max_retries
137
+ self.base_url = base_url
138
+ # Local models can be slow (long prompts, thinking, a busy GPU): give them more time.
139
+ if timeout is None:
140
+ timeout = 1200.0 if spec.provider == "ollama" else 180.0
141
+ self.client = OpenAI(base_url=base_url, api_key=api_key, timeout=timeout, max_retries=0)
142
+
143
+ def preflight(self) -> None:
144
+ """For Ollama, fail fast if the server is down or the model is not pulled (free, no generation)."""
145
+ if self.spec.provider != "ollama":
146
+ return
147
+ import openai
148
+
149
+ try:
150
+ names = {m.id for m in self.client.models.list()}
151
+ except openai.APIConnectionError as e:
152
+ raise ProviderUnreachable(
153
+ f"Cannot reach Ollama at {self.base_url}. Start it with `ollama serve` or set OLLAMA_HOST."
154
+ ) from e
155
+ except openai.APIError:
156
+ return # an older Ollama without /v1/models: let the first call tell
157
+ wanted = self.spec.model
158
+ if wanted not in names and f"{wanted}:latest" not in names:
159
+ raise ModelNotFound(f"Model '{wanted}' is not available in Ollama. Run `ollama pull {wanted}` first.")
160
+
161
+ def estimate_cost(self, prompt_tokens: int, max_out: int) -> float:
162
+ pin, pout = self.spec.price()
163
+ return (prompt_tokens * pin + max_out * pout) / 1_000_000
164
+
165
+ def chat(
166
+ self,
167
+ messages: list[dict],
168
+ *,
169
+ tools: list[dict] | None = None,
170
+ max_tokens: int = 1024,
171
+ temperature: float = 0.0,
172
+ est_prompt_tokens: int = 0,
173
+ json_mode: bool = False,
174
+ ) -> Completion:
175
+ import openai
176
+
177
+ self.budget.check(self.estimate_cost(est_prompt_tokens, max_tokens))
178
+ kwargs: dict = dict(
179
+ model=self.spec.model,
180
+ messages=messages,
181
+ max_tokens=max_tokens,
182
+ temperature=temperature,
183
+ )
184
+ if tools:
185
+ kwargs["tools"] = tools
186
+ kwargs["tool_choice"] = "auto"
187
+ if json_mode:
188
+ kwargs["response_format"] = {"type": "json_object"}
189
+
190
+ delay = 2.0
191
+ last_err: Exception | None = None
192
+ for attempt in range(self.max_retries + 1):
193
+ try:
194
+ resp = self.client.chat.completions.create(**kwargs)
195
+ break
196
+ except openai.AuthenticationError as e:
197
+ raise MissingApiKey(
198
+ f"{self.spec.provider} rejected the API key (401). Check {PROVIDERS[self.spec.provider][1]}."
199
+ ) from e
200
+ except openai.NotFoundError as e:
201
+ hint = f" Run `ollama pull {self.spec.model}` first." if self.spec.provider == "ollama" else ""
202
+ raise ModelNotFound(f"Model '{self.spec.model}' not found at {self.spec.provider}.{hint}") from e
203
+ except openai.APIConnectionError as e:
204
+ last_err = e
205
+ if self.spec.provider == "ollama":
206
+ raise ProviderUnreachable(
207
+ f"Cannot reach Ollama at {self.base_url}. Start it with `ollama serve` or set OLLAMA_HOST."
208
+ ) from e
209
+ if attempt >= self.max_retries:
210
+ raise ProviderUnreachable(f"Cannot reach {self.spec.provider} at {self.base_url}.") from e
211
+ except openai.PermissionDeniedError as e:
212
+ raise ToolbloatError(f"{self.spec.provider} refused the request (403): {_short(e)}") from e
213
+ except openai.BadRequestError as e:
214
+ msg = _short(e)
215
+ if "model" in msg.lower() and ("not" in msg.lower() or "invalid" in msg.lower()):
216
+ raise ModelNotFound(f"Model '{self.spec.model}' rejected by {self.spec.provider}: {msg}") from e
217
+ raise ToolbloatError(f"{self.spec.id} rejected the request: {msg}") from e
218
+ except (openai.RateLimitError, openai.InternalServerError, openai.APITimeoutError) as e:
219
+ last_err = e
220
+ if isinstance(e, openai.RateLimitError) and _is_quota(e):
221
+ raise BudgetExceeded(
222
+ f"{self.spec.provider} says the account has no credit left: {_short(e)}"
223
+ ) from e
224
+ if attempt >= self.max_retries:
225
+ raise ProviderUnreachable(
226
+ f"{self.spec.id} kept failing after {self.max_retries} retries: {_short(e)}"
227
+ ) from e
228
+ wait = delay + random.uniform(0, delay / 2)
229
+ headers = getattr(getattr(last_err, "response", None), "headers", None) or {}
230
+ retry_after = headers.get("retry-after")
231
+ if retry_after:
232
+ try:
233
+ wait = max(wait, min(float(retry_after), 120))
234
+ except ValueError:
235
+ pass
236
+ time.sleep(wait)
237
+ delay = min(delay * 2, 60)
238
+
239
+ usage = resp.usage
240
+ pt = getattr(usage, "prompt_tokens", 0) or 0
241
+ ct = getattr(usage, "completion_tokens", 0) or 0
242
+ pin, pout = self.spec.price()
243
+ cost = (pt * pin + ct * pout) / 1_000_000
244
+ self.budget.add(self.spec.provider, cost)
245
+
246
+ msg = resp.choices[0].message
247
+ tool_name = arguments = None
248
+ if msg.tool_calls:
249
+ fn = msg.tool_calls[0].function
250
+ tool_name, arguments = fn.name, fn.arguments
251
+ return Completion(tool_name, arguments, msg.content or "", pt, ct, cost)
252
+
253
+
254
+ def _short(e: Exception) -> str:
255
+ text = str(e)
256
+ return text if len(text) < 300 else text[:300] + "..."
257
+
258
+
259
+ def _is_quota(e: Exception) -> bool:
260
+ text = str(e).lower()
261
+ return any(w in text for w in ("insufficient", "quota", "credit", "spending limit", "payment"))
toolbloat/metrics.py ADDED
@@ -0,0 +1,197 @@
1
+ """Turn a results file into the numbers the report shows."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections import Counter, defaultdict
6
+ from statistics import mean
7
+
8
+ NONE = "(no call)"
9
+ UNKNOWN = "(unknown tool)"
10
+
11
+
12
+ def _server_of(chosen: str | None) -> str:
13
+ if chosen is None:
14
+ return NONE
15
+ if chosen.startswith("?:"):
16
+ return UNKNOWN
17
+ return chosen.split("/", 1)[0]
18
+
19
+
20
+ def _rate(num: int, den: int) -> float | None:
21
+ return num / den if den else None
22
+
23
+
24
+ def compute(results: dict) -> dict:
25
+ servers = [s for s in results["servers"] if not s.get("error")]
26
+ tools = [(s["name"], t) for s in servers for t in s["tools"]]
27
+ tool_tokens = {f"{srv}/{t['name']}": t["tokens"] for srv, t in tools}
28
+ total_tokens = sum(tool_tokens.values())
29
+ intents = {i["id"]: i for i in results["intents"]}
30
+ calls = [c for c in results["calls"] if c["intent_id"] in intents]
31
+ turns = results.get("settings", {}).get("turns_per_day", 100)
32
+ days = 30
33
+
34
+ # Per model, per condition
35
+ models = []
36
+ by_model = defaultdict(list)
37
+ for c in calls:
38
+ by_model[c["model"]].append(c)
39
+ prices = {m["id"]: (m.get("price_in", 0.0), m.get("price_out", 0.0)) for m in results["models"]}
40
+ for m in results["models"]:
41
+ mid = m["id"]
42
+ cs_all = by_model.get(mid, [])
43
+ # Compare like with like: keep only requests answered in both conditions.
44
+ answered = defaultdict(set)
45
+ for c in cs_all:
46
+ if not c.get("error"):
47
+ answered[c["intent_id"]].add(c["condition"])
48
+ paired = {i for i, conds in answered.items() if conds >= {"solo", "stack"}}
49
+ cs = [c for c in cs_all if c["intent_id"] in paired or c.get("error")]
50
+ row = {
51
+ "id": mid,
52
+ "price_in": prices[mid][0],
53
+ "calls": len(cs_all),
54
+ "paired": len(paired),
55
+ "cost": sum(c["cost"] for c in cs_all),
56
+ }
57
+ for cond in ("solo", "stack"):
58
+ cc = [c for c in cs if c["condition"] == cond and not c.get("error")]
59
+ row[f"{cond}_n"] = len(cc)
60
+ row[f"{cond}_acc"] = _rate(sum(c["correct"] for c in cc), len(cc))
61
+ row[f"{cond}_errors"] = sum(1 for c in cs if c["condition"] == cond and c.get("error"))
62
+ row[f"{cond}_prompt_tokens"] = mean([c["prompt_tokens"] for c in cc]) if cc else None
63
+ row[f"{cond}_no_call"] = _rate(sum(c["chosen"] is None for c in cc), len(cc))
64
+ tc = [c for c in cc if c["args_valid"] is not None and c["correct"]]
65
+ row[f"{cond}_args_valid"] = _rate(sum(c["args_valid"] for c in tc), len(tc))
66
+ st = [c for c in cs if c["condition"] == "stack" and not c.get("error")]
67
+ row["stack_server_acc"] = _rate(
68
+ sum(_server_of(c["chosen"]) == intents[c["intent_id"]]["tool"].split("/")[0] for c in st), len(st)
69
+ )
70
+ row["delta"] = (
71
+ row["stack_acc"] - row["solo_acc"] if row["stack_acc"] is not None and row["solo_acc"] is not None else None
72
+ )
73
+ # Intents the model got right alone but wrong once every server is loaded.
74
+ solo_ok = {c["intent_id"] for c in cs if c["condition"] == "solo" and c["correct"]}
75
+ row["broken_by_stack"] = sum(1 for c in st if c["intent_id"] in solo_ok and not c["correct"])
76
+ row["cross_server_errors"] = sum(
77
+ 1
78
+ for c in st
79
+ if not c["correct"]
80
+ and _server_of(c["chosen"]) not in (NONE, UNKNOWN, intents[c["intent_id"]]["tool"].split("/")[0])
81
+ )
82
+ # A provider that bills fewer prompt tokens than the definitions alone
83
+ # probably truncated the context (Ollama's default window is small).
84
+ spt = row["stack_prompt_tokens"]
85
+ row["truncated"] = bool(spt is not None and total_tokens and spt < 0.7 * total_tokens)
86
+ monthly = total_tokens * turns * days
87
+ row["monthly_tokens"] = monthly
88
+ row["monthly_usd"] = monthly * prices[mid][0] / 1_000_000
89
+ models.append(row)
90
+
91
+ # Cross-server confusion matrix, stack condition, all models
92
+ names = [s["name"] for s in servers]
93
+ cols = names + [NONE, UNKNOWN]
94
+ matrix = {r: Counter() for r in names}
95
+ per_model_matrix = {}
96
+ for c in calls:
97
+ if c["condition"] != "stack" or c.get("error"):
98
+ continue
99
+ exp = intents[c["intent_id"]]["tool"].split("/")[0]
100
+ if exp not in matrix:
101
+ continue
102
+ got = _server_of(c["chosen"])
103
+ matrix[exp][got] += 1
104
+ per_model_matrix.setdefault(c["model"], {r: Counter() for r in names})[exp][got] += 1
105
+ used_cols = [col for col in cols if col in names or any(matrix[r][col] for r in names)]
106
+
107
+ # Wrong picks in the stack condition, grouped by (expected tool, chosen tool)
108
+ pairs = Counter()
109
+ pair_models = defaultdict(set)
110
+ for c in calls:
111
+ if c["condition"] == "stack" and not c["correct"] and not c.get("error"):
112
+ exp = intents[c["intent_id"]]["tool"]
113
+ got = c["chosen"] or NONE
114
+ pairs[(exp, got)] += 1
115
+ pair_models[(exp, got)].add(c["model"])
116
+ confusions = [
117
+ {
118
+ "expected": e,
119
+ "chosen": g,
120
+ "count": n,
121
+ "models": sorted(pair_models[(e, g)]),
122
+ "cross_server": _server_of(g) not in (NONE, UNKNOWN, e.split("/")[0]),
123
+ }
124
+ for (e, g), n in pairs.most_common()
125
+ ]
126
+
127
+ # Tools never chosen by any model in the stack condition
128
+ chosen_stack = {c["chosen"] for c in calls if c["condition"] == "stack" and c["chosen"]}
129
+ stack_calls = sum(1 for c in calls if c["condition"] == "stack")
130
+ tested = {intents[c["intent_id"]]["tool"] for c in calls if c["condition"] == "stack"}
131
+ untested = sorted(k for k in tool_tokens if k not in tested)
132
+ never = (
133
+ [
134
+ {"tool": k, "tokens": v}
135
+ for k, v in sorted(tool_tokens.items(), key=lambda kv: -kv[1])
136
+ if k not in chosen_stack and k in tested
137
+ ]
138
+ if stack_calls
139
+ else []
140
+ )
141
+ never_tokens = sum(n["tokens"] for n in never)
142
+
143
+ per_server = sorted(
144
+ (
145
+ {
146
+ "name": s["name"],
147
+ "command": s["command"],
148
+ "tools": len(s["tools"]),
149
+ "tokens": sum(t["tokens"] for t in s["tools"]),
150
+ "share": (sum(t["tokens"] for t in s["tools"]) / total_tokens) if total_tokens else 0,
151
+ }
152
+ for s in servers
153
+ ),
154
+ key=lambda r: -r["tokens"],
155
+ )
156
+ heaviest_tools = sorted(({"tool": k, "tokens": v} for k, v in tool_tokens.items()), key=lambda r: -r["tokens"])
157
+
158
+ # Per-intent outcome table (compact): for each intent, per model solo/stack result
159
+ outcome = defaultdict(dict)
160
+ for c in calls:
161
+ outcome[c["intent_id"]][(c["model"], c["condition"])] = c
162
+ intent_rows = []
163
+ for iid, i in intents.items():
164
+ if i.get("skip"):
165
+ continue
166
+ cells = {}
167
+ for m in results["models"]:
168
+ for cond in ("solo", "stack"):
169
+ c = outcome[iid].get((m["id"], cond))
170
+ cells[f"{m['id']}|{cond}"] = (
171
+ None if c is None else {"ok": c["correct"], "chosen": c["chosen"], "error": bool(c.get("error"))}
172
+ )
173
+ intent_rows.append({"id": iid, "tool": i["tool"], "text": i["text"], "cells": cells})
174
+
175
+ return {
176
+ "total_tokens": total_tokens,
177
+ "n_tools": len(tools),
178
+ "n_servers": len(servers),
179
+ "failed_servers": [s for s in results["servers"] if s.get("error")],
180
+ "n_intents": sum(1 for i in intents.values() if not i.get("skip")),
181
+ "per_server": per_server,
182
+ "heaviest_tools": heaviest_tools,
183
+ "models": models,
184
+ "matrix": {"rows": names, "cols": used_cols, "cells": {r: dict(matrix[r]) for r in names}},
185
+ "per_model_matrix": {m: {r: dict(v) for r, v in mm.items()} for m, mm in per_model_matrix.items()},
186
+ "confusions": confusions,
187
+ "never_chosen": never,
188
+ "never_tokens": never_tokens,
189
+ "untested": untested if stack_calls else [],
190
+ "never_monthly_tokens": never_tokens * turns * days,
191
+ "turns_per_day": turns,
192
+ "monthly_tokens": total_tokens * turns * days,
193
+ "intent_rows": intent_rows,
194
+ "spend": results.get("spend", {}),
195
+ # Cost of every call in this report, including ones reused from the cache.
196
+ "total_spend": sum(c["cost"] for c in calls),
197
+ }
toolbloat/py.typed ADDED
File without changes
toolbloat/report.py ADDED
@@ -0,0 +1,67 @@
1
+ """Static HTML report (single file, no external scripts)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from importlib import resources
6
+ from pathlib import Path
7
+
8
+ from jinja2 import Environment, select_autoescape
9
+ from markupsafe import Markup, escape
10
+
11
+ from .metrics import NONE, UNKNOWN, compute
12
+
13
+
14
+ def _env() -> Environment:
15
+ env = Environment(autoescape=select_autoescape(["html", "j2"]), trim_blocks=True, lstrip_blocks=True)
16
+ env.filters["wbr"] = lambda x: Markup("/<wbr>").join(escape(p) for p in str(x).split("/"))
17
+ # Numbers are rendered in English and re-formatted for the active language in the browser
18
+ # (data-n holds the raw value, data-f the format), so ES shows 6.064 and EN 6,064.
19
+ env.filters["pct"] = lambda x: _n(x, "pct", None if x is None else f"{x * 100:.0f}%")
20
+ env.filters["num"] = lambda x: _n(x, "int", None if x is None else f"{x:,.0f}")
21
+ env.filters["big"] = lambda x: _n(x, "big", _big(x))
22
+ env.filters["usd"] = lambda x: _n(x, "usd", f"${x:,.2f}" if x >= 0.01 or x == 0 else f"${x:,.4f}")
23
+ env.filters["pts"] = lambda x: _n(x, "pts", None if x is None else f"{x * 100:+.0f}")
24
+ return env
25
+
26
+
27
+ def _n(value, fmt: str, text: str | None) -> Markup:
28
+ if value is None or text is None:
29
+ return Markup("–")
30
+ return Markup('<span data-n="{}" data-f="{}">{}</span>').format(value, fmt, text)
31
+
32
+
33
+ def _big(n: float) -> str:
34
+ for div, suf in ((1e9, "B"), (1e6, "M"), (1e3, "k")):
35
+ if n >= div:
36
+ return f"{n / div:.1f}{suf}"
37
+ return f"{n:.0f}"
38
+
39
+
40
+ def render(results: dict) -> str:
41
+ m = compute(results)
42
+ tpl = resources.files("toolbloat").joinpath("templates/report.html.j2").read_text(encoding="utf-8")
43
+ worst = None
44
+ rated = [r for r in m["models"] if r["delta"] is not None]
45
+ if rated:
46
+ worst = min(rated, key=lambda r: r["delta"])
47
+ max_cell = max([v for row in m["matrix"]["cells"].values() for v in row.values()] or [1])
48
+ return (
49
+ _env()
50
+ .from_string(tpl)
51
+ .render(
52
+ r=results,
53
+ m=m,
54
+ worst=worst,
55
+ max_cell=max_cell,
56
+ NONE=NONE,
57
+ UNKNOWN=UNKNOWN,
58
+ max_server_tokens=max([s["tokens"] for s in m["per_server"]] or [1]),
59
+ recorded=results.get("recorded", False),
60
+ )
61
+ )
62
+
63
+
64
+ def write_report(results: dict, path: Path) -> Path:
65
+ path.parent.mkdir(parents=True, exist_ok=True)
66
+ path.write_text(render(results), encoding="utf-8")
67
+ return path