rulelab-cli 0.1.2__py3-none-any.whl

rulelab/engine.py

@@ -0,0 +1,272 @@
+"""Adapter between rulelab and the commitarena engine (tasks, no-network sandbox, agent loop)."""
+
+from __future__ import annotations
+
+import json
+import os
+import platform
+import shutil
+import subprocess
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+from .errors import RulelabError
+from .runner import RunOutcome
+
+# Token model used to estimate cost before anything has run. The whole conversation is resent on
+# every step, so input tokens grow roughly quadratically with the number of steps:
+#   input ~= 1,000 * steps + 467 * steps^2     output ~= 150 * steps
+# Fitted to two measured codestral runs on sqlparse (15 steps: ~120k input; 30 steps: 450,755).
+# It assumes the agent uses every step, so it tends to over-estimate.
+
+
+def estimate_tokens(steps: int) -> tuple[int, int]:
+    return 1_000 * steps + 467 * steps * steps, 150 * steps
+
+
+def _ca():
+    try:
+        import commitarena
+    except ImportError as e:  # pragma: no cover - dependency is declared
+        raise RulelabError(
+            "the commitarena engine is not installed",
+            "Reinstall rulelab: uvx --from git+https://github.com/diegoperezg7/rulelab rulelab",
+        ) from e
+    return commitarena
+
+
+def hardware() -> str:
+    mem = ""
+    try:
+        if platform.system() == "Darwin":
+            b = int(subprocess.run(["sysctl", "-n", "hw.memsize"], capture_output=True, text=True).stdout)
+            chip = subprocess.run(
+                ["sysctl", "-n", "machdep.cpu.brand_string"], capture_output=True, text=True
+            ).stdout.strip()
+            mem = f"{chip}, {b // 2**30} GB RAM"
+        elif Path("/proc/meminfo").exists():
+            kb = int(Path("/proc/meminfo").read_text().split()[1])
+            mem = f"{kb // 2**20} GB RAM"
+    except Exception:  # noqa: BLE001 - best effort only
+        pass
+    return f"{platform.system()} {platform.machine()}" + (f" ({mem})" if mem else "")
+
+
+def _ollama_host() -> str:
+    host = os.environ.get("OLLAMA_HOST", "http://localhost:11434").strip().rstrip("/")
+    if not host.startswith(("http://", "https://")):
+        host = "http://" + host
+    return host.replace("://0.0.0.0", "://localhost", 1)
+
+
+def check_model(spec) -> None:
+    """Fail early with a clear message: missing key, Ollama down, or model not pulled."""
+    ca = _ca()
+    from commitarena.llm import PROVIDERS  # provider -> (base_url, env var)
+
+    if spec.is_local:
+        host = _ollama_host()
+        try:
+            with urllib.request.urlopen(f"{host}/api/tags", timeout=5) as r:
+                names = {m.get("name") for m in json.load(r).get("models", [])}
+        except (urllib.error.URLError, OSError, ValueError) as e:
+            raise RulelabError(
+                f"cannot reach Ollama at {host}",
+                "Start it with `ollama serve` (or the Ollama app), or set OLLAMA_HOST.",
+            ) from e
+        wanted = spec.name if ":" in spec.name else f"{spec.name}:latest"
+        if wanted not in names and spec.name not in names:
+            raise RulelabError(
+                f"model '{spec.name}' is not available in Ollama",
+                f"Pull it with `ollama pull {spec.name}`. Installed: {', '.join(sorted(n for n in names if n)) or 'none'}.",
+            )
+        return
+    env = PROVIDERS[spec.provider][1]
+    if env and not os.environ.get(env):
+        raise RulelabError(
+            f"missing API key for provider '{spec.provider}'",
+            f"Set the {env} environment variable, or use a local model such as ollama/qwen3:4b.",
+        )
+    if spec.unit_price() is None and spec.provider != "openrouter":
+        raise RulelabError(
+            f"no price known for {spec.id}, so the budget cannot be enforced",
+            "Pass --price IN,OUT (USD per million input/output tokens).",
+        )
+    del ca
+
+
+def check_docker() -> None:
+    ca = _ca()
+    if shutil.which("docker") is None:
+        raise RulelabError("Docker is not installed", "Install Docker Desktop (or Docker Engine) and start it.")
+    try:
+        ca.ensure_docker()
+    except ca.CommitArenaError as e:
+        raise RulelabError(str(e)) from e
+
+
+class CommitarenaRunner:
+    """Runs one agent attempt per call through commitarena's ``run_task``."""
+
+    def __init__(
+        self,
+        tasks: list,
+        model: str,
+        *,
+        context_name: str = "AGENTS.md",
+        inject: str = "both",
+        max_steps: int = 30,
+        price: tuple[float, float] | None = None,
+        tasks_source: str = "",
+        price_source: str | None = None,
+    ) -> None:
+        ca = _ca()
+        try:
+            self.spec = ca.ModelSpec.parse(model, price=price)
+        except ca.CommitArenaError as e:
+            raise RulelabError(str(e)) from e
+        self.model = self.spec.id
+        self.tasks = {t.id: t for t in tasks}
+        self.context_name = context_name
+        self.inject = inject
+        self.max_steps = max_steps
+        self.tasks_source = tasks_source
+        if self.spec.is_local:
+            self.price_source = "local model, 0 USD"
+        elif price is not None:
+            self.price_source = price_source or "--price"
+        else:
+            self.price_source = "commitarena price table (docs.mistral.ai/inference/pricing, checked 2026-11-03)"
+        self.budget = ca.Budget(max_cost=float("inf"))
+        pin, pout = self.spec.unit_price() or (0.0, 0.0)
+        self.price = (pin, pout)
+        self.estimate_note = (
+            f"assumes all {max_steps} steps are used: ~{estimate_tokens(max_steps)[0] // 1000}k input tokens "
+            f"per run at {pin}/{pout} USD per 1M"
+        )
+
+    def set_budget(self, max_cost: float, spent: float = 0.0) -> None:
+        self.budget.max_cost = max_cost
+        self.budget.spent = spent
+
+    def meta(self) -> dict:
+        ca = _ca()
+        return {
+            "engine": f"commitarena {ca.__version__}",
+            "tasks_source": self.tasks_source,
+            "inject": self.inject,
+            "max_steps": self.max_steps,
+            "hardware": hardware(),
+            "price_in_per_mtok": self.price[0],
+            "price_out_per_mtok": self.price[1],
+            "price_source": self.price_source,
+            "cost_method": "usage tokens of every model response x price per 1M tokens",
+            "task_info": {
+                tid: {"repo": str(t.repo), "commit": t.commit, "date": t.date} for tid, t in self.tasks.items()
+            },
+        }
+
+    def task_ids(self) -> list[str]:
+        return list(self.tasks)
+
+    def estimate_run_cost(self) -> float:
+        if self.spec.is_local:
+            return 0.0
+        pin, pout = self.price
+        tin, tout = estimate_tokens(self.max_steps)
+        return (tin * pin + tout * pout) / 1_000_000
+
+    def run(self, task_id: str, context: str | None, repeat: int) -> RunOutcome:
+        ca = _ca()
+        ctx = None
+        if context is not None:
+            ctx = ca.ContextFile(
+                content=context,
+                name=self.context_name,
+                in_repo=self.inject in ("both", "repo"),
+                in_prompt=self.inject in ("both", "prompt"),
+            )
+        try:
+            res = ca.run_task(
+                self.tasks[task_id],
+                self.spec,
+                budget=self.budget,
+                context=ctx,
+                strip_context_files=True,
+                agent_config=ca.AgentConfig(max_steps=self.max_steps),
+                sandbox_config=ca.SandboxConfig(cpus=2, memory="2g"),
+            )
+        except ca.BudgetExceeded:
+            return RunOutcome(None, error="budget_exceeded")
+        except (ca.MissingAPIKey, ca.DockerUnavailable, ca.ModelUnavailable) as e:
+            raise RulelabError(str(e)) from e
+        except ca.CommitArenaError as e:
+            return RunOutcome(None, error=str(e)[:500])
+        except (ValueError, OSError, RuntimeError) as e:
+            # An engine failure on one attempt (e.g. a malformed command from the model) must not
+            # lose the whole experiment: record it as an infrastructure error, excluded from stats.
+            return RunOutcome(None, error=f"engine error: {type(e).__name__}: {e}"[:500])
+        infra = res.stop_reason in ("model_error", "budget_exceeded") or (res.error and not res.steps)
+        return RunOutcome(
+            success=None if infra else bool(res.passed),
+            input_tokens=res.tokens_in,
+            output_tokens=res.tokens_out,
+            cost_usd=res.cost_usd,
+            turns=res.steps,
+            duration_s=res.wall_time_s,
+            error=(res.stop_reason if res.stop_reason == "budget_exceeded" else (res.error or None)),
+        )
+
+
+def load_task_list(tasks_file: Path | None, repo: str | None, cache_dir: Path, max_tasks: int) -> tuple[list, str]:
+    """Tasks from an existing tasks.json, or mined from ``repo`` with commitarena (cached)."""
+    ca = _ca()
+    try:
+        if tasks_file is not None:
+            return ca.load_tasks(str(tasks_file)), str(tasks_file)
+        if repo is None:
+            raise RulelabError("pass --repo or --tasks-file", "--repo mines tasks from git history with commitarena.")
+        cached = cache_dir / "tasks.json"
+        if cached.exists():
+            return ca.load_tasks(str(cached)), str(repo)
+        from commitarena.mine import mine
+
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        tasks, _meta = mine(repo, out=cached, max_tasks=max_tasks)
+        return tasks, str(repo)
+    except ca.CommitArenaError as e:
+        raise RulelabError(str(e)) from e
+
+
+def make_runner(
+    *,
+    repo: str | None = None,
+    tasks_file: Path | None = None,
+    model: str,
+    cache_dir: Path = Path("rulelab-results"),
+    max_tasks: int = 6,
+    context_name: str = "AGENTS.md",
+    inject: str = "both",
+    max_steps: int = 30,
+    price: tuple[float, float] | None = None,
+    price_source: str | None = None,
+    check: bool = True,
+) -> CommitarenaRunner:
+    runner_probe = CommitarenaRunner([], model, price=price)  # validates the model id early
+    if check:
+        check_model(runner_probe.spec)
+        check_docker()
+    tasks, source = load_task_list(tasks_file, repo, cache_dir, max_tasks)
+    if not tasks:
+        raise RulelabError("no tasks found", "Mine more commits (--max-tasks) or pass another --tasks-file.")
+    return CommitarenaRunner(
+        tasks,
+        model,
+        context_name=context_name,
+        inject=inject,
+        max_steps=max_steps,
+        price=price,
+        tasks_source=source,
+        price_source=price_source,
+    )

rulelab/errors.py

@@ -0,0 +1,9 @@
+"""User-facing errors: printed as one clear line plus a hint, never as a traceback."""
+
+from __future__ import annotations
+
+
+class RulelabError(Exception):
+    def __init__(self, message: str, hint: str | None = None) -> None:
+        super().__init__(message)
+        self.hint = hint

rulelab/experiment.py

@@ -0,0 +1,195 @@
+"""Run the leave-one-out experiment under a hard budget and persist results as JSON."""
+
+from __future__ import annotations
+
+import json
+import math
+import platform
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Callable
+
+from . import __version__
+from .configs import Config, build_configs, count_runs
+from .rules import RuleFile
+from .runner import RunOutcome, Runner
+
+SCHEMA = 1
+
+
+class BudgetError(Exception):
+    """The experiment would cost more than --max-cost."""
+
+    def __init__(self, needed: float, max_cost: float, runs: int, per_run: float) -> None:
+        self.needed, self.max_cost, self.runs, self.per_run = needed, max_cost, runs, per_run
+        self.suggested = math.ceil(round(needed * 100, 6)) / 100
+        super().__init__(
+            f"This experiment needs about {self.suggested:.2f} USD ({runs} agent runs x {per_run:.4f} USD) "
+            f"but --max-cost is {max_cost:.2f} USD. Re-run with --max-cost {self.suggested:.2f} to allow it."
+        )
+
+
+@dataclass
+class Plan:
+    configs: list[Config]
+    tasks: list[str]
+    repeats: int
+    per_run_usd: float
+
+    @property
+    def runs(self) -> int:
+        return count_runs(len(self.configs), len(self.tasks), self.repeats)
+
+    @property
+    def estimated_usd(self) -> float:
+        return self.runs * self.per_run_usd
+
+    def schedule(self) -> list[tuple[int, str, Config]]:
+        """Interleaved order (repeat > task > config) so a partial run stays balanced."""
+        return [
+            (rep, task, cfg)
+            for rep in range(1, self.repeats + 1)
+            for task in self.tasks
+            for cfg in self.configs
+        ]
+
+
+def make_plan(rule_file: RuleFile, runner: Runner, repeats: int, tasks: list[str] | None = None) -> Plan:
+    if repeats < 1:
+        raise ValueError("--repeats must be at least 1")
+    if not rule_file.rules:
+        raise ValueError("no rules found in the file: nothing to ablate")
+    available = runner.task_ids()
+    if tasks:
+        missing = [t for t in tasks if t not in available]
+        if missing:
+            raise ValueError(f"unknown task(s): {', '.join(missing)}. Available: {', '.join(available)}")
+        chosen = list(tasks)
+    else:
+        chosen = available
+    if not chosen:
+        raise ValueError("the runner has no tasks")
+    return Plan(build_configs(rule_file), chosen, repeats, runner.estimate_run_cost())
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+
+
+def new_results(rule_file: RuleFile, plan: Plan, runner: Runner, max_cost: float, meta: dict | None = None) -> dict:
+    return {
+        "schema": SCHEMA,
+        "tool": "rulelab",
+        "version": __version__,
+        "created": _now(),
+        "finished": None,
+        "status": "running",
+        "model": runner.model,
+        "meta": {
+            "python": platform.python_version(),
+            "platform": f"{platform.system()} {platform.machine()}",
+            **(meta or {}),
+        },
+        "file": {
+            "path": rule_file.path,
+            "granularity": rule_file.granularity,
+            "text": rule_file.text,
+            "tokens": rule_file.tokens,
+            "rules": [r.to_dict() for r in rule_file.rules],
+        },
+        "configs": [{"id": c.id, "removed": c.removed} for c in plan.configs],
+        "tasks": plan.tasks,
+        "repeats": plan.repeats,
+        "max_cost": max_cost,
+        "estimated_usd": plan.estimated_usd,
+        "spent_usd": 0.0,
+        "runs": [],
+    }
+
+
+def save(results: dict, path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(".tmp")
+    tmp.write_text(json.dumps(results, indent=1, ensure_ascii=False) + "\n", encoding="utf-8")
+    tmp.replace(path)
+
+
+def load(path: Path) -> dict:
+    data = json.loads(Path(path).read_text(encoding="utf-8"))
+    if data.get("tool") != "rulelab" or "runs" not in data:
+        raise ValueError(f"{path} is not a rulelab results file")
+    return data
+
+
+def execute(
+    plan: Plan,
+    runner: Runner,
+    results: dict,
+    out: Path,
+    max_cost: float,
+    on_run: Callable[[int, int, dict], None] | None = None,
+    concurrency: int = 1,
+) -> dict:
+    """Run every scheduled attempt not already in ``results`` (so it can resume).
+
+    Up to ``concurrency`` attempts run at once. Before starting each one, the measured spend plus
+    the expected cost of every attempt in flight (and this one) is checked against ``max_cost``;
+    if it does not fit, no new attempts start, the ones in flight finish, and the experiment ends
+    with status ``budget_exceeded``. Everything measured is saved after each attempt.
+    """
+    done = {(r["config"], r["task"], r["repeat"]) for r in results["runs"]}
+    schedule = plan.schedule()
+    total = len(schedule)
+    todo = [(i, rep, task, cfg) for i, (rep, task, cfg) in enumerate(schedule, start=1) if (cfg.id, task, rep) not in done]
+    spent = float(results.get("spent_usd", 0.0))
+    results["status"] = "running"
+    stop: str | None = None
+    finished = 0
+    pool = ThreadPoolExecutor(max_workers=max(1, concurrency))
+    in_flight: dict = {}
+    queue = list(todo)
+    try:
+        while queue or in_flight:
+            while queue and stop is None and len(in_flight) < max(1, concurrency):
+                if spent + (len(in_flight) + 1) * plan.per_run_usd > max_cost + 1e-12:
+                    stop = "budget_exceeded"
+                    break
+                i, rep, task, cfg = queue.pop(0)
+                in_flight[pool.submit(runner.run, task, cfg.context, rep)] = (i, rep, task, cfg)
+            if not in_flight:
+                break
+            ready, _ = wait(list(in_flight), return_when=FIRST_COMPLETED)
+            for fut in ready:
+                i, rep, task, cfg = in_flight.pop(fut)
+                outcome: RunOutcome = fut.result()
+                spent += outcome.cost_usd
+                if outcome.error == "budget_exceeded":
+                    stop = "budget_exceeded"
+                    results["spent_usd"] = round(spent, 6)
+                    continue
+                record = {"config": cfg.id, "task": task, "repeat": rep, **asdict(outcome)}
+                results["runs"].append(record)
+                results["spent_usd"] = round(spent, 6)
+                save(results, out)
+                finished += 1
+                if on_run:
+                    on_run(len(done) + finished, total, record)
+        results["status"] = stop or ("complete" if not queue else "interrupted")
+    except KeyboardInterrupt:
+        results["status"] = "interrupted"
+        pool.shutdown(wait=False, cancel_futures=True)
+    finally:
+        pool.shutdown(wait=False, cancel_futures=True)
+    results["finished"] = _now()
+    save(results, out)
+    return results
+
+
+def check_budget(plan: Plan, max_cost: float, done_runs: int = 0, already_spent: float = 0.0) -> None:
+    """Refuse to start when the whole experiment cannot fit in ``max_cost``."""
+    remaining_runs = max(plan.runs - done_runs, 0)
+    needed = remaining_runs * plan.per_run_usd + already_spent
+    if needed > max_cost + 1e-12:
+        raise BudgetError(needed, max_cost, remaining_runs, plan.per_run_usd)

rulelab/report.py

@@ -0,0 +1,205 @@
+"""Static HTML report (ES/EN, light/dark, mobile) with the annotated rules file."""
+
+from __future__ import annotations
+
+import math
+from pathlib import Path
+
+from jinja2 import Environment, PackageLoader, select_autoescape
+from markupsafe import Markup, escape
+
+from . import __version__
+from .analysis import Analysis, file_monthly_usd, file_tokens_per_run
+from .configs import FULL, NONE
+
+VERDICT_LABELS = {
+    "helps": ("helps", "ayuda"),
+    "no_effect": ("no effect", "sin efecto"),
+    "hurts": ("hurts", "empeora"),
+    "inconclusive": ("inconclusive", "no concluyente"),
+}
+
+PLURAL_ES = {"helps": "ayudan", "no_effect": "sin efecto", "hurts": "empeoran", "inconclusive": "no concluyentes"}
+
+
+def _num(x: float) -> bool:
+    return x is not None and not (isinstance(x, float) and math.isnan(x))
+
+
+def pp(x: float | None, signed: bool = True) -> str:
+    if x is None or not _num(x):
+        return "n/a"
+    v = round(x * 100)
+    if v == 0:
+        return "0"
+    s = f"{v:+d}" if signed else f"{v:d}"
+    return s.replace("-", "−")
+
+
+def pct(x: float | None) -> str:
+    if x is None or not _num(x):
+        return "n/a"
+    return f"{round(x * 100)}%"
+
+
+def signed_int(x: float | None) -> str:
+    if x is None or not _num(x):
+        return "n/a"
+    v = round(x)
+    return (f"{v:+,d}" if v else "0").replace("-", "−")
+
+
+def usd(x: float | None, digits: int = 2) -> str:
+    if x is None or not _num(x):
+        return "n/a"
+    if 0 < abs(x) < 10 ** (-digits):
+        return f"<{10 ** (-digits):.{digits}f}"
+    return f"{x:,.{digits}f}"
+
+
+def _es(text: str) -> str:
+    """English-formatted number -> Spanish (1,953.5 -> 1.953,5)."""
+    return text.replace(",", "\0").replace(".", ",").replace("\0", ".")
+
+
+def _both(en: str, es: str) -> Markup:
+    return Markup('<span lang="en">%s</span><span lang="es">%s</span>') % (escape(en), escape(es))
+
+
+def b_num(x: float | None, digits: int = 0) -> Markup:
+    if x is None or not _num(x):
+        return _both("n/a", "n/d")
+    en = f"{x:,.{digits}f}"
+    return _both(en, _es(en))
+
+
+def b_signed_int(x: float | None) -> Markup:
+    en = signed_int(x)
+    return _both(en, "n/d" if en == "n/a" else _es(en))
+
+
+def b_usd(x: float | None, digits: int = 2) -> Markup:
+    en = usd(x, digits)
+    if en == "n/a":
+        return _both("n/a", "n/d")
+    return _both(f"{en} USD", f"{_es(en)} US$")
+
+
+def b_pct(x: float | None) -> Markup:
+    en = pct(x)
+    return _both(en, "n/d" if en == "n/a" else en.replace("%", "\u00a0%"))
+
+
+def b_plain(x) -> Markup:
+    en = str(x)
+    return _both(en, _es(en))
+
+
+def ci_bar(low: float, high: float, est: float, margin: float, width: int = 168) -> dict:
+    """Geometry for the interval glyph on a fixed -100..+100 pp axis."""
+    if not (_num(low) and _num(high)):
+        return {"ok": False}
+
+    def x(v: float) -> float:
+        v = max(-1.0, min(1.0, v))
+        return round((v + 1) / 2 * width, 1)
+
+    return {
+        "ok": True,
+        "w": width,
+        "x0": x(low),
+        "x1": x(high),
+        "xe": x(est) if _num(est) else None,
+        "z": x(0),
+        "m0": x(-margin),
+        "m1": x(margin),
+    }
+
+
+def _annotated(a: Analysis) -> list[dict]:
+    """Split the original file into blocks: fixed lines and rule blocks with their result."""
+    lines = a.results["file"]["text"].replace("\r\n", "\n").split("\n")
+    if lines and lines[-1] == "":
+        lines = lines[:-1]
+    by_start = {r.start_line: r for r in a.rules}
+    blocks: list[dict] = []
+    i = 1
+    fixed: list[str] = []
+    while i <= len(lines):
+        r = by_start.get(i)
+        if r:
+            if fixed:
+                blocks.append({"rule": None, "text": "\n".join(fixed)})
+                fixed = []
+            blocks.append({"rule": r, "text": "\n".join(lines[r.start_line - 1 : r.end_line])})
+            i = r.end_line + 1
+        else:
+            fixed.append(lines[i - 1])
+            i += 1
+    if fixed:
+        blocks.append({"rule": None, "text": "\n".join(fixed)})
+    # drop empty fixed blocks made only of blank lines between rules
+    return [b for b in blocks if b["rule"] or b["text"].strip()]
+
+
+def render_report(a: Analysis, *, recorded_note: dict | None = None) -> str:
+    env = Environment(
+        loader=PackageLoader("rulelab", "templates"),
+        autoescape=select_autoescape(["html", "j2"]),
+        trim_blocks=True,
+        lstrip_blocks=True,
+    )
+    env.filters.update(pp=pp, pct=b_pct, usd=b_usd, signed_int=b_signed_int, num=b_num, plain=b_plain)
+    res = a.results
+    configs = []
+    for c in res["configs"]:
+        rate, s, n = a.rates[c["id"]]
+        runs = [r for r in res["runs"] if r["config"] == c["id"] and r.get("success") is not None]
+        configs.append(
+            {
+                "id": c["id"],
+                "removed": c["removed"],
+                "rate": rate,
+                "s": s,
+                "n": n,
+                "planned": len(res["tasks"]) * res["repeats"],
+                "tokens": (sum(r["input_tokens"] for r in runs) / len(runs)) if runs else float("nan"),
+                "cost": (sum(r["cost_usd"] for r in runs) / len(runs)) if runs else float("nan"),
+            }
+        )
+    rules = [
+        {
+            "r": r,
+            "bar": ci_bar(r.success.low, r.success.high, r.success.estimate, a.margin),
+            "share": r.tokens / res["file"]["tokens"] if res["file"]["tokens"] else 0,
+        }
+        for r in a.rules
+    ]
+    template = env.get_template("report.html.j2")
+    return template.render(
+        a=a,
+        res=res,
+        rules=rules,
+        configs=configs,
+        blocks=_annotated(a),
+        verdicts=VERDICT_LABELS,
+        plural_es=PLURAL_ES,
+        whole_bar=ci_bar(a.whole_file.low, a.whole_file.high, a.whole_file.estimate, a.margin)
+        if a.whole_file
+        else {"ok": False},
+        full_rate=a.rates.get(FULL),
+        none_rate=a.rates.get(NONE),
+        file_monthly=file_monthly_usd(a),
+        file_per_run=file_tokens_per_run(a),
+        total_runs=len(res["runs"]),
+        planned_runs=len(res["configs"]) * len(res["tasks"]) * res["repeats"],
+        recorded=recorded_note,
+        version=__version__,
+    )
+
+
+def write_report(a: Analysis, out: Path, *, recorded_note: dict | None = None) -> Path:
+    out = Path(out)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(render_report(a, recorded_note=recorded_note), encoding="utf-8")
+    return out