commitarena 0.1.1__py3-none-any.whl

commitarena/errors.py

@@ -0,0 +1,40 @@
+"""User-facing errors. The CLI prints these without a traceback."""
+
+from __future__ import annotations
+
+
+class CommitArenaError(Exception):
+    """Base class. `str(err)` is a complete, actionable message."""
+
+
+class MissingAPIKey(CommitArenaError):
+    def __init__(self, provider: str, env_var: str):
+        super().__init__(
+            f"Missing API key for provider '{provider}'. Set the {env_var} environment variable "
+            f"(for example in your shell or a .env file you source), or use a local model such as "
+            f"'ollama/qwen3:4b'."
+        )
+        self.provider = provider
+        self.env_var = env_var
+
+
+class DockerUnavailable(CommitArenaError):
+    pass
+
+
+class ModelUnavailable(CommitArenaError):
+    """Model not found, or the provider (e.g. Ollama) cannot be reached."""
+
+
+class BudgetExceeded(CommitArenaError):
+    def __init__(self, spent: float, max_cost: float):
+        super().__init__(
+            f"Budget exceeded: spent ${spent:.4f} of --max-cost ${max_cost:.2f}. Stopped before the next "
+            f"model call. Partial results were kept; raise --max-cost to continue."
+        )
+        self.spent = spent
+        self.max_cost = max_cost
+
+
+class TaskError(CommitArenaError):
+    pass

commitarena/llm.py

@@ -0,0 +1,206 @@
+"""One OpenAI-compatible client with provider presets, pricing and a spend cap."""
+
+from __future__ import annotations
+
+import os
+import random
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+from .errors import BudgetExceeded, CommitArenaError, MissingAPIKey, ModelUnavailable
+
+
+def _ollama_base() -> str:
+    host = os.environ.get("OLLAMA_HOST", "http://localhost:11434").strip().rstrip("/")
+    if not host.startswith(("http://", "https://")):
+        host = "http://" + host
+    if host.startswith("http://0.0.0.0"):
+        host = host.replace("0.0.0.0", "localhost", 1)
+    return host + "/v1"
+
+
+# provider -> (base_url, env var with the key or None for local)
+PROVIDERS: dict[str, tuple[str, str | None]] = {
+    "openrouter": ("https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"),
+    "openai": ("https://api.openai.com/v1", "OPENAI_API_KEY"),
+    "mistral": ("https://api.mistral.ai/v1", "MISTRAL_API_KEY"),
+    "ollama": ("", None),  # resolved at call time from OLLAMA_HOST
+}
+
+# USD per million tokens (input, output). List prices from docs.mistral.ai/inference/pricing,
+# checked 2026-11-03. Override with --price or ModelSpec(price=...).
+PRICES: dict[str, tuple[float, float]] = {
+    "mistral/codestral-latest": (0.30, 0.90),
+    "mistral/codestral-2508": (0.30, 0.90),
+    "mistral/mistral-small-latest": (0.15, 0.60),
+    "mistral/mistral-small-2603": (0.15, 0.60),
+    "mistral/ministral-8b-latest": (0.15, 0.15),
+    "mistral/ministral-8b-2512": (0.15, 0.15),
+    "mistral/mistral-large-4": (0.70, 2.10),
+    "mistral/mistral-large-4-0": (0.70, 2.10),
+}
+
+
+@dataclass(frozen=True)
+class ModelSpec:
+    """A model id of the form ``provider/name``, e.g. ``ollama/qwen3:4b``."""
+
+    provider: str
+    name: str
+    price: tuple[float, float] | None = None  # USD per 1M (input, output)
+
+    @classmethod
+    def parse(cls, spec: str, price: tuple[float, float] | None = None) -> ModelSpec:
+        if "/" not in spec:
+            raise CommitArenaError(
+                f"Model '{spec}' must look like provider/name, e.g. ollama/qwen3:4b or "
+                f"mistral/codestral-latest. Providers: {', '.join(PROVIDERS)}."
+            )
+        provider, name = spec.split("/", 1)
+        if provider not in PROVIDERS:
+            raise CommitArenaError(f"Unknown provider '{provider}'. Use one of: {', '.join(PROVIDERS)}.")
+        return cls(provider, name, price)
+
+    @property
+    def id(self) -> str:
+        return f"{self.provider}/{self.name}"
+
+    @property
+    def is_local(self) -> bool:
+        return self.provider == "ollama"
+
+    def unit_price(self) -> tuple[float, float] | None:
+        if self.price is not None:
+            return self.price
+        if self.is_local:
+            return (0.0, 0.0)
+        return PRICES.get(self.id)
+
+
+@dataclass
+class Budget:
+    """Shared spend cap in USD. Checked before every model call; local models cost 0."""
+
+    max_cost: float = 1.0
+    spent: float = 0.0
+
+    def check(self) -> None:
+        if self.spent >= self.max_cost:
+            raise BudgetExceeded(self.spent, self.max_cost)
+
+    def add(self, usd: float) -> None:
+        self.spent += usd
+
+
+@dataclass
+class Usage:
+    tokens_in: int = 0
+    tokens_out: int = 0
+    cost_usd: float = 0.0
+    calls: int = 0
+
+    def add(self, other: Usage) -> None:
+        self.tokens_in += other.tokens_in
+        self.tokens_out += other.tokens_out
+        self.cost_usd += other.cost_usd
+        self.calls += other.calls
+
+
+@dataclass
+class ChatClient:
+    """Thin wrapper over the `openai` SDK. Retries 429/5xx with exponential backoff."""
+
+    model: ModelSpec
+    budget: Budget = field(default_factory=Budget)
+    max_retries: int = 12
+    timeout: float = 600.0
+    _client: Any = None
+    _connected: bool = False  # a local server that answered once gets retries on later drops
+
+    def __post_init__(self) -> None:
+        if self.model.unit_price() is None and self.model.provider != "openrouter":
+            raise CommitArenaError(
+                f"No price known for {self.model.id}, so the budget cannot be enforced. Pass "
+                f"--price '{self.model.id}=IN,OUT' (USD per million tokens)."
+            )
+
+    def _sdk(self):
+        if self._client is None:
+            from openai import OpenAI
+
+            base, env = PROVIDERS[self.model.provider]
+            if self.model.provider == "ollama":
+                base, key = _ollama_base(), "ollama"
+                self.timeout = max(self.timeout, 1800.0)  # a busy local server can be slow
+            else:
+                key = os.environ.get(env or "", "")
+                if not key:
+                    raise MissingAPIKey(self.model.provider, env or "")
+            self._client = OpenAI(base_url=base, api_key=key, timeout=self.timeout, max_retries=0)
+        return self._client
+
+    def chat(self, messages: list[dict], tools: list[dict] | None = None, **kw) -> tuple[Any, Usage]:
+        import openai
+
+        self.budget.check()
+        client = self._sdk()
+        extra: dict[str, Any] = {}
+        if self.model.provider == "openrouter":
+            extra["extra_body"] = {"usage": {"include": True}}
+        delay = 2.0
+        for attempt in range(self.max_retries + 1):
+            try:
+                resp = client.chat.completions.create(
+                    model=self.model.name, messages=messages, tools=tools, **extra, **kw
+                )
+                self._connected = True
+                break
+            except openai.AuthenticationError as e:
+                _, env = PROVIDERS[self.model.provider]
+                raise CommitArenaError(
+                    f"{self.model.provider} rejected the API key in {env} (401). Check the key."
+                ) from e
+            except openai.NotFoundError as e:
+                hint = (
+                    f" Pull it first: ollama pull {self.model.name}"
+                    if self.model.is_local
+                    else " Check the model name with the provider."
+                )
+                raise ModelUnavailable(f"Model '{self.model.id}' was not found.{hint}") from e
+            except openai.APIConnectionError as e:
+                if self.model.is_local and (attempt >= 3 or not self._connected):
+                    raise ModelUnavailable(
+                        f"Cannot reach Ollama at {_ollama_base()}. Start it with 'ollama serve' "
+                        f"or set OLLAMA_HOST."
+                    ) from e
+                if attempt == self.max_retries:
+                    raise ModelUnavailable(f"Cannot reach {self.model.provider}: {e}") from e
+            except (openai.RateLimitError, openai.InternalServerError, openai.APITimeoutError) as e:
+                if attempt == self.max_retries:
+                    raise ModelUnavailable(
+                        f"{self.model.provider} kept failing after {attempt + 1} attempts "
+                        f"(last error: {type(e).__name__}). Try later or use a local model."
+                    ) from e
+            except openai.APIStatusError as e:
+                if e.status_code == 402:
+                    raise CommitArenaError(f"{self.model.provider} reports no credit left (402).") from e
+                raise CommitArenaError(f"{self.model.provider} error {e.status_code}: {e.message}") from e
+            time.sleep(min(delay, 90.0) * (0.75 + random.random() / 2))
+            delay *= 2
+        usage = self._usage(resp)
+        self.budget.add(usage.cost_usd)
+        return resp, usage
+
+    def _usage(self, resp: Any) -> Usage:
+        u = getattr(resp, "usage", None)
+        tin = int(getattr(u, "prompt_tokens", 0) or 0)
+        tout = int(getattr(u, "completion_tokens", 0) or 0)
+        cost = None
+        if u is not None and self.model.provider == "openrouter":
+            extra = getattr(u, "model_extra", None) or {}
+            cost = extra.get("cost")
+        if cost is None:
+            pin, pout = self.model.unit_price() or (0.0, 0.0)
+            cost = (tin * pin + tout * pout) / 1_000_000
+        return Usage(tin, tout, float(cost), 1)

commitarena/mine.py

@@ -0,0 +1,248 @@
+"""Mine validated FAIL_TO_PASS tasks from a repository's git history (Python + pytest)."""
+
+from __future__ import annotations
+
+import shutil
+import tempfile
+from collections import Counter
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+
+from .sandbox import EnvSpec, Sandbox, SandboxConfig, build_image, ensure_docker, image_exists
+from .tasks import (
+    Task,
+    build_workspace,
+    extract_commit,
+    git,
+    is_doc_path,
+    is_test_path,
+    repo_slug,
+    resolve_repo,
+    unified_diff,
+)
+from .testrun import run_tests
+
+MAX_TEST_DIFF_CHARS = 12000
+
+
+@dataclass
+class Candidate:
+    sha: str
+    parent: str
+    date: str
+    test_paths: list[str]
+    source_paths: list[str]
+    source_lines: int
+
+
+def find_candidates(
+    repo: Path,
+    rev: str = "HEAD",
+    since: str | None = None,
+    until: str | None = None,
+    limit: int = 200,
+    max_source_files: int = 4,
+    max_source_lines: int = 200,
+) -> tuple[list[Candidate], Counter]:
+    """Non-merge commits that change both test files and .py source files."""
+    args = ["log", "--no-merges", "--format=%H%x09%P%x09%cI", f"-n{limit}"]
+    if since:
+        args.append(f"--since={since}")
+    if until:
+        args.append(f"--until={until}")
+    args.append(rev)
+    skipped: Counter = Counter()
+    out: list[Candidate] = []
+    for line in git(repo, *args).splitlines():
+        sha, parents, date = line.split("\t")
+        if not parents or " " in parents:
+            skipped["root or merge commit"] += 1
+            continue
+        numstat = git(repo, "diff", "--numstat", parents, sha)
+        tests, sources, lines = [], [], 0
+        for row in numstat.splitlines():
+            added, deleted, path = row.split("\t", 2)
+            if is_test_path(path):
+                tests.append(path)
+            elif path.endswith(".py") and not is_doc_path(path):
+                sources.append(path)
+                if added != "-":
+                    lines += int(added) + int(deleted)
+        if not tests or not sources:
+            skipped["does not touch both tests and source"] += 1
+            continue
+        if len(sources) > max_source_files or lines > max_source_lines:
+            skipped["fix too large"] += 1
+            continue
+        out.append(Candidate(sha, parents, date, tests, sources, lines))
+    return out, skipped
+
+
+def _test_side_files(repo: Path, cand: Candidate) -> dict[str, str | None]:
+    files: dict[str, str | None] = {}
+    names = git(repo, "diff", "--name-status", "--no-renames", cand.parent, cand.sha)
+    for row in names.splitlines():
+        status, path = row.split("\t", 1)
+        if not is_test_path(path):
+            continue
+        if status.startswith("D"):
+            files[path] = None
+        else:
+            try:
+                files[path] = git(repo, "show", f"{cand.sha}:{path}", text=False).decode()
+            except UnicodeDecodeError:
+                continue
+    return files
+
+
+def build_prompt(f2p: list[str], test_diff: str, messages: dict[str, str], test_paths: list[str]) -> str:
+    """Task description from the failing tests only. The commit message is never used."""
+    shown = f2p[:25]
+    lines = [
+        "The repository in /work has a bug or a missing behavior. These tests currently fail and must pass "
+        "after your change:",
+        "",
+        *[f"- {t}" for t in shown],
+    ]
+    if len(f2p) > len(shown):
+        lines.append(f"- ... and {len(f2p) - len(shown)} more")
+    if len(test_diff) > MAX_TEST_DIFF_CHARS:
+        test_diff = test_diff[:MAX_TEST_DIFF_CHARS] + "\n... [test diff truncated]\n"
+    lines += [
+        "",
+        "The tests were written to describe the expected behavior:",
+        "",
+        "```diff",
+        test_diff.rstrip(),
+        "```",
+    ]
+    msgs = [f"{t}:\n{messages[t][:800]}" for t in shown[:3] if t in messages]
+    if msgs:
+        lines += ["", "Current failure output (truncated):", "", "```", "\n\n".join(msgs), "```"]
+    py_tests = [p for p in test_paths if p.endswith(".py")]
+    lines += [
+        "",
+        "Change the source code, not the tests: test files are restored before scoring. Do not break the "
+        "rest of the test suite.",
+    ]
+    if py_tests:
+        lines.append(f"Run the relevant tests with: python -m pytest -q {' '.join(py_tests)}")
+    return "\n".join(lines)
+
+
+def mine(
+    repo: str,
+    *,
+    out: Path,
+    rev: str = "HEAD",
+    since: str | None = None,
+    until: str | None = None,
+    scan: int = 200,
+    max_tasks: int = 10,
+    env: EnvSpec | None = None,
+    sandbox: SandboxConfig | None = None,
+    test_timeout: int = 600,
+    on_event: Callable[[str, dict], None] | None = None,
+) -> tuple[list[Task], dict]:
+    """Find candidate commits and keep those whose new tests fail before the fix and pass after it."""
+    from .tasks import save_tasks
+
+    emit = on_event or (lambda kind, data: None)
+    ensure_docker()
+    env = env or EnvSpec()
+    sandbox = sandbox or SandboxConfig(command_timeout=test_timeout)
+    path = resolve_repo(repo)
+    slug = repo_slug(repo)
+    tag = env.image_tag(slug)
+    work = Path(tempfile.mkdtemp(prefix="commitarena-mine-"))
+    try:
+        if not image_exists(tag):
+            emit("build", {"image": tag})
+            snap = work / "head"
+            extract_commit(path, rev, snap)
+            build_image(tag, env, snap, work)
+        cands, skipped = find_candidates(path, rev, since, until, scan)
+        emit("candidates", {"count": len(cands), "skipped": dict(skipped)})
+        tasks: list[Task] = []
+        rejected: Counter = Counter()
+        for cand in cands:
+            if len(tasks) >= max_tasks:
+                break
+            emit("validate", {"sha": cand.sha[:10], "date": cand.date})
+            test_files = _test_side_files(path, cand)
+            stub = Task(
+                id="", repo=repo, commit=cand.sha, parent=cand.parent, date=cand.date, prompt="",
+                fail_to_pass=[], pass_to_pass=[], test_files=test_files, gold_patch="", image=tag,
+            )  # fmt: skip
+            before = build_workspace(stub, work / f"{cand.sha[:10]}-before", with_git=False)
+            after = work / f"{cand.sha[:10]}-after"
+            extract_commit(path, cand.sha, after)
+            with Sandbox(tag, sandbox) as sb:
+                sb.copy_in(before)
+                res_before = run_tests(sb, test_timeout)
+            with Sandbox(tag, sandbox) as sb:
+                sb.copy_in(after)
+                res_after = run_tests(sb, test_timeout)
+            reason = None
+            if not res_after.ok or not res_before.ok:
+                reason = "test run crashed or timed out"
+            f2p = sorted(res_after.passed() - res_before.passed())
+            p2p = sorted(res_after.passed() & res_before.passed())
+            broken = res_before.passed() - res_after.passed()
+            if reason is None and not f2p:
+                reason = "no test goes from failing to passing"
+            if reason is None and broken:
+                reason = "fix breaks previously passing tests"
+            if reason:
+                rejected[reason] += 1
+                emit("reject", {"sha": cand.sha[:10], "reason": reason})
+            else:
+                parent_tree = work / f"{cand.sha[:10]}-parent"
+                extract_commit(path, cand.parent, parent_tree)
+                test_diff = unified_diff(parent_tree, after, only=sorted(test_files))
+                gold_paths = [
+                    p for p in set(_all_changed(path, cand)) if not is_test_path(p) and not is_doc_path(p)
+                ]
+                gold = unified_diff(before, after, only=gold_paths)
+                task = Task(
+                    id=f"{slug}-{cand.sha[:8]}",
+                    repo=repo,
+                    commit=cand.sha,
+                    parent=cand.parent,
+                    date=cand.date,
+                    prompt=build_prompt(f2p, test_diff, res_before.messages, sorted(test_files)),
+                    fail_to_pass=f2p,
+                    pass_to_pass=p2p,
+                    test_files=test_files,
+                    gold_patch=gold,
+                    image=tag,
+                    env=env.__dict__.copy(),
+                    source_files=cand.source_paths,
+                    title_hint=", ".join(cand.source_paths),
+                )
+                tasks.append(task)
+                emit("task", {"id": task.id, "f2p": len(f2p), "p2p": len(p2p)})
+                shutil.rmtree(parent_tree, ignore_errors=True)
+            shutil.rmtree(before, ignore_errors=True)
+            shutil.rmtree(after, ignore_errors=True)
+        meta = {
+            "repo": repo,
+            "rev": rev,
+            "image": tag,
+            "env": env.__dict__.copy(),
+            "candidates": len(cands),
+            "skipped": dict(skipped),
+            "rejected": dict(rejected),
+            "validated": len(tasks),
+        }
+        save_tasks(out, tasks, meta)
+        return tasks, meta
+    finally:
+        shutil.rmtree(work, ignore_errors=True)
+
+
+def _all_changed(repo: Path, cand: Candidate) -> list[str]:
+    return [
+        p for p in git(repo, "diff", "--name-only", "--no-renames", cand.parent, cand.sha).splitlines() if p
+    ]

commitarena/report.py

@@ -0,0 +1,132 @@
+"""Static HTML report and leaderboard aggregation."""
+
+from __future__ import annotations
+
+import json
+from importlib import resources
+from pathlib import Path
+
+from . import __version__
+from .tasks import load_tasks
+
+# Upper bounds used when no --cutoff is given: the model's public release date.
+# Anything committed after it cannot be in its training data. Override per run with --cutoff.
+DEFAULT_CUTOFFS: dict[str, str] = {
+    "ollama/qwen3:4b": "2025-04-29",  # Qwen3 release (Qwen blog, 2025-04-29)
+    "ollama/qwen2.5-coder:7b": "2024-09-19",  # Qwen2.5-Coder release
+    "mistral/codestral-latest": "2025-07-31",  # codestral-2508, "released end of July 2025" (Mistral docs)
+    "mistral/codestral-2508": "2025-07-31",
+    "mistral/mistral-small-latest": "2026-03-01",  # mistral-small-2603 (v26.03)
+    "mistral/mistral-small-2603": "2026-03-01",
+    "mistral/ministral-8b-latest": "2025-12-01",  # ministral-8b-2512 (Ministral 3, v25.12)
+    "mistral/ministral-8b-2512": "2025-12-01",
+    "mistral/mistral-large-4": "2026-10-01",  # Mistral Large 4, v26.10 (Mistral docs)
+    "mistral/mistral-large-4-0": "2026-10-01",
+    "ollama/qwen3:4b-16k": "2025-04-29",  # same weights as qwen3:4b, larger context window
+}
+
+
+def _split():
+    return {"passed": 0, "total": 0}
+
+
+def leaderboard(results: list[dict], cutoffs: dict[str, str]) -> list[dict]:
+    """Aggregate per model. 'before' = tasks dated before the model's cutoff (may be memorized)."""
+    rows: dict[str, dict] = {}
+    for r in results:
+        m = rows.setdefault(
+            r["model"],
+            {"model": r["model"], "passed": 0, "total": 0, "cost": 0.0, "tokens": 0, "time": 0.0, "steps": 0,
+             "before": _split(), "after": _split(), "cutoff": cutoffs.get(r["model"]), "stops": {}},
+        )  # fmt: skip
+        m["total"] += 1
+        m["passed"] += int(bool(r["passed"]))
+        m["cost"] += r.get("cost_usd", 0.0)
+        m["tokens"] += r.get("tokens_in", 0) + r.get("tokens_out", 0)
+        m["time"] += r.get("wall_time_s", 0.0)
+        m["steps"] += r.get("steps", 0)
+        stop = r.get("stop_reason") or "unknown"
+        m["stops"][stop] = m["stops"].get(stop, 0) + 1
+        cut = cutoffs.get(r["model"])
+        if cut and r.get("task_date"):
+            side = "before" if r["task_date"][:10] < cut else "after"
+            m[side]["total"] += 1
+            m[side]["passed"] += int(bool(r["passed"]))
+    out = []
+    for m in rows.values():
+        n = m["total"] or 1
+        m["rate"] = m["passed"] / n
+        m["avg_time"] = m["time"] / n
+        m["avg_steps"] = m["steps"] / n
+        out.append(m)
+    out.sort(key=lambda m: (-m["rate"], m["cost"]))
+    return out
+
+
+def load_run(run_dir: Path, cutoff_overrides: dict[str, str] | None = None) -> dict:
+    meta = json.loads((run_dir / "run.json").read_text()) if (run_dir / "run.json").exists() else {}
+    results = [
+        json.loads(line) for line in (run_dir / "results.jsonl").read_text().splitlines() if line.strip()
+    ]
+    tasks = load_tasks(run_dir / "tasks.json") if (run_dir / "tasks.json").exists() else []
+    cutoffs = {**DEFAULT_CUTOFFS, **meta.get("cutoffs", {}), **(cutoff_overrides or {})}
+    return {"meta": meta, "results": results, "tasks": tasks, "cutoffs": cutoffs}
+
+
+def _commit_url(repo: str, sha: str) -> str | None:
+    r = repo.removesuffix(".git").rstrip("/")
+    if r.startswith("https://github.com/") or r.startswith("https://gitlab.com/"):
+        return f"{r}/commit/{sha}"
+    return None
+
+
+def build_payload(run_dir: Path, cutoff_overrides: dict[str, str] | None = None) -> dict:
+    data = load_run(run_dir, cutoff_overrides)
+    used = {r["task_id"] for r in data["results"]}
+    tasks = [
+        {
+            "id": t.id,
+            "date": t.date,
+            "files": t.source_files,
+            "f2p": len(t.fail_to_pass),
+            "p2p": len(t.pass_to_pass),
+            "gold": t.gold_patch,
+            "url": _commit_url(t.repo, t.commit),
+            "commit": t.commit[:10],
+        }
+        for t in sorted(data["tasks"], key=lambda t: t.date)
+        if t.id in used
+    ]
+    models = sorted({r["model"] for r in data["results"]})
+    return {
+        "meta": {**data["meta"], "report_version": __version__},
+        "tasks": tasks,
+        "models": models,
+        "cutoffs": {m: data["cutoffs"].get(m) for m in models},
+        "board": leaderboard(data["results"], data["cutoffs"]),
+        "results": [
+            {k: r.get(k) for k in ("task_id", "model", "passed", "fail_to_pass_passed", "fail_to_pass_total",
+                                   "pass_to_pass_failed", "cost_usd", "tokens_in", "tokens_out", "steps",
+                                   "wall_time_s", "stop_reason", "patch", "ignored_test_edits")}
+            for r in data["results"]
+        ],
+    }  # fmt: skip
+
+
+def render(payload: dict) -> str:
+    template = resources.files("commitarena").joinpath("report_template.html").read_text()
+    blob = json.dumps(payload, ensure_ascii=False).replace("</", "<\\/")
+    return template.replace("/*__DATA__*/{}", blob)
+
+
+def write_report(
+    run_dir: Path, out: Path | None = None, cutoff_overrides: dict[str, str] | None = None
+) -> Path:
+    out = out or run_dir / "report.html"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(render(build_payload(run_dir, cutoff_overrides)))
+    return out
+
+
+def demo_dir() -> Path:
+    return Path(str(resources.files("commitarena").joinpath("demo")))