commitarena 0.1.1__py3-none-any.whl

commitarena/tasks.py

@@ -0,0 +1,229 @@
+"""Task model, task files and workspace construction from a git repository."""
+
+from __future__ import annotations
+
+import io
+import json
+import os
+import re
+import subprocess
+import tarfile
+from dataclasses import asdict, dataclass, field
+from pathlib import Path, PurePosixPath
+
+from .errors import TaskError
+from .sandbox import EnvSpec
+
+TASKS_FORMAT = 1
+
+_TEST_DIRS = {"tests", "test", "testing"}
+_DOC_RE = re.compile(
+    r"(^|/)(docs?/|examples?/|benchmarks?/|CHANGE(S|LOG)|HISTORY|AUTHORS|NEWS)|\.(md|rst|txt)$", re.I
+)
+IGNORED_PARTS = {".git", "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache"}
+
+
+def is_test_path(path: str) -> bool:
+    """Files the agent may not change: anything under a tests dir, test_*.py, *_test.py, conftest.py."""
+    p = PurePosixPath(path)
+    if any(part in _TEST_DIRS for part in p.parts[:-1]):
+        return True
+    n = p.name
+    return n == "conftest.py" or (n.endswith(".py") and (n.startswith("test_") or n.endswith("_test.py")))
+
+
+def is_doc_path(path: str) -> bool:
+    return bool(_DOC_RE.search(path))
+
+
+def is_ignored(rel: str) -> bool:
+    parts = PurePosixPath(rel).parts
+    return any(p in IGNORED_PARTS or p.endswith(".egg-info") for p in parts) or rel.endswith(".pyc")
+
+
+@dataclass
+class Task:
+    id: str
+    repo: str  # clone URL or local path
+    commit: str
+    parent: str
+    date: str  # committer date, ISO 8601
+    prompt: str
+    fail_to_pass: list[str]
+    pass_to_pass: list[str]
+    test_files: dict[str, str | None]  # test-side files at the fix commit (None = deleted)
+    gold_patch: str  # human fix (non-test changes), unified diff
+    image: str
+    env: dict = field(default_factory=dict)  # EnvSpec fields
+    source_files: list[str] = field(default_factory=list)
+    title_hint: str = ""  # short neutral label (never the commit message)
+
+    @property
+    def env_spec(self) -> EnvSpec:
+        return EnvSpec(**self.env) if self.env else EnvSpec()
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, d: dict) -> Task:
+        known = {f for f in cls.__dataclass_fields__}
+        return cls(**{k: v for k, v in d.items() if k in known})
+
+
+def save_tasks(path: Path, tasks: list[Task], meta: dict | None = None) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    data = {"format": TASKS_FORMAT, "meta": meta or {}, "tasks": [t.to_dict() for t in tasks]}
+    path.write_text(json.dumps(data, indent=1, ensure_ascii=False))
+
+
+def load_tasks(path: str | os.PathLike, ids: list[str] | None = None) -> list[Task]:
+    """Load mined tasks from a tasks.json written by ``commitarena mine``."""
+    p = Path(path)
+    if p.is_dir():
+        p = p / "tasks.json"
+    if not p.exists():
+        raise TaskError(f"Task file not found: {p}. Create one with: commitarena mine <repo>")
+    data = json.loads(p.read_text())
+    if data.get("format") != TASKS_FORMAT:
+        raise TaskError(f"{p} has unsupported format {data.get('format')!r} (expected {TASKS_FORMAT}).")
+    tasks = [Task.from_dict(t) for t in data["tasks"]]
+    if ids:
+        wanted = set(ids)
+        tasks = [t for t in tasks if t.id in wanted or any(t.id.endswith(i) for i in wanted)]
+        if not tasks:
+            raise TaskError(f"None of the task ids {sorted(wanted)} are in {p}.")
+    return tasks
+
+
+def load_tasks_meta(path: str | os.PathLike) -> dict:
+    p = Path(path)
+    if p.is_dir():
+        p = p / "tasks.json"
+    return json.loads(p.read_text()).get("meta", {})
+
+
+# ---------------------------------------------------------------- git helpers
+
+
+def git(repo: Path, *args: str, text: bool = True) -> str | bytes:
+    cp = subprocess.run(["git", "-C", str(repo), *args], capture_output=True)
+    if cp.returncode != 0:
+        raise TaskError(f"git {' '.join(args)} failed: {cp.stderr.decode(errors='replace').strip()}")
+    return cp.stdout.decode(errors="replace") if text else cp.stdout
+
+
+def extract_commit(repo: Path, rev: str, dest: Path) -> None:
+    data = git(repo, "archive", "--format=tar", rev, text=False)
+    dest.mkdir(parents=True, exist_ok=True)
+    with tarfile.open(fileobj=io.BytesIO(data)) as tar:
+        tar.extractall(dest, filter="data")
+
+
+def cache_dir() -> Path:
+    base = os.environ.get("COMMITARENA_CACHE") or os.path.join(
+        os.environ.get("XDG_CACHE_HOME", os.path.expanduser("~/.cache")), "commitarena"
+    )
+    return Path(base)
+
+
+def repo_slug(repo: str) -> str:
+    s = repo.rstrip("/").removesuffix(".git")
+    s = s.split(":")[-1] if s.startswith("git@") else s
+    parts = [p for p in re.split(r"[/\\]", s) if p][-2:]
+    return re.sub(r"[^a-z0-9]+", "-", "-".join(parts).lower()).strip("-") or "repo"
+
+
+def resolve_repo(repo: str) -> Path:
+    """Local path as-is; URLs are cloned (once) into the cache."""
+    local = Path(repo).expanduser()
+    if local.exists():
+        if not (local / ".git").exists() and not (local / "HEAD").exists():
+            raise TaskError(f"{local} is not a git repository.")
+        return local.resolve()
+    dest = cache_dir() / "repos" / repo_slug(repo)
+    if not (dest / ".git").exists():
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        cp = subprocess.run(["git", "clone", "-q", repo, str(dest)], capture_output=True)
+        if cp.returncode != 0:
+            raise TaskError(f"git clone {repo} failed: {cp.stderr.decode(errors='replace').strip()}")
+    return dest
+
+
+def write_files(root: Path, files: dict[str, str | None]) -> None:
+    for rel, content in files.items():
+        target = root / rel
+        if content is None:
+            if target.exists():
+                target.unlink()
+            continue
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(content)
+
+
+def init_workspace_git(root: Path) -> None:
+    """Fresh one-commit history so the agent can use git diff without seeing the real history."""
+    env = {**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_SYSTEM": os.devnull}
+    for args in (
+        ["init", "-q", "-b", "main"],
+        ["add", "-A"],
+        ["-c", "user.name=commitarena", "-c", "user.email=commitarena@localhost",
+         "commit", "-q", "--no-gpg-sign", "-m", "task start"],
+    ):  # fmt: skip
+        subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=True, env=env)
+
+
+def build_workspace(task: Task, dest: Path, with_git: bool = True) -> Path:
+    """Repo at the parent commit + the fix commit's test files. This is what the agent sees."""
+    repo = resolve_repo(task.repo)
+    extract_commit(repo, task.parent, dest)
+    write_files(dest, task.test_files)
+    if with_git:
+        init_workspace_git(dest)
+    return dest
+
+
+def tree_files(root: Path) -> dict[str, Path]:
+    out = {}
+    for p in root.rglob("*"):
+        if p.is_file() and not p.is_symlink():
+            rel = p.relative_to(root).as_posix()
+            if not is_ignored(rel):
+                out[rel] = p
+    return out
+
+
+def changed_files(before: Path, after: Path) -> dict[str, bytes | None]:
+    """Files that differ between two trees: rel path -> new bytes (None = deleted)."""
+    a, b = tree_files(before), tree_files(after)
+    out: dict[str, bytes | None] = {}
+    for rel in sorted(set(a) | set(b)):
+        if rel not in b:
+            out[rel] = None
+        elif rel not in a or a[rel].read_bytes() != b[rel].read_bytes():
+            out[rel] = b[rel].read_bytes()
+    return out
+
+
+def unified_diff(before: Path, after: Path, only: list[str] | None = None) -> str:
+    """git diff --no-index between two trees, restricted to `only` paths if given."""
+    import difflib
+
+    a, b = tree_files(before), tree_files(after)
+    paths = sorted(set(a) | set(b)) if only is None else sorted(only)
+    chunks: list[str] = []
+    for rel in paths:
+        try:
+            old = a[rel].read_text().splitlines(keepends=True) if rel in a else []
+            new = b[rel].read_text().splitlines(keepends=True) if rel in b else []
+        except UnicodeDecodeError:
+            chunks.append(f"Binary file {rel} differs\n")
+            continue
+        if old == new:
+            continue
+        diff = difflib.unified_diff(
+            old, new, f"a/{rel}" if rel in a else "/dev/null", f"b/{rel}" if rel in b else "/dev/null", n=3
+        )
+        text = "".join(line if line.endswith("\n") else line + "\n" for line in diff)
+        chunks.append(f"diff --git a/{rel} b/{rel}\n{text}")
+    return "".join(chunks)

commitarena/testrun.py

@@ -0,0 +1,77 @@
+"""Run pytest inside a sandbox and parse JUnit XML into per-test outcomes."""
+
+from __future__ import annotations
+
+import shlex
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass, field
+
+from .sandbox import Sandbox
+
+JUNIT_PATH = "/tmp/commitarena-junit.xml"
+
+PASSED, FAILED, SKIPPED = "passed", "failed", "skipped"
+
+
+@dataclass
+class TestOutcomes:
+    status: dict[str, str] = field(default_factory=dict)  # test id -> passed/failed/skipped
+    messages: dict[str, str] = field(default_factory=dict)  # failure text (truncated)
+    output: str = ""
+    ok: bool = True  # False if pytest crashed or produced no XML
+
+    def passed(self) -> set[str]:
+        return {t for t, s in self.status.items() if s == PASSED}
+
+    def failed(self) -> set[str]:
+        return {t for t, s in self.status.items() if s == FAILED}
+
+
+def pytest_command(paths: list[str] | None = None, extra: str = "") -> str:
+    target = " ".join(shlex.quote(p) for p in paths) if paths else ""
+    return (
+        f"rm -f {JUNIT_PATH}; python -m pytest -p no:cacheprovider -o addopts='' -q "
+        f"--continue-on-collection-errors --junitxml={JUNIT_PATH} {extra} {target}"
+    ).strip()
+
+
+def parse_junit(xml_text: str, max_msg: int = 1500) -> dict[str, tuple[str, str]]:
+    """Return {test_id: (status, message)}. Test id is ``classname::name``."""
+    out: dict[str, tuple[str, str]] = {}
+    root = ET.fromstring(xml_text)
+    for case in root.iter("testcase"):
+        cls = case.get("classname", "")
+        name = case.get("name", "")
+        tid = f"{cls}::{name}" if cls else name
+        status, msg = PASSED, ""
+        for child in case:
+            if child.tag in ("failure", "error"):
+                status = FAILED
+                msg = ((child.get("message") or "") + "\n" + (child.text or "")).strip()[:max_msg]
+                break
+            if child.tag == "skipped":
+                status = SKIPPED
+        # an error in teardown after a pass also counts as failed (handled by break above)
+        prev = out.get(tid)
+        if prev is None or status == FAILED:
+            out[tid] = (status, msg)
+    return out
+
+
+def run_tests(sb: Sandbox, timeout: int = 600, paths: list[str] | None = None) -> TestOutcomes:
+    res = sb.exec(pytest_command(paths), timeout=timeout)
+    xml = sb.exec(f"cat {JUNIT_PATH}", timeout=60)
+    to = TestOutcomes(output=res.output[-6000:])
+    if xml.exit_code != 0 or "<testsuite" not in xml.output:
+        to.ok = False
+        return to
+    try:
+        parsed = parse_junit(xml.output[xml.output.index("<") :])
+    except ET.ParseError:
+        to.ok = False
+        return to
+    for tid, (status, msg) in parsed.items():
+        to.status[tid] = status
+        if msg:
+            to.messages[tid] = msg
+    return to

commitarena-0.1.1.dist-info/METADATA

@@ -0,0 +1,242 @@
+Metadata-Version: 2.5
+Name: commitarena
+Version: 0.1.1
+Summary: Your git history is the benchmark. Replay real bug fixes against any model, in a sandbox with no network.
+Project-URL: Homepage, https://diegoperezg7.github.io/commitarena/
+Project-URL: Source, https://github.com/diegoperezg7/commitarena
+Project-URL: Issues, https://github.com/diegoperezg7/commitarena/issues
+Author: Diego Pérez
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agents,benchmark,docker,evaluation,git,llm,swe-bench
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: openai==3.42.1
+Requires-Dist: rich==15.0.0
+Requires-Dist: typer==0.27.3
+Description-Content-Type: text/markdown
+
+<p align="center"><img src="docs/assets/banner.svg" alt="commitarena" width="640"></p>
+
+<p align="center">
+<a href="https://github.com/diegoperezg7/commitarena/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/diegoperezg7/commitarena/actions/workflows/ci.yml/badge.svg"></a>
+<a href="LICENSE"><img alt="License: MIT" src="https://img.shields.io/badge/license-MIT-3b5bd4"></a>
+<a href="https://github.com/diegoperezg7/commitarena/releases"><img alt="Release" src="https://img.shields.io/github/v/release/diegoperezg7/commitarena?color=3b5bd4"></a>
+<a href="https://github.com/diegoperezg7/commitarena/stargazers"><img alt="Stars" src="https://img.shields.io/github/stars/diegoperezg7/commitarena?color=3b5bd4"></a>
+</p>
+
+<p align="center"><b>Your git history is the benchmark. Replay real bug fixes against any model, in a sandbox with no network.</b></p>
+
+<p align="center"><a href="README.es.md">Español</a> · <a href="https://diegoperezg7.github.io/commitarena/">Example report</a></p>
+
+<p align="center"><img src="docs/assets/demo.gif" alt="commitarena demo in a terminal" width="760"></p>
+
+commitarena mines commits from your repository where a bug fix came with a test, checks in Docker that the
+test fails without the fix and passes with it, and then asks each model to make that test pass. Every
+model gets the same small agent loop with one `bash` tool, and every command it runs executes in a
+container started with `--network none`. The report splits the pass rate into fixes committed before and
+after each model's training cutoff, so you can see whether a model solves the bug or remembers the patch.
+
+## Why
+
+There are good tools close to this. What I wanted and did not find in one place:
+
+| | What it does | Difference |
+|---|---|---|
+| [sjarmak/codeprobe](https://github.com/sjarmak/codeprobe) (10 stars) | Mines merged PRs, runs agent CLIs, ranks with cost and an HTML report. | Its README says each agent runs "inside a `--network=bridge` container (the agent needs the model API)". In commitarena the model is called from the host and the code the model writes never has a network. |
+| [PozziTiv4ik/Repo-Trials](https://github.com/PozziTiv4ik/Repo-Trials) (0 stars) | FAIL_TO_PASS tasks from your git history. | commitarena adds the same harness for every model, local ones included, and the before/after-cutoff split. |
+| [harbor-framework/harbor](https://github.com/harbor-framework/harbor) (6,755 stars) | A general framework for agent evaluations. | commitarena is one command for one question: which model fixes the bugs of this repo. |
+
+Star counts from the GitHub API on 2026-11-03.
+
+Three things commitarena does on purpose:
+
+1. **No network for model-written code.** The agent loop runs on your machine and only sends chat
+   requests. Each `bash` call goes through `docker exec` into a container with `--network none`,
+   `--cap-drop ALL`, `no-new-privileges`, 2 CPU, 2 GB, a PID limit and a per-command timeout.
+2. **One harness for every model.** Same system prompt, same single tool, same step and time limits,
+   whether it is a hosted model or a 4B model on Ollama. No vendor agent, no extra tools.
+3. **Solve or remember.** Public repos are in training data. The report shows pass rate on fixes dated
+   before each model's cutoff next to fixes dated after it. Cutoffs are configurable; by default the
+   model's release date is used, which is a safe upper bound.
+
+## Quickstart
+
+The demo needs no API keys and no Docker. It prints a recorded run and writes the HTML report.
+
+```bash
+uvx --from git+https://github.com/diegoperezg7/commitarena@v0.1.1 commitarena demo
+```
+
+Or from a clone:
+
+```bash
+git clone https://github.com/diegoperezg7/commitarena && cd commitarena && uv run commitarena demo
+```
+
+`uvx commitarena` is coming to PyPI.
+
+## Example output
+
+<!-- demo:start -->
+```text
+$ commitarena demo
+Recorded result (not a live run): 2026-11-03, commitarena 0.1.0, models mistral/codestral-latest,
+mistral/ministral-8b-latest, mistral/mistral-large-4, on Darwin arm64 (Apple M4, 16 GB RAM).
+Repo: https://github.com/andialbrecht/sqlparse (BSD-3-Clause)
+Leaderboard
+┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━┓
+┃ model                       ┃  pass ┃ rate ┃ before cutoff ┃ after cutoff ┃ cost $ ┃    tokens ┃ avg s ┃ avg steps ┃
+┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━┩
+│ mistral/mistral-large-4     │ 11/11 │ 100% │           8/8 │          3/3 │ 0.5401 │   672,848 │   118 │      11.1 │
+│ mistral/ministral-8b-latest │  1/11 │   9% │           0/5 │          1/6 │ 0.2445 │ 1,630,017 │    68 │      28.3 │
+│ mistral/codestral-latest    │  0/11 │   0% │           0/4 │          0/7 │ 0.6507 │ 2,038,536 │    75 │      27.8 │
+└─────────────────────────────┴───────┴──────┴───────────────┴──────────────┴────────┴───────────┴───────┴───────────┘
+```
+<!-- demo:end -->
+
+<p align="center"><a href="https://diegoperezg7.github.io/commitarena/"><img src="docs/assets/report-light.png" alt="report" width="760"></a></p>
+
+How to read it: with the same prompt, tool, limits and sandbox, Mistral Large 4 fixed all 11 bugs,
+including the 3 committed after its release (October 2026), so it was solving, not recalling. Ministral 8B
+fixed one, and that one is dated after its own cutoff. Codestral fixed none. Three tasks after a cutoff is
+a small sample: treat the split as a signal, not a verdict.
+
+**Why Codestral scores 0/11.** It is the model, not the scoring: it repeated an earlier command verbatim
+in 170 of its 304 tool calls, kept doing so after 163 "you already ran this exact command" notices from
+the harness, ran out of its 30 steps in 9 of 11 attempts, left the code untouched in 6, and never edited a
+test file. This is the start of the `PRIMARY KEY` task, whose human fix is one regex line in
+`sqlparse/keywords.py` (full trajectories are in [docs/demo-run/trajectories](docs/demo-run/trajectories)):
+
+```text
+12 grep -n 'PRIMARY' sqlparse/keywords.py
+13 sed -n '440,450p' sqlparse/keywords.py
+14 sed -n '440,450p' sqlparse/keywords.py      <- repeat notice
+15 grep -n 'KEY' sqlparse/keywords.py
+...
+23 grep -n 'PRIMARY' sqlparse/keywords.py      <- repeat notice
+24 sed -n '440,450p' sqlparse/keywords.py      <- repeat notice
+25 grep -n 'KEY' sqlparse/keywords.py          <- repeat notice
+```
+
+## Use it on your repo
+
+You need Docker running, plus Ollama or an API key.
+
+```bash
+# 1. Mine tasks: commits that change source and tests, validated in Docker
+commitarena mine https://github.com/you/yourrepo --max-tasks 10
+
+# 2. Run models (a local one and a hosted one)
+export MISTRAL_API_KEY=...
+commitarena run --tasks tasks/you-yourrepo/tasks.json \
+  --models ollama/qwen3:4b-16k,mistral/codestral-latest --max-cost 1
+
+# 3. The report is written to runs/<timestamp>/report.html; re-render it with other cutoffs
+commitarena report runs/<timestamp> --cutoff mistral/codestral-latest=2025-07-31 --open
+```
+
+## How it works
+
+```mermaid
+flowchart LR
+  subgraph host[Your machine]
+    G[git history] --> M[mine: commits touching src + tests]
+    M --> V{Docker, no network:<br/>new tests fail before,<br/>pass after?}
+    V -->|yes| T[tasks.json<br/>failing tests + test diff<br/>no commit message]
+    T --> L[agent loop<br/>one bash tool]
+    L <-->|chat API| P[(model provider<br/>or Ollama)]
+  end
+  L -->|docker exec| S[sandbox<br/>--network none<br/>2 CPU, 2 GB]
+  S --> R[copy out the workspace]
+  R --> X[keep non-test changes only,<br/>restore original tests]
+  X --> Y[fresh no-network container:<br/>full test suite]
+  Y --> Z[report.html<br/>leaderboard, matrix,<br/>cutoff split, diffs]
+```
+
+- **Mining.** Non-merge commits that change at least one test file and at least one `.py` source file
+  (small fixes only by default). For each one the test suite runs twice in a no-network container: the
+  parent commit plus the new test files, and the fix commit. A task is kept when some tests go from
+  failing to passing (FAIL_TO_PASS) and nothing that passed before breaks (PASS_TO_PASS). The test image
+  is built once per repo with your install command, which is the only step with network.
+- **The prompt** lists the failing tests, the test diff and the current failure output. The commit
+  message, the issue and the human fix are never shown. The workspace gets a fresh one-commit git history
+  so the real history cannot leak the fix.
+- **Scoring.** After the attempt the workspace is copied out. Only changes to non-test files are applied
+  onto a pristine copy, so edits to tests or `conftest.py` are discarded, and the full suite runs in a
+  fresh container. Pass means every FAIL_TO_PASS test passes and no PASS_TO_PASS test fails.
+- **Metrics.** Pass, cost (provider-reported for OpenRouter, otherwise tokens times list price; local
+  models cost 0), input and output tokens, wall time and steps (model calls).
+
+## Configuration
+
+| Flag | Default | |
+|---|---|---|
+| `run --models` | | Comma-separated `provider/model`. Providers: `openrouter`, `openai`, `mistral`, `ollama`. |
+| `run --max-cost` | `1.0` | USD. Checked before every model call; the run stops with partial results. |
+| `run --max-steps` / `--max-time` | `30` / `1200` | Model calls and seconds per attempt. |
+| `run --cutoff MODEL=DATE` | release date | Training cutoff used for the before/after split (repeatable). |
+| `run --price MODEL=IN,OUT` | built-in | USD per million tokens for models without a known price. |
+| `run --context-file F` | | Inject an AGENTS.md / CLAUDE.md: `--context-mode both\|repo\|prompt`. |
+| `run --strip-context-files` | off | Remove the repo's own AGENTS.md / CLAUDE.md before the attempt. |
+| `run --cpus` / `--memory` / `--command-timeout` | `2` / `2g` / `120` | Sandbox limits. |
+| `mine --since` / `--until` / `--scan` / `--max-tasks` | | Which commits to consider. |
+| `mine --python` / `--setup` | `3.12` / `pip install -e .` | How the test image is built. |
+
+Environment variables: `OPENROUTER_API_KEY`, `OPENAI_API_KEY`, `MISTRAL_API_KEY`, `OLLAMA_HOST`,
+`COMMITARENA_CACHE` (clones, default `~/.cache/commitarena`). See [.env.example](.env.example). There is
+no telemetry.
+
+**Ollama context window.** Ollama's OpenAI-compatible endpoint ignores `num_ctx`, and its default
+window can be 4096 tokens, which silently drops the task text and gives worse results. commitarena warns
+about it. Use a variant with `num_ctx 16384` (the one used in development is `qwen3:4b-16k`); the agent
+loop elides older tool outputs to stay inside 16K.
+
+```bash
+printf 'FROM qwen3:4b\nPARAMETER num_ctx 16384\n' > Modelfile && ollama create qwen3:4b-16k -f Modelfile
+commitarena run --tasks tasks.json --models ollama/qwen3:4b-16k
+```
+
+Local models go through exactly the same code path (qwen3:4b-16k was run during development), but it is not
+in the recorded demo. On this machine it spent every 4096-token reply reasoning and never called the
+tool, and its thinking could not be turned off through the OpenAI endpoint (`think: false`,
+`reasoning_effort: "none"` and `/no_think` were all tried). Raising the reply limit for one model would
+break the same-harness rule, so it was left out rather than shown with a misleading 0.
+
+## Python API
+
+```python
+from commitarena import load_tasks, run_task, Budget, ContextFile
+
+task = load_tasks("tasks.json")[0]
+res = run_task(task, "ollama/qwen3:4b-16k", budget=Budget(max_cost=1.0),
+               context=ContextFile(open("AGENTS.md").read(), in_repo=True, in_prompt=True))
+print(res.passed, res.cost_usd, res.tokens_in, res.steps, res.wall_time_s)
+```
+
+## Demo data
+
+The recorded demo uses 11 tasks mined from [sqlparse](https://github.com/andialbrecht/sqlparse)
+(BSD-3-Clause, Copyright (c) 2016, Andi Albrecht). The bundled files contain the failing tests, the test
+diffs and the upstream fixes as diffs, each linked to its commit. The upstream license is bundled verbatim in
+[src/commitarena/demo/LICENSE-sqlparse](src/commitarena/demo/LICENSE-sqlparse).
+
+## Roadmap
+
+- Other languages and test runners
+- Claude Code and Codex as contestants
+- GitHub issue text as an optional prompt
+- pass@k and repeated attempts
+- Parallel runs
+- Export to Harbor
+
+## License
+
+MIT. See [LICENSE](LICENSE).

commitarena-0.1.1.dist-info/RECORD

@@ -0,0 +1,22 @@
+commitarena/__init__.py,sha256=ugkyzT5UoXywcg8nmA7kEyXvu_tZjxquz8nhCnt9b70,994
+commitarena/__main__.py,sha256=bYt9eEaoRQWdejEHFD8REx9jxVEdZptECFsV7F49Ink,30
+commitarena/agent.py,sha256=Jea4mQ1H5TGl0tUbKYsr6fmNMhHAbaA-L6NXzYI9-Ms,7993
+commitarena/cli.py,sha256=IOkyFxoE3USefvsKLdgBtaLKyIzB8cqbWEC-XVKjM8c,13235
+commitarena/errors.py,sha256=vp5lOo83HKl60GFcY514ntNljROpSfRCNTkdyEsNDEk,1239
+commitarena/llm.py,sha256=0SMln61wmsvy5J96qpOaHeUB87ZIqrnQLi6IhHynxXc,8071
+commitarena/mine.py,sha256=G-wdQEpo85W1I5ZOlYx1iRr3nOdGenFWXp6Fzszk9lY,9257
+commitarena/report.py,sha256=Z_pQ2wcRvjDFY343KzHpLAMHPug_iQqXUvssC8PYVWc,5459
+commitarena/report_template.html,sha256=3k7RsciKwoKy87KmA14LB3Spmqc-mTQl8sdICt0dv6c,27837
+commitarena/runner.py,sha256=9IBKt3ylkn0ImbpzTU3eEMJ74OXDOobFpiFdDl4_k7A,11323
+commitarena/sandbox.py,sha256=X90jX7KGEAUzy4tBC4V6Qs2h23zE-WP-BJM3BAO-mCg,6778
+commitarena/tasks.py,sha256=rQYHi6zDYsPIPywhU68HhiqtXzwWKhdH4AS5b8_Gzio,8438
+commitarena/testrun.py,sha256=YK7FiAQU6anB_N1IPSMdNGZu-Ah4w9ZvNeCd12PWSg0,2776
+commitarena/demo/LICENSE-sqlparse,sha256=wZOCNbgNOekxOOrontw69n4Y7LxA0mZSn6V7Lc5CYxA,1537
+commitarena/demo/results.jsonl,sha256=dLsfxwd0d6S9r8EDhjxe_1o4ieWGdqCBDiYIvtiJ-Dw,96821
+commitarena/demo/run.json,sha256=51N2M9SSVkqgpcUFWmyBGTk5hAKeVr4f4ItiiGU8Lcc,1645
+commitarena/demo/tasks.json,sha256=mQ0HRjUNogAHZda8umcFXANWp3MkM7O-z4w-nukM5AE,609679
+commitarena-0.1.1.dist-info/METADATA,sha256=0QnCWqHap67CQD0olCFoP4DRNadLNIdE8qb1Bw9C1v4,14085
+commitarena-0.1.1.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+commitarena-0.1.1.dist-info/entry_points.txt,sha256=uZVCdVojRaQjNHgSzdDQ7_6IHTidFxlDHytV1u63Fqo,53
+commitarena-0.1.1.dist-info/licenses/LICENSE,sha256=A7Jh3ub2oGhc2l6P9raz5AeuQKzRIinCPYAXENE2Lig,1069
+commitarena-0.1.1.dist-info/RECORD,,

commitarena-0.1.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

commitarena-0.1.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+commitarena = commitarena.cli:main

commitarena-0.1.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Diego Pérez
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.