repo-bug-hunter 0.2.0__tar.gz

repo_bug_hunter-0.2.0/.gitignore

@@ -0,0 +1,19 @@
+.venv/
+__pycache__/
+*.egg-info/
+.pytest_cache/
+dist/
+.dist/
+# SWE-bench harness logs: large, and re-read by `repo-bug-hunter evaluate --collect-only`
+logs/
+# Your runs stay local (the GitHub Actions workflow keeps its own on the repo-bug-hunter-results branch).
+# The pilot runs are kept as worked examples (LEARNING_GUIDE.md, IMPLEMENTATION.md).
+runs/*
+!runs/local_pilot/
+!runs/local_pilot_tf/
+!runs/openrouter_pilot/
+!runs/free_pilot/
+/site/
+# Secrets never belong in the repository
+.env
+*.key

repo_bug_hunter-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Manish Maurya
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

repo_bug_hunter-0.2.0/PKG-INFO

@@ -0,0 +1,150 @@
+Metadata-Version: 2.5
+Name: repo-bug-hunter
+Version: 0.2.0
+Summary: An AI coding agent that fixes real GitHub bugs, and a lab that measures it: graded by hidden tests, every run replayable.
+Project-URL: Homepage, https://github.com/Manishmaurya89/repo-bug-hunter
+Project-URL: Documentation, https://github.com/Manishmaurya89/repo-bug-hunter/blob/main/IMPLEMENTATION.md
+Project-URL: Issues, https://github.com/Manishmaurya89/repo-bug-hunter/issues
+Author: Manish Maurya
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai-agents,benchmark,coding-agent,evaluation,llm,swe-bench
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: anthropic>=1.0
+Requires-Dist: datasets>=2.18
+Requires-Dist: openai>=3.22.0
+Requires-Dist: swebench>=4.0
+Description-Content-Type: text/markdown
+
+# repo-bug-hunter
+
+[![PyPI](https://img.shields.io/pypi/v/repo-bug-hunter)](https://pypi.org/project/repo-bug-hunter/)
+[![tests](https://github.com/Manishmaurya89/repo-bug-hunter/actions/workflows/tests.yml/badge.svg)](https://github.com/Manishmaurya89/repo-bug-hunter/actions/workflows/tests.yml)
+[![license: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/Manishmaurya89/repo-bug-hunter/blob/main/LICENSE)
+[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/Manishmaurya89/repo-bug-hunter)
+
+An AI coding agent that fixes real GitHub bugs, and a lab that measures how well it does it.
+
+Give it a GitHub issue and it works the way a developer would: it explores the repository, reproduces the bug, edits the code and runs the tests until it can submit a fix. The fix is then graded by the project's own hidden tests from [SWE-bench Verified](https://www.swebench.com/), and every step the agent took can be replayed in the browser.
+
+## What it does
+
+- Fixes bugs from real GitHub issues inside a Docker sandbox, using six simple tools.
+- Grades each fix with the tests the project's maintainers wrote for the real fix.
+- Replays every run step by step on a static web page.
+- Compares two versions of the agent on the same tasks, with paired statistics.
+- Builds new tasks from merged GitHub pull requests, to test on bugs newer than the model.
+- Works with Claude, any OpenRouter model, local models in Ollama, or any OpenAI-compatible server.
+
+The agent is deliberately small: one file of under 200 lines, a plain loop around a chat model, with no agent framework, so every prompt and tool call is easy to follow. The sandbox is the same Docker image the grader uses and has no network, so the agent can't install packages or look up the real fix.
+
+## Quick start
+
+You need Python 3.10+ and Docker, with about 30 GB of free disk space, since each task's Docker image is a few GB. On Apple Silicon, turn on Rosetta in Docker Desktop's settings; the images are x86-64.
+
+```bash
+pip install repo-bug-hunter          # or: uv tool install repo-bug-hunter
+export OPENROUTER_API_KEY=...        # from openrouter.ai/settings/keys; see Models for Claude
+repo-bug-hunter doctor               # checks Docker, disk, the model and the key
+repo-bug-hunter smoke                # fixes a toy bug end to end, in a few minutes
+```
+
+Then run it on real bugs. Results go to `runs/` in the current folder:
+
+```bash
+repo-bug-hunter run --name pilot --n 5 --difficulty "<15 min fix"
+repo-bug-hunter evaluate runs/pilot     # grade with the official SWE-bench harness
+repo-bug-hunter analyze runs/pilot      # resolve rate, cost, steps, why tasks failed
+repo-bug-hunter viewer runs/pilot       # build the replay site in ./site
+python3 -m http.server -d site 8000     # open http://localhost:8000
+```
+
+### Without installing anything
+
+- **GitHub Actions:** fork this repository and enable workflows in the fork's **Actions** tab. Add `OPENROUTER_API_KEY` under **Settings → Secrets and variables → Actions**, and set **Settings → Pages → Source** to **GitHub Actions**. Then run the **experiment** workflow. Each task runs on its own GitHub machine, the official grader scores it, and the replay site is published to your GitHub Pages.
+- **Codespaces:** click the badge at the top for a ready-made environment in the browser, then run the commands above with `uv run` in front, such as `uv run repo-bug-hunter doctor`.
+
+## The test-first experiment
+
+The question this project was built to answer: does an agent fix more bugs if it must first reproduce the bug with a failing test?
+
+In `test_first` mode, the harness enforces it, not the prompt:
+
+1. Source files are locked until the agent registers a test command that fails on the current code.
+2. When the agent submits, the harness runs that test again. If it still fails, the submission is sent back once.
+3. So the agent can't get stuck: after three tests that don't fail, the source unlocks anyway, and a second submit is always accepted.
+
+Run both modes on the same tasks and compare:
+
+```bash
+repo-bug-hunter run --name baseline   --variant baseline   --n 50
+repo-bug-hunter run --name test_first --variant test_first --n 50
+repo-bug-hunter evaluate runs/baseline && repo-bug-hunter evaluate runs/test_first
+repo-bug-hunter analyze runs/baseline runs/test_first --out results.md
+```
+
+The report pairs the two runs task by task, using McNemar's exact test and a bootstrap confidence interval for the difference, so noise from a small sample isn't mistaken for an improvement.
+
+## How it works
+
+- [agent.py](https://github.com/Manishmaurya89/repo-bug-hunter/blob/main/repo_bug_hunter/agent.py) is the loop. It sends the conversation to the model, runs the tools the model asks for, and repeats until the agent submits or hits a step or cost limit. With Claude it uses prompt caching and adaptive thinking.
+- [tools.py](https://github.com/Manishmaurya89/repo-bug-hunter/blob/main/repo_bug_hunter/tools.py) has the tools (`read_file`, `search`, `edit_file`, `write_file`, `bash`, `submit`), the test-first gate, and the code that turns the final repository into a patch.
+- [env.py](https://github.com/Manishmaurya89/repo-bug-hunter/blob/main/repo_bug_hunter/env.py) runs everything in the task's own SWE-bench Docker image, without network access, and caps output, file reads and processes so a runaway command can't hurt the machine.
+- [evaluate.py](https://github.com/Manishmaurya89/repo-bug-hunter/blob/main/repo_bug_hunter/evaluate.py) grades with the official harness. A bug counts as fixed only if the hidden tests for the real fix pass and nothing that passed before breaks.
+- [analyze.py](https://github.com/Manishmaurya89/repo-bug-hunter/blob/main/repo_bug_hunter/analyze.py) reports the resolve rate, cost and steps, and gives every failure one reason: gave up, wrong file, broke other tests, and so on.
+- [viewer.py](https://github.com/Manishmaurya89/repo-bug-hunter/blob/main/repo_bug_hunter/viewer.py) builds the replay site, and [pr.py](https://github.com/Manishmaurya89/repo-bug-hunter/blob/main/repo_bug_hunter/pr.py) builds tasks from merged GitHub pull requests:
+
+```bash
+repo-bug-hunter pr more-itertools/more-itertools#1305 --name fresh
+```
+
+## Models
+
+| Provider | How to choose it | API key |
+|---|---|---|
+| Anthropic | `--model claude-sonnet-5-5` | `ANTHROPIC_API_KEY` |
+| OpenRouter | `--model vendor/model` | `OPENROUTER_API_KEY` |
+| Ollama (local) | `--provider ollama --model NAME` | none |
+| Other OpenAI-compatible servers | `--base-url URL --model NAME` | `LLM_API_KEY` |
+
+Without `--model`, a free OpenRouter model is used, so you can try everything without paying. `repo-bug-hunter doctor` lists the free models that currently support tool calling. Free models allow about 50 requests a day, roughly two tasks, or 1,000 a day after buying $10 of OpenRouter credits once. When a limit runs out, the run stops cleanly, and running the same command later continues where it stopped. Free providers may log prompts, so don't point them at private code.
+
+Local models need tool calling and a context window of at least 32k tokens. Ollama's default is 4k, so create a variant first:
+
+```bash
+printf 'FROM gemma4\nPARAMETER num_ctx 32768\n' > Modelfile && ollama create gemma4-32k -f Modelfile
+```
+
+## Limitations
+
+- SWE-bench Verified is public, so strong models may have seen some of the real fixes; OpenAI stopped reporting it in February 2026 for this reason. Paired comparisons are less affected, and `repo-bug-hunter pr` tests on fresh bugs.
+- `repo-bug-hunter pr` supports Python projects tested with pytest, and the pull request must name the issue it fixes.
+- Failure reasons are heuristics. For example, "wrong file" compares the patch with the files the real fix changed, so a correct fix in another file would be miscounted.
+
+## Development
+
+```bash
+git clone https://github.com/Manishmaurya89/repo-bug-hunter && cd repo-bug-hunter
+uv sync
+uv run pytest                        # the Docker tests run only when Docker is up
+```
+
+- [IMPLEMENTATION.md](https://github.com/Manishmaurya89/repo-bug-hunter/blob/main/IMPLEMENTATION.md): how each part is built and why, including the bugs found along the way.
+- [LEARNING_GUIDE.md](https://github.com/Manishmaurya89/repo-bug-hunter/blob/main/LEARNING_GUIDE.md): every idea explained from zero, with exercises.
+
+## License
+
+[MIT](https://github.com/Manishmaurya89/repo-bug-hunter/blob/main/LICENSE)

repo_bug_hunter-0.2.0/README.md

@@ -0,0 +1,127 @@
+# repo-bug-hunter
+
+[![PyPI](https://img.shields.io/pypi/v/repo-bug-hunter)](https://pypi.org/project/repo-bug-hunter/)
+[![tests](https://github.com/Manishmaurya89/repo-bug-hunter/actions/workflows/tests.yml/badge.svg)](https://github.com/Manishmaurya89/repo-bug-hunter/actions/workflows/tests.yml)
+[![license: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/Manishmaurya89/repo-bug-hunter)
+
+An AI coding agent that fixes real GitHub bugs, and a lab that measures how well it does it.
+
+Give it a GitHub issue and it works the way a developer would: it explores the repository, reproduces the bug, edits the code and runs the tests until it can submit a fix. The fix is then graded by the project's own hidden tests from [SWE-bench Verified](https://www.swebench.com/), and every step the agent took can be replayed in the browser.
+
+```mermaid
+flowchart LR
+    issue["GitHub issue"] --> agent["Agent loop in a Docker sandbox<br/>read · search · edit · run tests"]
+    agent --> patch["Patch"]
+    patch --> grader["Hidden tests<br/>official SWE-bench grader"]
+    grader --> report["Resolved or not<br/>cost · steps · replay"]
+```
+
+## What it does
+
+- Fixes bugs from real GitHub issues inside a Docker sandbox, using six simple tools.
+- Grades each fix with the tests the project's maintainers wrote for the real fix.
+- Replays every run step by step on a static web page.
+- Compares two versions of the agent on the same tasks, with paired statistics.
+- Builds new tasks from merged GitHub pull requests, to test on bugs newer than the model.
+- Works with Claude, any OpenRouter model, local models in Ollama, or any OpenAI-compatible server.
+
+The agent is deliberately small: one file of under 200 lines, a plain loop around a chat model, with no agent framework, so every prompt and tool call is easy to follow. The sandbox is the same Docker image the grader uses and has no network, so the agent can't install packages or look up the real fix.
+
+## Quick start
+
+You need Python 3.10+ and Docker, with about 30 GB of free disk space, since each task's Docker image is a few GB. On Apple Silicon, turn on Rosetta in Docker Desktop's settings; the images are x86-64.
+
+```bash
+pip install repo-bug-hunter          # or: uv tool install repo-bug-hunter
+export OPENROUTER_API_KEY=...        # from openrouter.ai/settings/keys; see Models for Claude
+repo-bug-hunter doctor               # checks Docker, disk, the model and the key
+repo-bug-hunter smoke                # fixes a toy bug end to end, in a few minutes
+```
+
+Then run it on real bugs. Results go to `runs/` in the current folder:
+
+```bash
+repo-bug-hunter run --name pilot --n 5 --difficulty "<15 min fix"
+repo-bug-hunter evaluate runs/pilot     # grade with the official SWE-bench harness
+repo-bug-hunter analyze runs/pilot      # resolve rate, cost, steps, why tasks failed
+repo-bug-hunter viewer runs/pilot       # build the replay site in ./site
+python3 -m http.server -d site 8000     # open http://localhost:8000
+```
+
+### Without installing anything
+
+- **GitHub Actions:** fork this repository and enable workflows in the fork's **Actions** tab. Add `OPENROUTER_API_KEY` under **Settings → Secrets and variables → Actions**, and set **Settings → Pages → Source** to **GitHub Actions**. Then run the **experiment** workflow. Each task runs on its own GitHub machine, the official grader scores it, and the replay site is published to your GitHub Pages.
+- **Codespaces:** click the badge at the top for a ready-made environment in the browser, then run the commands above with `uv run` in front, such as `uv run repo-bug-hunter doctor`.
+
+## The test-first experiment
+
+The question this project was built to answer: does an agent fix more bugs if it must first reproduce the bug with a failing test?
+
+In `test_first` mode, the harness enforces it, not the prompt:
+
+1. Source files are locked until the agent registers a test command that fails on the current code.
+2. When the agent submits, the harness runs that test again. If it still fails, the submission is sent back once.
+3. So the agent can't get stuck: after three tests that don't fail, the source unlocks anyway, and a second submit is always accepted.
+
+Run both modes on the same tasks and compare:
+
+```bash
+repo-bug-hunter run --name baseline   --variant baseline   --n 50
+repo-bug-hunter run --name test_first --variant test_first --n 50
+repo-bug-hunter evaluate runs/baseline && repo-bug-hunter evaluate runs/test_first
+repo-bug-hunter analyze runs/baseline runs/test_first --out results.md
+```
+
+The report pairs the two runs task by task, using McNemar's exact test and a bootstrap confidence interval for the difference, so noise from a small sample isn't mistaken for an improvement.
+
+## How it works
+
+- [agent.py](repo_bug_hunter/agent.py) is the loop. It sends the conversation to the model, runs the tools the model asks for, and repeats until the agent submits or hits a step or cost limit. With Claude it uses prompt caching and adaptive thinking.
+- [tools.py](repo_bug_hunter/tools.py) has the tools (`read_file`, `search`, `edit_file`, `write_file`, `bash`, `submit`), the test-first gate, and the code that turns the final repository into a patch.
+- [env.py](repo_bug_hunter/env.py) runs everything in the task's own SWE-bench Docker image, without network access, and caps output, file reads and processes so a runaway command can't hurt the machine.
+- [evaluate.py](repo_bug_hunter/evaluate.py) grades with the official harness. A bug counts as fixed only if the hidden tests for the real fix pass and nothing that passed before breaks.
+- [analyze.py](repo_bug_hunter/analyze.py) reports the resolve rate, cost and steps, and gives every failure one reason: gave up, wrong file, broke other tests, and so on.
+- [viewer.py](repo_bug_hunter/viewer.py) builds the replay site, and [pr.py](repo_bug_hunter/pr.py) builds tasks from merged GitHub pull requests:
+
+```bash
+repo-bug-hunter pr more-itertools/more-itertools#1305 --name fresh
+```
+
+## Models
+
+| Provider | How to choose it | API key |
+|---|---|---|
+| Anthropic | `--model claude-sonnet-5-5` | `ANTHROPIC_API_KEY` |
+| OpenRouter | `--model vendor/model` | `OPENROUTER_API_KEY` |
+| Ollama (local) | `--provider ollama --model NAME` | none |
+| Other OpenAI-compatible servers | `--base-url URL --model NAME` | `LLM_API_KEY` |
+
+Without `--model`, a free OpenRouter model is used, so you can try everything without paying. `repo-bug-hunter doctor` lists the free models that currently support tool calling. Free models allow about 50 requests a day, roughly two tasks, or 1,000 a day after buying $10 of OpenRouter credits once. When a limit runs out, the run stops cleanly, and running the same command later continues where it stopped. Free providers may log prompts, so don't point them at private code.
+
+Local models need tool calling and a context window of at least 32k tokens. Ollama's default is 4k, so create a variant first:
+
+```bash
+printf 'FROM gemma4\nPARAMETER num_ctx 32768\n' > Modelfile && ollama create gemma4-32k -f Modelfile
+```
+
+## Limitations
+
+- SWE-bench Verified is public, so strong models may have seen some of the real fixes; OpenAI stopped reporting it in February 2026 for this reason. Paired comparisons are less affected, and `repo-bug-hunter pr` tests on fresh bugs.
+- `repo-bug-hunter pr` supports Python projects tested with pytest, and the pull request must name the issue it fixes.
+- Failure reasons are heuristics. For example, "wrong file" compares the patch with the files the real fix changed, so a correct fix in another file would be miscounted.
+
+## Development
+
+```bash
+git clone https://github.com/Manishmaurya89/repo-bug-hunter && cd repo-bug-hunter
+uv sync
+uv run pytest                        # the Docker tests run only when Docker is up
+```
+
+- [IMPLEMENTATION.md](IMPLEMENTATION.md): how each part is built and why, including the bugs found along the way.
+- [LEARNING_GUIDE.md](LEARNING_GUIDE.md): every idea explained from zero, with exercises.
+
+## License
+
+[MIT](LICENSE)

repo_bug_hunter-0.2.0/pyproject.toml

@@ -0,0 +1,71 @@
+[project]
+name = "repo-bug-hunter"
+version = "0.2.0"
+description = "An AI coding agent that fixes real GitHub bugs, and a lab that measures it: graded by hidden tests, every run replayable."
+dynamic = ["readme"]
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Manish Maurya" }]
+requires-python = ">=3.10"
+keywords = ["ai-agents", "coding-agent", "llm", "swe-bench", "evaluation", "benchmark"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Operating System :: MacOS",
+    "Operating System :: POSIX :: Linux",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Quality Assurance",
+    "Topic :: Software Development :: Testing",
+]
+dependencies = [
+    "anthropic>=1.0",
+    "datasets>=2.18",
+    "openai>=3.22.0",
+    "swebench>=4.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/Manishmaurya89/repo-bug-hunter"
+Documentation = "https://github.com/Manishmaurya89/repo-bug-hunter/blob/main/IMPLEMENTATION.md"
+Issues = "https://github.com/Manishmaurya89/repo-bug-hunter/issues"
+
+[project.scripts]
+repo-bug-hunter = "repo_bug_hunter.cli:main"
+
+[dependency-groups]
+dev = ["pytest>=8"]
+
+[build-system]
+requires = ["hatchling", "hatch-fancy-pypi-readme"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["repo_bug_hunter"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/repo_bug_hunter", "/tests", "/README.md"]
+
+# PyPI shows the README without the Mermaid diagram, which only GitHub draws, and with
+# repository links made absolute: relative ones would point into pypi.org.
+[tool.hatch.metadata.hooks.fancy-pypi-readme]
+content-type = "text/markdown"
+
+[[tool.hatch.metadata.hooks.fancy-pypi-readme.fragments]]
+path = "README.md"
+
+[[tool.hatch.metadata.hooks.fancy-pypi-readme.substitutions]]
+pattern = '(?s)```mermaid\n.*?```\n\n'
+replacement = ''
+
+[[tool.hatch.metadata.hooks.fancy-pypi-readme.substitutions]]
+pattern = '\]\((?!https?://|#)([^)]+)\)'
+replacement = '](https://github.com/Manishmaurya89/repo-bug-hunter/blob/main/\1)'
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

repo_bug_hunter-0.2.0/repo_bug_hunter/agent.py

@@ -0,0 +1,176 @@
+"""The agent loop: ask Claude for tool calls, run them, repeat until submit or a limit."""
+
+from __future__ import annotations
+
+import time
+from typing import Callable
+
+import anthropic
+
+from .tools import Toolbox
+
+# USD per million tokens: input, output, 5-minute cache write, cache read.
+# List prices as of September 2026; check https://claude.com/pricing before quoting costs.
+PRICES = {
+    "claude-opus-5-5": (4.00, 20.00, 5.00, 0.20),
+    "claude-sonnet-5-5": (2.00, 10.00, 2.50, 0.20),
+    "claude-haiku-4-5": (1.00, 5.00, 1.25, 0.10),
+    "claude-fable-5-1": (10.00, 50.00, 12.50, 0.25),
+    # Possible refusal-fallback targets, so a fallback turn is still priced.
+    "claude-opus-5": (5.00, 25.00, 6.25, 0.50),
+    "claude-opus-4-8": (5.00, 25.00, 6.25, 0.50),
+    "claude-sonnet-5": (2.00, 10.00, 2.50, 0.20),
+}
+# Models that take `fallbacks: "default"` (re-run a classifier refusal on another model).
+FALLBACK_MODELS = {"claude-fable-5-1", "claude-opus-5-5", "claude-opus-5", "claude-sonnet-5-5"}
+
+SYSTEM = """\
+You are an autonomous software engineer fixing an issue in a Python repository. The repository \
+is checked out at {workdir} with its dependencies installed. There is no network access, so tests \
+that need the internet fail whatever you change.
+
+Resolve the issue by changing the library's source code. Hidden tests written by the project's \
+maintainers will check your change, so fix the underlying behavior the issue describes rather \
+than special-casing its example. Changes to existing test files are discarded before grading.
+
+Use the tools to explore the code, edit files, and run code and tests. Prefer edit_file and \
+write_file over editing files with shell commands. Nobody will answer questions, so keep working \
+until the fix is complete, then call submit."""
+
+TEST_FIRST = """
+
+Work test-first. Before changing any source file, write a test that reproduces the issue and \
+fails on the current code. A standalone script such as {workdir}/repro_test.py that exits \
+non-zero while the bug is present works for any project. Register it with record_failing_test, \
+which runs it and checks that it fails; source files stay locked until then. When you call \
+submit, the harness runs it again and it must pass."""
+
+
+def system_prompt(test_first: bool, workdir: str = "/testbed") -> str:
+    # The path must be where the repository really is: told /testbed while working elsewhere,
+    # a model goes looking for it across the whole machine.
+    return (SYSTEM + (TEST_FIRST if test_first else "")).format(workdir=workdir)
+
+
+def task_prompt(task: dict) -> str:
+    return f"<issue>\n{task['problem_statement'].strip()}\n</issue>\n\nRepository: {task['repo']}"
+
+
+def cost_of(model: str, usage) -> float:
+    # A provider that reports what a request cost (OpenRouter does) is taken at its word.
+    reported = getattr(usage, "cost_usd", None)
+    if reported is not None:
+        return float(reported)
+    # Models not in the table (local models, free tiers) are counted as free.
+    p_in, p_out, p_write, p_read = PRICES.get(model, (0, 0, 0, 0))
+    get = lambda f: getattr(usage, f, 0) or 0
+    return (get("input_tokens") * p_in + get("output_tokens") * p_out
+            + get("cache_creation_input_tokens") * p_write
+            + get("cache_read_input_tokens") * p_read) / 1e6
+
+
+def claude(model: str, effort: str, system: str, tools: list[dict]) -> Callable[[list], object]:
+    """Returns query(messages) -> Message, with caching and fallbacks configured."""
+    client = anthropic.Anthropic(max_retries=8)
+    params = dict(model=model, max_tokens=64_000, system=system, tools=tools,
+                  cache_control={"type": "ephemeral"})  # moves to the end of the history each turn
+    betas = []
+    if not model.startswith("claude-haiku"):  # Haiku 4.5 predates adaptive thinking and effort
+        params["thinking"] = {"type": "adaptive", "display": "summarized"}
+        params["output_config"] = {"effort": effort}
+    if model in FALLBACK_MODELS:
+        # A classifier refusal is re-run on another model. Steps record the serving model,
+        # so a run that fell back is visible in the results.
+        params["fallbacks"] = "default"
+        betas.append("server-side-fallback-2026-07-01")
+
+    def query(messages: list) -> object:
+        with client.beta.messages.stream(messages=messages, betas=betas, **params) as stream:
+            return stream.get_final_message()
+
+    return query
+
+
+def run_agent(task: dict, env, query: Callable[[list], object], *, model: str,
+              test_first: bool = False, max_steps: int = 50, max_cost: float = 2.0,
+              log: Callable[[str], None] = lambda s: None) -> dict:
+    tools = Toolbox(env, test_first=test_first)
+    messages: list = [{"role": "user", "content": task_prompt(task)}]
+    traj = {"instance_id": task["instance_id"], "variant": "test_first" if test_first else "baseline",
+            "model": model, "problem_statement": task["problem_statement"], "steps": [],
+            "exit_status": "step_limit", "cost": 0.0}
+    started = time.time()
+
+    for i in range(1, max_steps + 1):
+        try:
+            resp = query(messages)
+        except Exception as e:  # after the client's own retries; keep the trajectory so far
+            traj["exit_status"], traj["error"] = "api_error", f"{type(e).__name__}: {e}"
+            break
+        served_by = resp.model if resp.model in PRICES else model
+        step = {"i": i, "model": resp.model, "stop_reason": resp.stop_reason,
+                "cost": cost_of(served_by, resp.usage), "thinking": [], "text": [], "actions": [],
+                "usage": {k: getattr(resp.usage, k, 0) or 0 for k in (
+                    "input_tokens", "output_tokens", "cache_creation_input_tokens",
+                    "cache_read_input_tokens")}}
+        traj["steps"].append(step)
+        traj["cost"] += step["cost"]
+        # Append the content unchanged: thinking blocks must be passed back as-is.
+        messages.append({"role": "assistant", "content": resp.content})
+
+        for b in resp.content:
+            if b.type == "thinking" and b.thinking:
+                step["thinking"].append(b.thinking)
+            elif b.type == "text" and b.text.strip():
+                step["text"].append(b.text)
+        if resp.stop_reason == "refusal":
+            details = getattr(resp, "stop_details", None)
+            traj["exit_status"], traj["error"] = "refusal", str(getattr(details, "category", None))
+            break
+
+        calls = [b for b in resp.content if b.type == "tool_use"]
+        if not calls:
+            messages.append({"role": "user", "content": "No tool was called, so nothing ran. Writing a "
+                             "call out as text does not run it: use the tools to keep working, and call "
+                             "submit when the fix is complete."})
+            log(f"step {i} ${traj['cost']:.2f} (no tool call)")
+            continue
+
+        results = []
+        for b in calls:
+            t0 = time.time()
+            if resp.stop_reason == "max_tokens":
+                out, is_error = "The response hit max_tokens before this call was complete; it was not run.", True
+            else:
+                out, is_error = tools.call(b.name, b.input)
+            step["actions"].append({"tool": b.name, "input": b.input, "output": out,
+                                    "is_error": is_error, "seconds": round(time.time() - t0, 2)})
+            results.append({"type": "tool_result", "tool_use_id": b.id, "content": out,
+                            "is_error": is_error})
+            log(f"step {i} ${traj['cost']:.2f} {b.name}: {_brief(b.input)}")
+            if tools.submitted:
+                break
+        messages.append({"role": "user", "content": results})
+
+        if tools.submitted:
+            traj["exit_status"] = "submitted"
+            break
+        if traj["cost"] >= max_cost:
+            traj["exit_status"] = "cost_limit"
+            break
+
+    traj["patch"] = tools.patch()
+    traj["n_steps"] = len(traj["steps"])
+    traj["n_tool_calls"] = sum(len(s["actions"]) for s in traj["steps"])
+    traj["seconds"] = round(time.time() - started, 1)
+    traj["fallback_used"] = any(s["model"] != model for s in traj["steps"])
+    if test_first:
+        traj["repro"] = {"status": tools.repro_status, "command": tools.repro_command,
+                         "attempts": tools.repro_attempts,
+                         "passed_at_submit": tools.repro_passed_at_submit}
+    return traj
+
+
+def _brief(args: dict) -> str:
+    first = next(iter(args.values()), "") if args else ""
+    return str(first).replace("\n", " ")[:80]