bugbout 0.1.0__py3-none-any.whl

bugbout/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

bugbout/__init__.py

@@ -0,0 +1,60 @@
+"""bugbout: find out which coding agent actually fixes bugs in your codebase.
+
+Replay your project's own fixed bugs, let coding agents race to fix them again,
+and score every patch with the tests the maintainers wrote::
+
+    from bugbout import Bout, render_markdown
+
+    report = Bout.load(".").run()  # reads bugbout.toml
+    print(render_markdown(report))
+
+Agents can also be passed in directly, including ones of your own::
+
+    from bugbout import Bout, ClaudeCode, Codex, Project
+
+    bout = Bout(Project.load("."), contestants=[Codex("gpt-5.1-codex"), ClaudeCode("sonnet")])
+"""
+
+from bugbout.__about__ import __version__
+from bugbout.agents import (
+    AgentRun,
+    ClaudeCode,
+    Codex,
+    CodingAgent,
+    CommandAgent,
+    GeminiCli,
+    Invocation,
+    MiniSweAgent,
+    Session,
+    create_agent,
+)
+from bugbout.arena.briefs import ASSIGNMENTS
+from bugbout.arena.judge import POLICIES, JudgePolicy
+from bugbout.bout import Bout
+from bugbout.errors import BugboutError
+from bugbout.project import Examiner, Project
+from bugbout.report import Report, render_markdown, render_terminal
+
+__all__ = [
+    "ASSIGNMENTS",
+    "POLICIES",
+    "AgentRun",
+    "Bout",
+    "BugboutError",
+    "ClaudeCode",
+    "Codex",
+    "CodingAgent",
+    "CommandAgent",
+    "Examiner",
+    "GeminiCli",
+    "Invocation",
+    "JudgePolicy",
+    "MiniSweAgent",
+    "Project",
+    "Report",
+    "Session",
+    "__version__",
+    "create_agent",
+    "render_markdown",
+    "render_terminal",
+]

bugbout/__main__.py

@@ -0,0 +1,5 @@
+"""``python -m bugbout``."""
+
+from bugbout.cli import app
+
+app(prog_name="bugbout")

bugbout/agents/__init__.py

@@ -0,0 +1,26 @@
+"""Adapters that drive coding-agent CLIs headlessly, and the registry that names them."""
+
+from bugbout.agents.base import AgentRun, CodingAgent, Identity, Invocation, Session
+from bugbout.agents.claude_code import ClaudeCode
+from bugbout.agents.codex import Codex
+from bugbout.agents.command import CommandAgent
+from bugbout.agents.gemini_cli import GeminiCli
+from bugbout.agents.mini_swe import MiniSweAgent
+from bugbout.agents.registry import agent_types, create_agent
+from bugbout.agents.vendors import vendor_of
+
+__all__ = [
+    "AgentRun",
+    "ClaudeCode",
+    "Codex",
+    "CodingAgent",
+    "CommandAgent",
+    "GeminiCli",
+    "Identity",
+    "Invocation",
+    "MiniSweAgent",
+    "Session",
+    "agent_types",
+    "create_agent",
+    "vendor_of",
+]

bugbout/agents/base.py

@@ -0,0 +1,287 @@
+"""A coding agent, as bugbout sees it: a command line that edits a checkout.
+
+bugbout reimplements no agent. Each adapter drives the same off-the-shelf CLI a
+developer runs, headlessly, and judges it only by what it leaves on disk. Two
+rules hold for every adapter, because a bout is only as fair as its weakest
+contestant:
+
+* **Nothing personal leaks in.** A developer's agent configuration carries
+  plugins, memories, MCP servers and house rules. Each adapter starts its CLI
+  from a clean configuration, so the comparison is between agents, not set-ups.
+* **Nothing from the future leaks in.** Tasks are real bugs with public fixes,
+  so web tools are off, shell commands are sandboxed, and the checkout has no
+  ``.git`` to read.
+
+Writing an adapter for another agent means implementing :meth:`CodingAgent.invocation`
+and, optionally, :meth:`~CodingAgent.prepare` and :meth:`~CodingAgent.parse`.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import shutil
+import signal
+import subprocess
+import threading
+import time
+from abc import ABC, abstractmethod
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any, ClassVar, TypedDict
+
+from bugbout.agents.vendors import vendor_of
+from bugbout.errors import AgentNotAvailableError
+from bugbout.sandbox import Sandbox
+
+
+@dataclass(frozen=True, slots=True)
+class AgentRun:
+    """How one run of an agent ended, how long it took, and what it cost."""
+
+    agent: str
+    exit_code: int | None
+    timed_out: bool
+    seconds: float
+    summary: str = ""
+    usage: dict[str, int] = field(default_factory=dict)
+    error: str = ""
+
+    @property
+    def ok(self) -> bool:
+        return not self.timed_out and not self.error
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> AgentRun:
+        return cls(**{k: v for k, v in data.items() if k in set(cls.__slots__)})
+
+
+@dataclass(frozen=True, slots=True)
+class Session:
+    """Everything one run of an agent is given."""
+
+    prompt: str
+    workdir: Path
+    """The checkout the agent works in, and the only place it may change."""
+
+    logdir: Path
+    """Where the agent's own logs go."""
+
+    sandbox: Sandbox
+    """Confines the agent's commands, for agents that do not confine themselves."""
+
+
+@dataclass(frozen=True, slots=True)
+class Invocation:
+    """The command line that starts one run."""
+
+    argv: list[str]
+    env: dict[str, str] = field(default_factory=dict)
+    """Added to the caller's environment, which the agent inherits for its credentials."""
+
+    stdin: str | None = None
+    unset: tuple[str, ...] = ()
+    """Variables of the caller's environment the agent must not inherit."""
+
+
+class Identity(TypedDict, total=False):
+    """How a configured agent is told apart from others: overrides for the defaults."""
+
+    name: str
+    vendor: str
+    quota: str
+
+
+class CodingAgent(ABC):
+    """One coding-agent CLI, configured with one model."""
+
+    harness: ClassVar[str]
+    """The agent's own name, such as ``codex``; the default prefix of :attr:`name`."""
+
+    executable: str
+    """The command that starts it."""
+
+    def __init__(
+        self,
+        model: str = "",
+        *,
+        name: str | None = None,
+        vendor: str | None = None,
+        quota: str | None = None,
+    ) -> None:
+        self.model = model
+        """The model to run; empty for the CLI's own default."""
+        self.name = name or (f"{self.harness}/{model}" if model else self.harness)
+        """Unique within a bout; used in reports and paths."""
+        self.vendor = vendor or vendor_of(model or self.harness)
+        """Who made the model, so the judge can keep evidence independent."""
+        self.quota = quota or self.harness
+        """The rate limit this agent draws on. Agents sharing one run in a shared lane."""
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}({self.name!r})"
+
+    # --- what an adapter provides ---------------------------------------------------
+
+    @abstractmethod
+    def invocation(self, session: Session) -> Invocation:
+        """The command line for one run."""
+
+    def prepare(self, workdir: Path) -> None:  # noqa: B027 - an optional hook
+        """Set up the checkout before the agent starts, e.g. workspace settings."""
+
+    def parse(self, logdir: Path) -> tuple[str, dict[str, int]]:  # noqa: ARG002 - a default
+        """The agent's closing summary and token usage, read from its logs."""
+        return "", {}
+
+    def describe(self) -> dict[str, str]:
+        """What a reader needs to reproduce this agent's runs."""
+        return {
+            "harness": self.harness,
+            "version": version_of(self.command()),
+            "model": self.model or "the CLI's default",
+            "vendor": self.vendor,
+        }
+
+    # --- running ----------------------------------------------------------------------
+
+    def command(self) -> str:
+        """The executable to start: found on ``PATH`` or in a per-user install."""
+        return find_executable(self.executable) or self.executable
+
+    def check(self) -> None:
+        """Raise :class:`AgentNotAvailableError` unless the agent can be started."""
+        if find_executable(self.executable) is None:
+            raise AgentNotAvailableError(f"{self.name}: `{self.executable}` is not installed")
+
+    def run(
+        self, prompt: str, workdir: Path, logdir: Path, *, sandbox: Sandbox, timeout: int
+    ) -> AgentRun:
+        """Start the agent in ``workdir``, wait for it, and record how it ended."""
+        logdir.mkdir(parents=True, exist_ok=True)
+        self.prepare(workdir)
+        invocation = self.invocation(Session(prompt, workdir, logdir, sandbox))
+        started = time.monotonic()
+        code, timed_out, error = _execute(invocation, workdir, logdir, timeout)
+        summary, usage = self.parse(logdir)
+        return AgentRun(
+            agent=self.name,
+            exit_code=code,
+            timed_out=timed_out,
+            seconds=round(time.monotonic() - started, 1),
+            summary=summary,
+            usage=usage,
+            error=error,
+        )
+
+
+def _execute(
+    invocation: Invocation, workdir: Path, logdir: Path, timeout: int
+) -> tuple[int | None, bool, str]:
+    """Run to the end or the timeout: the exit code, whether it timed out, and any error."""
+    with (
+        (logdir / "stdout.log").open("w", encoding="utf-8") as out,
+        (logdir / "stderr.log").open("w", encoding="utf-8") as err,
+    ):
+        try:
+            # A session of its own, so a timeout can take down every process the
+            # agent spawned - an orphaned test run would otherwise keep going.
+            process = subprocess.Popen(  # noqa: S603 - an adapter's own command line
+                invocation.argv,
+                cwd=workdir,
+                env={**_inherited(invocation.unset), **invocation.env},
+                stdin=subprocess.PIPE,
+                stdout=out,
+                stderr=err,
+                text=True,
+                start_new_session=True,
+            )
+        except OSError as exc:
+            return None, False, f"could not start {invocation.argv[0]}: {exc}"
+        with _RUNNING_LOCK:
+            _RUNNING.add(process)
+        try:
+            process.communicate(input=invocation.stdin, timeout=timeout)
+        except subprocess.TimeoutExpired:
+            _kill(process)
+            return None, True, f"timed out after {timeout}s"
+        except BaseException:
+            _kill(process)  # interrupted: never leave an agent running unattended
+            raise
+        finally:
+            with _RUNNING_LOCK:
+                _RUNNING.discard(process)
+    code = process.returncode
+    return code, False, "" if code == 0 else f"exit code {code}"
+
+
+def _inherited(unset: tuple[str, ...]) -> dict[str, str]:
+    return {key: value for key, value in os.environ.items() if key not in unset}
+
+
+_RUNNING: set[subprocess.Popen[str]] = set()
+_RUNNING_LOCK = threading.Lock()
+
+
+def stop_all() -> None:
+    """Kill every agent still running, with everything it started.
+
+    Agents run in sessions of their own, so an interrupt in the terminal does
+    not reach them; whoever handles the interrupt calls this.
+    """
+    with _RUNNING_LOCK:
+        running = list(_RUNNING)
+    for process in running:
+        _kill(process)
+
+
+def _kill(process: subprocess.Popen[str]) -> None:
+    with contextlib.suppress(ProcessLookupError):
+        os.killpg(process.pid, signal.SIGKILL)
+    process.wait()
+
+
+def find_executable(name: str) -> str | None:
+    """``name`` on ``PATH``, or where a per-user npm or uv install puts it."""
+    for candidate in (
+        shutil.which(name),
+        str(Path.home() / ".npm-global" / "bin" / name),
+        str(Path.home() / ".local" / "bin" / name),
+    ):
+        if candidate and Path(candidate).is_file():
+            return candidate
+    return None
+
+
+def version_of(executable: str) -> str:
+    """The first line ``executable --version`` prints."""
+    try:
+        completed = subprocess.run(  # noqa: S603 - a resolved CLI path and a literal flag
+            [executable, "--version"], capture_output=True, text=True, timeout=30, check=False
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return "unavailable"
+    lines = completed.stdout.strip().splitlines()
+    return lines[0] if lines else "unknown"
+
+
+def read_jsonl(path: Path) -> list[dict[str, Any]]:
+    """Every JSON object in a line-delimited log, skipping anything else."""
+    if not path.exists():
+        return []
+    events: list[dict[str, Any]] = []
+    for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
+        line = line.strip()
+        if not line.startswith("{"):
+            continue
+        try:
+            parsed = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(parsed, dict):
+            events.append(parsed)
+    return events

bugbout/agents/claude_code.py

@@ -0,0 +1,107 @@
+"""Anthropic's Claude Code."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Unpack, override
+
+from bugbout.agents.base import CodingAgent, Identity, Invocation, Session, read_jsonl
+
+# Shell commands run in Claude Code's own sandbox and only there: if the sandbox
+# cannot start, Claude Code stops instead of running them unconfined.
+_SETTINGS = {
+    "sandbox": {
+        "enabled": True,
+        "failIfUnavailable": True,
+        "autoAllowBashIfSandboxed": True,
+        "allowUnsandboxedCommands": False,
+    }
+}
+
+
+# What a running Claude Code session exports to the commands it runs. bugbout may
+# itself be run from inside Claude Code; a contestant is a session of its own and
+# must never attach to that one.
+PARENT_SESSION = (
+    "CLAUDECODE",
+    "CLAUDE_AGENT_SDK_VERSION",
+    "CLAUDE_CODE_CHILD_SESSION",
+    "CLAUDE_CODE_EMIT_STARTUP_TIMING",
+    "CLAUDE_CODE_ENABLE_SDK_FILE_CHECKPOINTING",
+    "CLAUDE_CODE_ENABLE_TASKS",
+    "CLAUDE_CODE_ENTRYPOINT",
+    "CLAUDE_CODE_EXECPATH",
+    "CLAUDE_CODE_MESSAGING_SOCKET",
+    "CLAUDE_CODE_MESSAGING_TOKEN",
+    "CLAUDE_CODE_QUESTION_PREVIEW_FORMAT",
+    "CLAUDE_CODE_SESSION_ATTENDED",
+    "CLAUDE_CODE_SESSION_ID",
+    "CLAUDE_CODE_SSE_PORT",
+    "CLAUDE_EFFORT",
+    "CLAUDE_PID",
+    "ENABLE_IDE_INTEGRATION",
+)
+
+
+class ClaudeCode(CodingAgent):
+    """``claude --print`` from a clean configuration, in Claude Code's own sandbox.
+
+    Safe mode leaves out the user's CLAUDE.md, skills, plugins, hooks and MCP
+    servers; web tools are disallowed and no session is saved. Edits are
+    accepted inside the checkout only, shell commands run sandboxed, and anything
+    that would need a person's approval is refused rather than asked about.
+    """
+
+    harness = "claude-code"
+    executable = "claude"
+
+    def __init__(
+        self, model: str = "", *, effort: str | None = None, **identity: Unpack[Identity]
+    ) -> None:
+        """``model`` is a full model name or an alias such as ``sonnet``; empty for the default."""
+        identity.setdefault("vendor", "anthropic")  # an alias names no vendor
+        super().__init__(model, **identity)
+        self.effort = effort
+
+    @override
+    def invocation(self, session: Session) -> Invocation:
+        argv = [
+            self.command(),
+            "--print",
+            "--output-format", "stream-json",
+            "--verbose",
+            *(["--model", self.model] if self.model else []),
+            *(["--effort", self.effort] if self.effort else []),
+            "--safe-mode",
+            "--no-session-persistence",
+            "--permission-mode", "acceptEdits",
+            "--permission-prompts", "none",
+            "--settings", json.dumps(_SETTINGS),
+            "--disallowedTools", "WebFetch,WebSearch",
+        ]  # fmt: skip
+        return Invocation(argv, stdin=session.prompt, unset=PARENT_SESSION)
+
+    @override
+    def parse(self, logdir: Path) -> tuple[str, dict[str, int]]:
+        for event in reversed(read_jsonl(logdir / "stdout.log")):
+            if event.get("type") == "result":
+                usage = event.get("usage")
+                counts = (
+                    {k: v for k, v in usage.items() if isinstance(v, int)}
+                    if isinstance(usage, dict)
+                    else {}
+                )
+                if isinstance(turns := event.get("num_turns"), int):
+                    counts["turns"] = turns
+                return str(event.get("result") or "").strip(), counts
+        return "", {}
+
+    @override
+    def describe(self) -> dict[str, str]:
+        return {
+            **super().describe(),
+            "effort": self.effort or "model default",
+            "sandbox": "Claude Code's sandbox for shell commands; edits only in the checkout",
+            "configuration": "safe mode (no CLAUDE.md, skills, plugins, hooks or MCP); no web tools",
+        }

bugbout/agents/codex.py

@@ -0,0 +1,105 @@
+"""OpenAI's Codex CLI."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Unpack, override
+
+from bugbout.agents.base import CodingAgent, Identity, Invocation, Session, read_jsonl
+
+
+class Codex(CodingAgent):
+    """``codex exec`` from a clean configuration, in Codex's own sandbox.
+
+    The user's config and rules are ignored, the session is ephemeral, and every
+    feature that reaches beyond the checkout is switched off. Shell commands run
+    in Codex's ``workspace-write`` sandbox: no network, writes only in the
+    checkout. A model runs at its default reasoning effort unless told otherwise,
+    so agents are compared as shipped, not as tuned.
+    """
+
+    harness = "codex"
+    executable = "codex"
+
+    # Switched off through config overrides, which a Codex version that does not
+    # know a feature ignores - where ``--disable`` would refuse to start.
+    DISABLED_FEATURES = (
+        "apps",
+        "browser_use",
+        "browser_use_external",
+        "browser_use_full_cdp_access",
+        "computer_use",
+        "hooks",
+        "image_generation",
+        "in_app_browser",
+        "memories",
+        "plugins",
+        "remote_plugin",
+        "skill_mcp_dependency_install",
+        "skill_search",
+    )
+
+    def __init__(
+        self,
+        model: str = "",
+        *,
+        reasoning_effort: str | None = None,
+        local_provider: str | None = None,
+        **identity: Unpack[Identity],
+    ) -> None:
+        """``local_provider`` (``ollama`` or ``lmstudio``) serves an open model locally."""
+        identity.setdefault("quota", local_provider or self.harness)
+        super().__init__(model, **identity)
+        self.reasoning_effort = reasoning_effort
+        self.local_provider = local_provider
+
+    @override
+    def invocation(self, session: Session) -> Invocation:
+        effort = (
+            ["-c", f'model_reasoning_effort="{self.reasoning_effort}"']
+            if self.reasoning_effort
+            else []
+        )
+        local = ["--oss", "--local-provider", self.local_provider] if self.local_provider else []
+        argv = [
+            self.command(),
+            "exec",
+            "--ignore-user-config",
+            "--ignore-rules",
+            "--ephemeral",
+            "--skip-git-repo-check",
+            "--sandbox", "workspace-write",
+            "--cd", str(session.workdir),
+            *(["--model", self.model] if self.model else []),
+            *effort,
+            *local,
+            "-c", 'web_search="disabled"',
+            *[arg for feature in self.DISABLED_FEATURES for arg in ("-c", f"features.{feature}=false")],
+            "--json",
+            "--color", "never",
+            "--output-last-message", str(session.logdir / "last_message.txt"),
+            "-",
+        ]  # fmt: skip
+        return Invocation(argv, stdin=session.prompt)
+
+    @override
+    def parse(self, logdir: Path) -> tuple[str, dict[str, int]]:
+        last = logdir / "last_message.txt"
+        summary = last.read_text(encoding="utf-8").strip() if last.exists() else ""
+        usage: dict[str, int] = {}
+        for event in read_jsonl(logdir / "stdout.log"):
+            if isinstance(found := event.get("usage"), dict):
+                for key, value in found.items():
+                    if isinstance(value, int):
+                        usage[key] = usage.get(key, 0) + value
+        return summary, usage
+
+    @override
+    def describe(self) -> dict[str, str]:
+        return {
+            **super().describe(),
+            "reasoning_effort": self.reasoning_effort or "model default",
+            "provider": f"{self.local_provider} (local)" if self.local_provider else "OpenAI",
+            "sandbox": "Codex workspace-write: no network for commands, writes only in the checkout",
+            "configuration": "user config and rules ignored; web search, plugins and hooks off",
+        }

bugbout/agents/command.py

@@ -0,0 +1,83 @@
+"""Any coding agent with a command line, configured instead of coded."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from pathlib import Path
+from typing import override
+
+from bugbout.agents.base import CodingAgent, Invocation, Session
+from bugbout.agents.vendors import vendor_of
+from bugbout.errors import ConfigError
+from bugbout.sandbox import Confinement
+
+PLACEHOLDERS = ("prompt", "prompt_file", "workdir", "logdir", "model")
+
+
+class CommandAgent(CodingAgent):
+    """Runs a command line, with placeholders filled in for each run.
+
+    In the command, ``{prompt}``, ``{prompt_file}``, ``{workdir}``, ``{logdir}``
+    and ``{model}`` are replaced; when it uses neither ``{prompt}`` nor
+    ``{prompt_file}``, the prompt arrives on standard input. The whole process
+    runs in bugbout's sandbox - it may use the network to reach its model, but
+    write only in the checkout, its logs and the ``state_dirs`` it names.
+    """
+
+    harness = "command"
+
+    def __init__(
+        self,
+        command: Sequence[str],
+        *,
+        name: str,
+        model: str = "",
+        env: dict[str, str] | None = None,
+        state_dirs: Sequence[str] = (),
+        sandboxed: bool = True,
+        vendor: str | None = None,
+        quota: str | None = None,
+    ) -> None:
+        if not command:
+            raise ConfigError(f"agent {name!r}: the command is empty")
+        super().__init__(
+            model, name=name, vendor=vendor or vendor_of(model or name), quota=quota or name
+        )
+        self.argv = list(command)
+        self.executable = self.argv[0]
+        self.env = env or {}
+        self.state_dirs = tuple(Path(d).expanduser() for d in state_dirs)
+        self.sandboxed = sandboxed
+
+    @override
+    def invocation(self, session: Session) -> Invocation:
+        prompt_file = session.logdir / "prompt.md"
+        prompt_file.write_text(session.prompt, encoding="utf-8")
+        values = {
+            "prompt": session.prompt,
+            "prompt_file": str(prompt_file),
+            "workdir": str(session.workdir),
+            "logdir": str(session.logdir),
+            "model": self.model,
+        }
+        argv = [self.command(), *(_fill(arg, values) for arg in self.argv[1:])]
+        takes_prompt = any("{prompt}" in arg or "{prompt_file}" in arg for arg in self.argv)
+        if self.sandboxed:
+            writable = (session.workdir, session.logdir, *self.state_dirs)
+            argv = session.sandbox.wrap(argv, Confinement(writable=writable, network=True))
+        return Invocation(argv, env=self.env, stdin=None if takes_prompt else session.prompt)
+
+    @override
+    def describe(self) -> dict[str, str]:
+        return {
+            **super().describe(),
+            "command": " ".join(self.argv),
+            "sandbox": "bugbout's sandbox" if self.sandboxed else "none",
+        }
+
+
+def _fill(arg: str, values: dict[str, str]) -> str:
+    # Plain replacement, not str.format: arguments may hold braces of their own.
+    for key in PLACEHOLDERS:
+        arg = arg.replace("{" + key + "}", values[key])
+    return arg