coderace 0.1.0__py3-none-any.whl

coderace/__init__.py

@@ -0,0 +1,3 @@
+"""coderace - Race coding agents against each other on real tasks."""
+
+__version__ = "0.1.0"

coderace/adapters/__init__.py

@@ -0,0 +1,23 @@
+"""Agent adapters for coderace."""
+
+from coderace.adapters.aider import AiderAdapter
+from coderace.adapters.base import BaseAdapter
+from coderace.adapters.claude import ClaudeAdapter
+from coderace.adapters.codex import CodexAdapter
+from coderace.adapters.gemini import GeminiAdapter
+
+ADAPTERS: dict[str, type[BaseAdapter]] = {
+    "claude": ClaudeAdapter,
+    "codex": CodexAdapter,
+    "aider": AiderAdapter,
+    "gemini": GeminiAdapter,
+}
+
+__all__ = [
+    "ADAPTERS",
+    "BaseAdapter",
+    "ClaudeAdapter",
+    "CodexAdapter",
+    "AiderAdapter",
+    "GeminiAdapter",
+]

coderace/adapters/aider.py

@@ -0,0 +1,20 @@
+"""Aider adapter."""
+
+from __future__ import annotations
+
+from coderace.adapters.base import BaseAdapter
+
+
+class AiderAdapter(BaseAdapter):
+    """Adapter for Aider coding assistant."""
+
+    name = "aider"
+
+    def build_command(self, task_description: str) -> list[str]:
+        return [
+            "aider",
+            "--message",
+            task_description,
+            "--yes",
+            "--no-auto-commits",
+        ]

coderace/adapters/base.py

@@ -0,0 +1,60 @@
+"""Base adapter interface."""
+
+from __future__ import annotations
+
+import subprocess
+import time
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from coderace.types import AgentResult
+
+
+class BaseAdapter(ABC):
+    """Abstract base class for coding agent adapters."""
+
+    name: str = "base"
+
+    @abstractmethod
+    def build_command(self, task_description: str) -> list[str]:
+        """Build the CLI command to invoke this agent."""
+        ...
+
+    def run(self, task_description: str, workdir: Path, timeout: int) -> AgentResult:
+        """Run the agent on a task and capture results."""
+        cmd = self.build_command(task_description)
+        start = time.monotonic()
+        timed_out = False
+
+        try:
+            proc = subprocess.run(
+                cmd,
+                cwd=workdir,
+                capture_output=True,
+                text=True,
+                timeout=timeout,
+            )
+            exit_code = proc.returncode
+            stdout = proc.stdout
+            stderr = proc.stderr
+        except subprocess.TimeoutExpired as e:
+            timed_out = True
+            exit_code = -1
+            stdout = (e.stdout or b"").decode("utf-8", errors="replace") if e.stdout else ""
+            stderr = (e.stderr or b"").decode("utf-8", errors="replace") if e.stderr else ""
+        except FileNotFoundError:
+            timed_out = False
+            exit_code = 127
+            stdout = ""
+            stderr = f"Command not found: {cmd[0]}"
+
+        wall_time = time.monotonic() - start
+
+        return AgentResult(
+            agent=self.name,
+            exit_code=exit_code,
+            stdout=stdout,
+            stderr=stderr,
+            wall_time=wall_time,
+            timed_out=timed_out,
+        )

coderace/adapters/claude.py

@@ -0,0 +1,21 @@
+"""Claude Code adapter."""
+
+from __future__ import annotations
+
+from coderace.adapters.base import BaseAdapter
+
+
+class ClaudeAdapter(BaseAdapter):
+    """Adapter for Claude Code CLI."""
+
+    name = "claude"
+
+    def build_command(self, task_description: str) -> list[str]:
+        return [
+            "claude",
+            "--print",
+            "--output-format",
+            "json",
+            "-p",
+            task_description,
+        ]

coderace/adapters/codex.py

@@ -0,0 +1,20 @@
+"""Codex CLI adapter."""
+
+from __future__ import annotations
+
+from coderace.adapters.base import BaseAdapter
+
+
+class CodexAdapter(BaseAdapter):
+    """Adapter for OpenAI Codex CLI."""
+
+    name = "codex"
+
+    def build_command(self, task_description: str) -> list[str]:
+        return [
+            "codex",
+            "--quiet",
+            "--full-auto",
+            "-p",
+            task_description,
+        ]

coderace/adapters/gemini.py

@@ -0,0 +1,19 @@
+"""Gemini CLI adapter."""
+
+from __future__ import annotations
+
+from coderace.adapters.base import BaseAdapter
+
+
+class GeminiAdapter(BaseAdapter):
+    """Adapter for Google Gemini CLI."""
+
+    name = "gemini"
+
+    def build_command(self, task_description: str) -> list[str]:
+        return [
+            "gemini",
+            "--non-interactive",
+            "-p",
+            task_description,
+        ]

coderace/cli.py

@@ -0,0 +1,306 @@
+"""CLI interface for coderace."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from coderace import __version__
+from coderace.adapters import ADAPTERS
+from coderace.git_ops import (
+    add_worktree,
+    branch_name_for,
+    checkout,
+    create_branch,
+    get_current_ref,
+    get_diff_stat,
+    has_uncommitted_changes,
+    prune_worktrees,
+    remove_worktree,
+)
+from coderace.reporter import print_results, save_results_json
+from coderace.scorer import compute_score
+from coderace.task import create_template, load_task
+from coderace.types import AgentResult, Score
+
+app = typer.Typer(
+    name="coderace",
+    help="Race coding agents against each other on real tasks.",
+    no_args_is_help=True,
+)
+console = Console()
+
+
+@app.command()
+def init(
+    name: str = typer.Argument(help="Task name"),
+    output_dir: Path = typer.Option(".", "--output", "-o", help="Output directory"),
+) -> None:
+    """Create a task YAML template."""
+    path = create_template(name, output_dir)
+    console.print(f"[green]Created task template:[/green] {path}")
+    console.print("Edit the file to define your task, then run: coderace run " + str(path))
+
+
+def _run_agent_sequential(
+    agent_name: str,
+    task_description: str,
+    repo: Path,
+    branch: str,
+    base_ref: str,
+    timeout: int,
+) -> tuple[AgentResult | None, int]:
+    """Run a single agent sequentially (on the main repo). Returns (result, lines_changed)."""
+    try:
+        create_branch(repo, branch, base_ref)
+    except Exception:
+        return None, 0
+
+    adapter = ADAPTERS[agent_name]()
+    result = adapter.run(task_description, repo, timeout)
+
+    _, lines = get_diff_stat(repo, base_ref)
+    return result, lines
+
+
+def _run_agent_worktree(
+    agent_name: str,
+    task_description: str,
+    repo: Path,
+    branch: str,
+    base_ref: str,
+    timeout: int,
+) -> tuple[AgentResult | None, int]:
+    """Run a single agent in a git worktree (for parallel execution)."""
+    import tempfile
+
+    worktree_dir = Path(tempfile.mkdtemp(prefix=f"coderace-{agent_name}-"))
+
+    try:
+        # Create branch first (from main repo)
+        create_branch(repo, branch, base_ref)
+        # Checkout back so worktree can use the branch
+        checkout(repo, base_ref)
+        # Create worktree
+        add_worktree(repo, worktree_dir, branch)
+
+        adapter = ADAPTERS[agent_name]()
+        result = adapter.run(task_description, worktree_dir, timeout)
+
+        _, lines = get_diff_stat(worktree_dir, base_ref)
+        return result, lines
+    except Exception:
+        return None, 0
+    finally:
+        remove_worktree(repo, worktree_dir)
+        # Clean up temp dir if it still exists
+        import shutil
+
+        if worktree_dir.exists():
+            shutil.rmtree(worktree_dir, ignore_errors=True)
+
+
+@app.command()
+def run(
+    task_file: Path = typer.Argument(help="Path to task YAML file"),
+    agents: list[str] | None = typer.Option(None, "--agent", "-a", help="Override agent list"),
+    parallel: bool = typer.Option(False, "--parallel", "-p", help="Run agents in parallel"),
+) -> None:
+    """Run all agents on a task and score the results."""
+    task = load_task(task_file)
+
+    if agents:
+        task.agents = agents
+
+    repo = task.repo
+    if not repo.exists():
+        console.print(f"[red]Repo not found:[/red] {repo}")
+        raise typer.Exit(1)
+
+    if has_uncommitted_changes(repo):
+        console.print("[red]Repo has uncommitted changes. Commit or stash first.[/red]")
+        raise typer.Exit(1)
+
+    base_ref = get_current_ref(repo)
+    mode = "parallel" if parallel else "sequential"
+    console.print(f"[dim]Base ref: {base_ref[:8]} | Mode: {mode}[/dim]")
+    console.print(f"[dim]Task: {task.name}[/dim]")
+    console.print(f"[dim]Agents: {', '.join(task.agents)}[/dim]")
+    console.print()
+
+    # Validate agents
+    valid_agents = [a for a in task.agents if a in ADAPTERS]
+    invalid = set(task.agents) - set(valid_agents)
+    for name in invalid:
+        console.print(f"[red]Unknown agent: {name}[/red]")
+
+    if not valid_agents:
+        console.print("[red]No valid agents to run.[/red]")
+        raise typer.Exit(1)
+
+    agent_results: list[AgentResult] = []
+    diff_lines_map: dict[str, int] = {}
+
+    if parallel and len(valid_agents) > 1:
+        console.print("[cyan]Racing agents in parallel...[/cyan]")
+        from concurrent.futures import ThreadPoolExecutor, as_completed
+
+        futures = {}
+        with ThreadPoolExecutor(max_workers=len(valid_agents)) as executor:
+            for agent_name in valid_agents:
+                branch = branch_name_for(task.name, agent_name)
+                future = executor.submit(
+                    _run_agent_worktree,
+                    agent_name,
+                    task.description,
+                    repo,
+                    branch,
+                    base_ref,
+                    task.timeout,
+                )
+                futures[future] = agent_name
+
+            for future in as_completed(futures):
+                agent_name = futures[future]
+                result, lines = future.result()
+                if result:
+                    agent_results.append(result)
+                    diff_lines_map[agent_name] = lines
+                    if result.timed_out:
+                        console.print(
+                            f"  [yellow]{agent_name}: timed out after {task.timeout}s[/yellow]"
+                        )
+                    elif result.exit_code != 0:
+                        console.print(
+                            f"  [yellow]{agent_name}: exit code {result.exit_code}[/yellow]"
+                        )
+                    else:
+                        console.print(
+                            f"  [green]{agent_name}: completed in {result.wall_time:.1f}s[/green]"
+                        )
+                else:
+                    console.print(f"  [red]{agent_name}: failed to run[/red]")
+
+        prune_worktrees(repo)
+    else:
+        # Sequential mode
+        for agent_name in valid_agents:
+            branch = branch_name_for(task.name, agent_name)
+            console.print(f"[cyan]Running {agent_name}...[/cyan]")
+
+            result, lines = _run_agent_sequential(
+                agent_name, task.description, repo, branch, base_ref, task.timeout
+            )
+
+            if result is None:
+                console.print(f"  [red]Failed to create branch for {agent_name}[/red]")
+                continue
+
+            agent_results.append(result)
+            diff_lines_map[agent_name] = lines
+
+            if result.timed_out:
+                console.print(f"  [yellow]Timed out after {task.timeout}s[/yellow]")
+            elif result.exit_code != 0:
+                console.print(f"  [yellow]Exit code: {result.exit_code}[/yellow]")
+            else:
+                console.print(f"  [green]Completed in {result.wall_time:.1f}s[/green]")
+
+            console.print(f"  [dim]Lines changed: {lines}[/dim]")
+
+    if not agent_results:
+        console.print("[red]No agents ran successfully.[/red]")
+        raise typer.Exit(1)
+
+    # Score each agent (checkout each branch for test/lint)
+    all_wall_times = [r.wall_time for r in agent_results]
+    all_diff_lines = [diff_lines_map.get(r.agent, 0) for r in agent_results]
+    scores: list[Score] = []
+
+    for result in agent_results:
+        branch = branch_name_for(task.name, result.agent)
+        checkout(repo, branch)
+
+        score = compute_score(
+            result=result,
+            test_command=task.test_command,
+            lint_command=task.lint_command,
+            workdir=repo,
+            diff_lines=diff_lines_map.get(result.agent, 0),
+            all_wall_times=all_wall_times,
+            all_diff_lines=all_diff_lines,
+        )
+        scores.append(score)
+
+    # Return to base ref
+    checkout(repo, base_ref)
+
+    # Display and save results
+    console.print()
+    print_results(scores, console)
+
+    results_dir = Path(task_file).parent / ".coderace"
+    json_path = results_dir / f"{task.name}-results.json"
+    save_results_json(scores, json_path)
+    console.print(f"\n[dim]Results saved to {json_path}[/dim]")
+
+
+@app.command()
+def results(
+    task_file: Path = typer.Argument(help="Path to task YAML file"),
+) -> None:
+    """Show results from the last run."""
+    task = load_task(task_file)
+    results_dir = Path(task_file).parent / ".coderace"
+    json_path = results_dir / f"{task.name}-results.json"
+
+    if not json_path.exists():
+        console.print(f"[red]No results found. Run the task first:[/red] coderace run {task_file}")
+        raise typer.Exit(1)
+
+    from coderace.reporter import load_results_json
+
+    data = load_results_json(json_path)
+
+    from rich.table import Table
+
+    table = Table(title=f"coderace results: {task.name}", show_lines=True)
+    table.add_column("Rank", justify="center", style="bold")
+    table.add_column("Agent", style="cyan")
+    table.add_column("Score", justify="right", style="bold green")
+    table.add_column("Tests", justify="center")
+    table.add_column("Exit", justify="center")
+    table.add_column("Lint", justify="center")
+    table.add_column("Time (s)", justify="right")
+    table.add_column("Lines", justify="right")
+
+    for entry in data:
+        b = entry["breakdown"]
+        table.add_row(
+            str(entry["rank"]),
+            entry["agent"],
+            f"{entry['composite_score']:.1f}",
+            _bool_icon(b["tests_pass"]),
+            _bool_icon(b["exit_clean"]),
+            _bool_icon(b["lint_clean"]),
+            f"{b['wall_time']:.1f}",
+            str(b["lines_changed"]),
+        )
+
+    console.print(table)
+
+
+@app.command()
+def version() -> None:
+    """Show coderace version."""
+    console.print(f"coderace {__version__}")
+
+
+def _bool_icon(val: bool) -> str:
+    return "[green]PASS[/green]" if val else "[red]FAIL[/red]"
+
+
+if __name__ == "__main__":
+    app()

coderace/git_ops.py

@@ -0,0 +1,151 @@
+"""Git operations for branch isolation."""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+
+class GitError(Exception):
+    """Raised when a git operation fails."""
+
+
+def get_current_ref(repo: Path) -> str:
+    """Get the current HEAD commit hash."""
+    result = subprocess.run(
+        ["git", "rev-parse", "HEAD"],
+        cwd=repo,
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        raise GitError(f"Failed to get HEAD: {result.stderr.strip()}")
+    return result.stdout.strip()
+
+
+def create_branch(repo: Path, branch_name: str, base_ref: str) -> None:
+    """Create and checkout a new branch from base_ref."""
+    result = subprocess.run(
+        ["git", "checkout", "-b", branch_name, base_ref],
+        cwd=repo,
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        raise GitError(f"Failed to create branch {branch_name}: {result.stderr.strip()}")
+
+
+def checkout(repo: Path, ref: str) -> None:
+    """Checkout a ref (branch name or commit hash)."""
+    result = subprocess.run(
+        ["git", "checkout", ref],
+        cwd=repo,
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        raise GitError(f"Failed to checkout {ref}: {result.stderr.strip()}")
+
+
+def get_diff_stat(repo: Path, base_ref: str) -> tuple[str, int]:
+    """Get diff stat and total lines changed against base_ref.
+
+    Returns (diff_stat_text, total_lines_changed).
+    """
+    stat_result = subprocess.run(
+        ["git", "diff", "--stat", base_ref],
+        cwd=repo,
+        capture_output=True,
+        text=True,
+    )
+
+    numstat_result = subprocess.run(
+        ["git", "diff", "--numstat", base_ref],
+        cwd=repo,
+        capture_output=True,
+        text=True,
+    )
+
+    total_lines = 0
+    for line in numstat_result.stdout.strip().splitlines():
+        parts = line.split("\t")
+        if len(parts) >= 2:
+            try:
+                added = int(parts[0]) if parts[0] != "-" else 0
+                removed = int(parts[1]) if parts[1] != "-" else 0
+                total_lines += added + removed
+            except ValueError:
+                continue
+
+    return stat_result.stdout.strip(), total_lines
+
+
+def branch_name_for(task_name: str, agent_name: str) -> str:
+    """Generate a branch name for a task/agent combo."""
+    return f"coderace/{agent_name}-{task_name}"
+
+
+def has_uncommitted_changes(repo: Path) -> bool:
+    """Check if the repo has uncommitted changes."""
+    result = subprocess.run(
+        ["git", "status", "--porcelain"],
+        cwd=repo,
+        capture_output=True,
+        text=True,
+    )
+    return bool(result.stdout.strip())
+
+
+def stash_changes(repo: Path) -> bool:
+    """Stash uncommitted changes. Returns True if something was stashed."""
+    if not has_uncommitted_changes(repo):
+        return False
+    result = subprocess.run(
+        ["git", "stash", "push", "-m", "coderace: auto-stash"],
+        cwd=repo,
+        capture_output=True,
+        text=True,
+    )
+    return result.returncode == 0
+
+
+def stash_pop(repo: Path) -> None:
+    """Pop the most recent stash."""
+    subprocess.run(
+        ["git", "stash", "pop"],
+        cwd=repo,
+        capture_output=True,
+        text=True,
+    )
+
+
+def add_worktree(repo: Path, worktree_path: Path, branch: str) -> None:
+    """Create a git worktree for a branch."""
+    result = subprocess.run(
+        ["git", "worktree", "add", str(worktree_path), branch],
+        cwd=repo,
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        raise GitError(f"Failed to create worktree: {result.stderr.strip()}")
+
+
+def remove_worktree(repo: Path, worktree_path: Path) -> None:
+    """Remove a git worktree."""
+    subprocess.run(
+        ["git", "worktree", "remove", str(worktree_path), "--force"],
+        cwd=repo,
+        capture_output=True,
+        text=True,
+    )
+
+
+def prune_worktrees(repo: Path) -> None:
+    """Prune stale worktree metadata."""
+    subprocess.run(
+        ["git", "worktree", "prune"],
+        cwd=repo,
+        capture_output=True,
+        text=True,
+    )

coderace/reporter.py

@@ -0,0 +1,92 @@
+"""Results reporting (terminal table + JSON)."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from rich.console import Console
+from rich.table import Table
+
+from coderace.types import Score
+
+
+def print_results(scores: list[Score], console: Console | None = None) -> str:
+    """Print a rich table of results. Returns the table as a string."""
+    console = console or Console()
+
+    # Sort by composite score descending
+    ranked = sorted(scores, key=lambda s: s.composite, reverse=True)
+
+    table = Table(title="coderace results", show_lines=True)
+    table.add_column("Rank", justify="center", style="bold")
+    table.add_column("Agent", style="cyan")
+    table.add_column("Score", justify="right", style="bold green")
+    table.add_column("Tests", justify="center")
+    table.add_column("Exit", justify="center")
+    table.add_column("Lint", justify="center")
+    table.add_column("Time (s)", justify="right")
+    table.add_column("Lines", justify="right")
+
+    for i, score in enumerate(ranked, 1):
+        b = score.breakdown
+        table.add_row(
+            str(i),
+            score.agent,
+            f"{score.composite:.1f}",
+            _bool_icon(b.tests_pass),
+            _bool_icon(b.exit_clean),
+            _bool_icon(b.lint_clean),
+            f"{b.wall_time:.1f}",
+            str(b.lines_changed),
+        )
+
+    console.print(table)
+
+    # Return as string for testing
+    str_console = Console(file=None, force_terminal=False, width=100)
+    with str_console.capture() as capture:
+        str_console.print(table)
+    return capture.get()
+
+
+def save_results_json(scores: list[Score], output_path: Path) -> None:
+    """Save results as JSON."""
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    data = {
+        "results": [
+            {
+                "rank": i,
+                "agent": score.agent,
+                "composite_score": score.composite,
+                "breakdown": {
+                    "tests_pass": score.breakdown.tests_pass,
+                    "exit_clean": score.breakdown.exit_clean,
+                    "lint_clean": score.breakdown.lint_clean,
+                    "wall_time": round(score.breakdown.wall_time, 2),
+                    "lines_changed": score.breakdown.lines_changed,
+                },
+                "tests_output": score.tests_output,
+                "lint_output": score.lint_output,
+                "diff_stat": score.diff_stat,
+            }
+            for i, score in enumerate(
+                sorted(scores, key=lambda s: s.composite, reverse=True), 1
+            )
+        ]
+    }
+
+    output_path.write_text(json.dumps(data, indent=2) + "\n")
+
+
+def load_results_json(path: Path) -> list[dict]:
+    """Load results from a JSON file."""
+    if not path.exists():
+        raise FileNotFoundError(f"No results found at {path}")
+    data = json.loads(path.read_text())
+    return data.get("results", [])
+
+
+def _bool_icon(val: bool) -> str:
+    return "[green]PASS[/green]" if val else "[red]FAIL[/red]"

coderace/scorer.py

@@ -0,0 +1,120 @@
+"""Scoring engine for agent results."""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+from coderace.types import AgentResult, Score, ScoreBreakdown
+
+# Weights must sum to 1.0
+WEIGHTS = {
+    "tests_pass": 0.40,
+    "exit_clean": 0.20,
+    "lint_clean": 0.15,
+    "wall_time": 0.15,
+    "lines_changed": 0.10,
+}
+
+
+def run_command(cmd: str, cwd: Path, timeout: int = 120) -> tuple[int, str]:
+    """Run a shell command, return (exit_code, output)."""
+    try:
+        proc = subprocess.run(
+            cmd,
+            shell=True,
+            cwd=cwd,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+        output = (proc.stdout + "\n" + proc.stderr).strip()
+        return proc.returncode, output
+    except subprocess.TimeoutExpired:
+        return -1, "Command timed out"
+    except Exception as e:
+        return -1, str(e)
+
+
+def compute_score(
+    result: AgentResult,
+    test_command: str,
+    lint_command: str | None,
+    workdir: Path,
+    diff_lines: int,
+    all_wall_times: list[float],
+    all_diff_lines: list[int],
+) -> Score:
+    """Compute a composite score for an agent result."""
+    breakdown = ScoreBreakdown()
+
+    # Tests
+    test_exit, test_output = run_command(test_command, workdir)
+    breakdown.tests_pass = test_exit == 0
+
+    # Exit clean
+    breakdown.exit_clean = result.exit_code == 0 and not result.timed_out
+
+    # Lint
+    lint_output = ""
+    if lint_command:
+        lint_exit, lint_output = run_command(lint_command, workdir)
+        breakdown.lint_clean = lint_exit == 0
+    else:
+        breakdown.lint_clean = True  # No lint command = pass
+
+    # Raw metrics
+    breakdown.wall_time = result.wall_time
+    breakdown.lines_changed = diff_lines
+
+    # Compute composite
+    composite = 0.0
+
+    # Boolean metrics
+    composite += WEIGHTS["tests_pass"] * (100.0 if breakdown.tests_pass else 0.0)
+    composite += WEIGHTS["exit_clean"] * (100.0 if breakdown.exit_clean else 0.0)
+    composite += WEIGHTS["lint_clean"] * (100.0 if breakdown.lint_clean else 0.0)
+
+    # Wall time: normalize (lower is better)
+    composite += WEIGHTS["wall_time"] * _normalize_lower_better(
+        result.wall_time, all_wall_times
+    )
+
+    # Lines changed: normalize (fewer is better)
+    composite += WEIGHTS["lines_changed"] * _normalize_lower_better(
+        float(diff_lines), [float(x) for x in all_diff_lines]
+    )
+
+    return Score(
+        agent=result.agent,
+        composite=round(composite, 1),
+        breakdown=breakdown,
+        tests_output=test_output,
+        lint_output=lint_output,
+    )
+
+
+def _normalize_lower_better(value: float, all_values: list[float]) -> float:
+    """Normalize a metric where lower is better. Returns 0-100 score."""
+    if not all_values:
+        return 50.0
+
+    valid = [v for v in all_values if v > 0]
+    if not valid:
+        return 50.0
+
+    if len(valid) == 1:
+        return 100.0 if value == valid[0] else 50.0
+
+    min_val = min(valid)
+    max_val = max(valid)
+
+    if max_val == min_val:
+        return 100.0
+
+    if value <= 0:
+        return 50.0
+
+    # Invert: best (min) gets 100, worst (max) gets 0
+    normalized = 100.0 * (1.0 - (value - min_val) / (max_val - min_val))
+    return max(0.0, min(100.0, normalized))

coderace/task.py

@@ -0,0 +1,70 @@
+"""Task definition loading and validation."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+from coderace.types import Task
+
+
+def load_task(path: str | Path) -> Task:
+    """Load a task from a YAML file."""
+    path = Path(path)
+    if not path.exists():
+        raise FileNotFoundError(f"Task file not found: {path}")
+
+    with open(path) as f:
+        data = yaml.safe_load(f)
+
+    if not isinstance(data, dict):
+        raise ValueError(f"Task file must contain a YAML mapping, got {type(data).__name__}")
+
+    required = {"name", "description", "test_command", "agents"}
+    missing = required - set(data.keys())
+    if missing:
+        raise ValueError(f"Missing required fields: {', '.join(sorted(missing))}")
+
+    repo_str = data.get("repo", ".")
+    repo = Path(repo_str)
+    if not repo.is_absolute():
+        repo = path.parent / repo
+
+    task = Task(
+        name=data["name"],
+        description=data["description"],
+        repo=repo.resolve(),
+        test_command=data["test_command"],
+        agents=data["agents"],
+        lint_command=data.get("lint_command"),
+        timeout=data.get("timeout", 300),
+    )
+
+    errors = task.validate()
+    if errors:
+        raise ValueError(f"Invalid task: {'; '.join(errors)}")
+
+    return task
+
+
+def create_template(name: str, output_dir: Path | None = None) -> Path:
+    """Create a task YAML template."""
+    output_dir = output_dir or Path.cwd()
+    path = output_dir / f"{name}.yaml"
+
+    template = f"""name: {name}
+description: |
+  Describe the task here. What should the agent fix or build?
+  Be specific about which files to change and what the expected behavior is.
+repo: .
+test_command: pytest tests/ -x
+lint_command: ruff check .
+timeout: 300
+agents:
+  - claude
+  - codex
+  - aider
+"""
+    path.write_text(template)
+    return path

coderace/types.py

@@ -0,0 +1,73 @@
+"""Shared types for coderace."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass
+class Task:
+    """A task definition loaded from YAML."""
+
+    name: str
+    description: str
+    repo: Path
+    test_command: str
+    agents: list[str]
+    lint_command: str | None = None
+    timeout: int = 300
+
+    def validate(self) -> list[str]:
+        """Return list of validation errors, empty if valid."""
+        errors: list[str] = []
+        if not self.name:
+            errors.append("Task name is required")
+        if not self.description:
+            errors.append("Task description is required")
+        if not self.test_command:
+            errors.append("test_command is required")
+        if not self.agents:
+            errors.append("At least one agent is required")
+        known = {"claude", "codex", "aider", "gemini"}
+        for agent in self.agents:
+            if agent not in known:
+                errors.append(f"Unknown agent: {agent!r} (known: {', '.join(sorted(known))})")
+        if self.timeout < 1:
+            errors.append("timeout must be positive")
+        return errors
+
+
+@dataclass
+class AgentResult:
+    """Raw result from running an agent on a task."""
+
+    agent: str
+    exit_code: int
+    stdout: str
+    stderr: str
+    wall_time: float
+    timed_out: bool = False
+
+
+@dataclass
+class ScoreBreakdown:
+    """Per-metric scores before weighting."""
+
+    tests_pass: bool = False
+    exit_clean: bool = False
+    lint_clean: bool = False
+    wall_time: float = 0.0
+    lines_changed: int = 0
+
+
+@dataclass
+class Score:
+    """Final computed score for an agent run."""
+
+    agent: str
+    composite: float
+    breakdown: ScoreBreakdown = field(default_factory=ScoreBreakdown)
+    tests_output: str = ""
+    lint_output: str = ""
+    diff_stat: str = ""

coderace-0.1.0.dist-info/METADATA

@@ -0,0 +1,142 @@
+Metadata-Version: 2.4
+Name: coderace
+Version: 0.1.0
+Summary: Race coding agents against each other on real tasks
+Project-URL: Homepage, https://github.com/mikiships/coderace
+Project-URL: Repository, https://github.com/mikiships/coderace
+Author: mikiships
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai,aider,benchmark,claude,codex,coding-agents
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0
+Requires-Dist: typer>=0.9.0
+Provides-Extra: dev
+Requires-Dist: pytest-mock>=3.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.4.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# coderace
+
+Race coding agents against each other on real tasks in your repo.
+
+Define a task. Run it against Claude Code, Codex, and Aider. Get a scored comparison table.
+
+## Install
+
+```bash
+pip install coderace
+```
+
+## Quick Start
+
+```bash
+# Create a task template
+coderace init fix-auth-bug
+
+# Edit the task file (describe the bug, set test command)
+# Then race the agents:
+coderace run fix-auth-bug.yaml
+
+# Or race them in parallel (uses git worktrees):
+coderace run fix-auth-bug.yaml --parallel
+
+# View results from the last run
+coderace results fix-auth-bug.yaml
+```
+
+## Task Format
+
+```yaml
+name: fix-auth-bug
+description: |
+  The login endpoint returns 500 when email contains a plus sign.
+  Fix the email validation in auth/validators.py.
+repo: .
+test_command: pytest tests/test_auth.py -x
+lint_command: ruff check .
+timeout: 300
+agents:
+  - claude
+  - codex
+  - aider
+```
+
+## What It Does
+
+For each agent in the task:
+
+1. Creates a fresh git branch (`coderace/<agent>-<task>`)
+2. Invokes the agent CLI with the task description
+3. Runs your test command
+4. Runs your lint command (optional)
+5. Computes a composite score
+
+## Scoring
+
+| Metric | Weight | Description |
+|--------|--------|-------------|
+| Tests pass | 40% | Did the test command exit 0? |
+| Exit clean | 20% | Did the agent itself exit 0 without timeout? |
+| Lint clean | 15% | Did the lint command exit 0? |
+| Wall time | 15% | Faster is better (normalized across agents) |
+| Lines changed | 10% | Fewer is better (normalized across agents) |
+
+## Output
+
+Terminal table with Rich formatting:
+
+```
+┌──────┬────────┬───────┬───────┬──────┬──────┬──────────┬───────┐
+│ Rank │ Agent  │ Score │ Tests │ Exit │ Lint │ Time (s) │ Lines │
+├──────┼────────┼───────┼───────┼──────┼──────┼──────────┼───────┤
+│  1   │ claude │  85.0 │ PASS  │ PASS │ PASS │     10.5 │    42 │
+│  2   │ codex  │  70.0 │ PASS  │ PASS │ FAIL │     15.2 │    98 │
+│  3   │ aider  │  55.0 │ FAIL  │ PASS │ PASS │      8.1 │    31 │
+└──────┴────────┴───────┴───────┴──────┴──────┴──────────┴───────┘
+```
+
+Results also saved as JSON in `.coderace/<task>-results.json`.
+
+## Supported Agents
+
+| Agent | CLI | Command |
+|-------|-----|---------|
+| Claude Code | `claude` | `claude --print --output-format json -p "<task>"` |
+| Codex | `codex` | `codex --quiet --full-auto -p "<task>"` |
+| Aider | `aider` | `aider --message "<task>" --yes --no-auto-commits` |
+| Gemini CLI | `gemini` | `gemini --non-interactive -p "<task>"` |
+
+Each agent must be installed and authenticated separately.
+
+## Parallel Mode
+
+Use `--parallel` (or `-p`) to run all agents simultaneously using git worktrees. Each agent gets its own isolated working directory, so they don't interfere with each other.
+
+```bash
+coderace run task.yaml --parallel
+```
+
+Sequential mode (default) runs agents one at a time on the same repo.
+
+## Requirements
+
+- Python 3.10+
+- Git
+- At least one coding agent CLI installed
+
+## License
+
+MIT

coderace-0.1.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+coderace/__init__.py,sha256=wSg2jh_3Y65Nfh0k8INcZ8UXs4_tnd5FkzNFUQfld-c,93
+coderace/cli.py,sha256=EhxDadkkewIEPvtu3wQpVh7RP_LR6qI6G4KpPr-JHQ8,9933
+coderace/git_ops.py,sha256=UmCmMdEiI8vvpUQwiJLk9Vy5KRNSKlI3w1FusUVdSdY,4155
+coderace/reporter.py,sha256=FCgeHv3w0_kdqs_75j4iVsMRLbKDlXVICzarf33zao4,3021
+coderace/scorer.py,sha256=UpcpkKpuvcDKy1fOuBZtT9Y9si3-lXGFseUfS1cqP0Y,3284
+coderace/task.py,sha256=CXT0uYT5H_31mSSRl76jF3xpvIWQr23MYqVJ4L3vTPs,1833
+coderace/types.py,sha256=XHiL2mjTv1rQ4PfnIqAlmA-3jwiixVF-5y_5JrpSMPI,1846
+coderace/adapters/__init__.py,sha256=bgXcH-bsJuWALSfx4DjFOy1aHIJX589gn4b2hv4OvCI,573
+coderace/adapters/aider.py,sha256=HQMaZoQO9ruRGKeT4YixATVfDTU28-HMgGBQSPDMC3o,429
+coderace/adapters/base.py,sha256=aFE_FG-cVN5EKmu7p6Mbmz5LtFhMF-IpqiowyHVgSDA,1757
+coderace/adapters/claude.py,sha256=Cb3fgEEyPbNMlUb9QDBcdKEUAT-E-0HrdQdDtM0gf6Y,444
+coderace/adapters/codex.py,sha256=K6EHDlTDyFFozpU52sWjxq9oQlB2l8gEbK4G-QK4SHU,416
+coderace/adapters/gemini.py,sha256=LUG9Pgf0jM3GrlDpra0VLDNFrzJzdVmlhz5-C1rFhgQ,404
+coderace-0.1.0.dist-info/METADATA,sha256=-SDIOSsf9lKjhfjtWH2pUEBDFVhVD9X-ZqB4KMRbhzM,4538
+coderace-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+coderace-0.1.0.dist-info/entry_points.txt,sha256=2Mg40OC1qw_gQDf51mcIyW5nL323ETJcLYJSsztcUdY,46
+coderace-0.1.0.dist-info/licenses/LICENSE,sha256=_xg5HZ1WlLJEQpwpHZeQrhtyhgKC7ke4BL_qH6U3Jzo,1066
+coderace-0.1.0.dist-info/RECORD,,

coderace-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

coderace-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+coderace = coderace.cli:app

coderace-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 mikiships
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.