kodo-agent 0.5.0__py3-none-any.whl

benchmark/__main__.py

@@ -0,0 +1,426 @@
+"""SWE-bench benchmark: kodo vs raw Claude Code / Cursor / Codex / Gemini.
+
+By default, connects to the central server (KODO_BENCH_URL) to receive task
+assignments and auto-detects available backends. Use --local for standalone runs.
+
+Usage:
+    uv run python -m benchmark                           # server decides everything
+    uv run python -m benchmark --local --subset benchmark/subsets/pro-20.json
+    uv run python -m benchmark --local --arm kodo:solo --limit 2 --skip-eval
+"""
+
+from __future__ import annotations
+
+# Suppress noisy urllib3/chardet version mismatch warning from requests
+# (triggered transitively via datasets/swebench imports).
+import warnings
+
+warnings.filterwarnings(
+    "ignore",
+    message=r"urllib3.*doesn't match a supported version",
+)
+
+from dotenv import find_dotenv, load_dotenv
+
+load_dotenv(find_dotenv(usecwd=True))
+
+import argparse
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from benchmark._util import detect_backends, log, setup_logging
+
+WORKSPACE = Path.home() / ".kodo" / "benchmark"
+
+
+def main() -> int:
+    """CLI entrypoint for the SWE-bench benchmark harness."""
+    parser = argparse.ArgumentParser(
+        description="SWE-bench benchmark: kodo vs raw Claude Code",
+    )
+
+    # Dataset and task selection
+    parser.add_argument(
+        "--dataset",
+        choices=["pro", "verified", "lite"],
+        default="pro",
+        help="SWE-bench variant (default: pro)",
+    )
+    parser.add_argument(
+        "--subset",
+        type=Path,
+        default=None,
+        help="Path to a subset JSON file (e.g. subsets/pro-20.json). "
+        "Overrides --dataset and --instance-ids.",
+    )
+    parser.add_argument("--limit", type=int, default=None, help="Run first N tasks")
+    parser.add_argument(
+        "--instance-ids", nargs="+", default=None, help="Specific instance IDs"
+    )
+    parser.add_argument(
+        "--repo",
+        type=str,
+        default=None,
+        help="Filter to repo (e.g. 'ansible/ansible')",
+    )
+    parser.add_argument(
+        "--language",
+        type=str,
+        default=None,
+        help="Filter by language (e.g. 'python', 'go', 'js'). Pro only.",
+    )
+    parser.add_argument("--offset", type=int, default=0, help="Skip first N tasks")
+
+    # Arm selection
+    parser.add_argument(
+        "--arm",
+        action="append",
+        default=None,
+        help="Arm to benchmark. Repeatable. 'claude', 'cursor', 'codex', "
+        "'gemini' for raw CLI tools; 'kodo' for default team, "
+        "'kodo:<team>' for a specific team (e.g. 'kodo:quick'). "
+        "Default: claude + kodo.",
+    )
+
+    # Execution
+    parser.add_argument(
+        "--timeout",
+        type=int,
+        default=7200,
+        help="Per-task timeout for non-orchestrated arms in seconds (default: 7200 / 2h)",
+    )
+    parser.add_argument(
+        "--timeout-kodo",
+        type=int,
+        default=43200,
+        help="Per-task timeout for kodo arms in seconds (default: 43200 / 12h)",
+    )
+    parser.add_argument(
+        "--workspace", type=Path, default=WORKSPACE, help="Workspace directory"
+    )
+    parser.add_argument(
+        "--run-id", type=str, default=None, help="Resume or reference a run ID"
+    )
+    parser.add_argument(
+        "--parallel", type=int, default=1, help="Concurrent tasks (default: 1)"
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=0,
+        help="Seed for deduplication. Same task+arm+seed won't re-run. "
+        "Use different seeds to get multiple runs of the same tasks (default: 0).",
+    )
+
+    # Phase control
+    parser.add_argument(
+        "--status",
+        action="store_true",
+        help="Show status of all benchmark runs and exit",
+    )
+    parser.add_argument(
+        "--skip-eval", action="store_true", help="Skip swebench evaluation"
+    )
+    parser.add_argument(
+        "--evaluate-only",
+        action="store_true",
+        help="Only evaluate existing predictions",
+    )
+    parser.add_argument(
+        "--report-only",
+        action="store_true",
+        help="Only generate report from existing results",
+    )
+    parser.add_argument(
+        "--publish",
+        action="store_true",
+        help="Publish results to GitHub Pages for the online viewer",
+    )
+    parser.add_argument(
+        "--extract-patch",
+        nargs=2,
+        metavar=("INSTANCE_ID", "ARM"),
+        help="Print a patch from published data",
+    )
+    parser.add_argument(
+        "--upload-pending",
+        action="store_true",
+        help="Upload results not yet sent to the online server (requires KODO_BENCH_URL/TOKEN)",
+    )
+    parser.add_argument(
+        "--evaluate-pending",
+        action="store_true",
+        help="Fetch unevaluated predictions from the online server and run Docker-based "
+        "swebench evaluation locally. Uploads results back when done. "
+        "(Requires KODO_BENCH_URL/TOKEN and Docker.)",
+    )
+    parser.add_argument(
+        "--mirror-online",
+        action="store_true",
+        help="Mirror public online benchmark data into local JSON files for plotting",
+    )
+    parser.add_argument(
+        "--mirror-out",
+        type=Path,
+        default=WORKSPACE / "mirror",
+        help="Output directory for --mirror-online (default: ~/.kodo/benchmark/mirror)",
+    )
+    parser.add_argument(
+        "--mirror-patches",
+        action="store_true",
+        help="With --mirror-online, also download patches.json",
+    )
+
+    # Mode
+    parser.add_argument(
+        "--local",
+        action="store_true",
+        help="Run locally instead of connecting to the central server. "
+        "Required when KODO_BENCH_URL/TOKEN are not set.",
+    )
+    parser.add_argument(
+        "--backends",
+        type=str,
+        default=None,
+        help="Override backend detection (e.g. 'claude,kodo:solo'). "
+        "Default: auto-detect from PATH.",
+    )
+
+    args = parser.parse_args()
+    setup_logging()
+    workspace: Path = args.workspace
+    workspace.mkdir(parents=True, exist_ok=True)
+
+    # UTC timestamp as run ID
+    run_id = args.run_id or datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+    arms = args.arm if args.arm else ["claude", "kodo"]
+
+    if args.status:
+        from benchmark.report import print_status
+
+        return print_status(workspace)
+
+    if args.publish:
+        from benchmark.online.publish import publish_results
+
+        return publish_results(workspace, run_id=args.run_id)
+
+    if args.extract_patch:
+        from benchmark.online.publish import extract_patch
+
+        return extract_patch(args.extract_patch[0], args.extract_patch[1])
+
+    if args.upload_pending:
+        from benchmark.online.upload_tracker import flush_pending_uploads
+
+        return flush_pending_uploads(workspace)
+
+    if args.evaluate_pending:
+        from benchmark.evaluate_pending import evaluate_pending
+
+        return evaluate_pending(workspace, dataset_arg=args.dataset, arms=args.arm)
+
+    if args.mirror_online:
+        from benchmark.online.mirror import mirror_dataset
+
+        mirror_dataset(
+            args.dataset,
+            out_dir=args.mirror_out,
+            include_patches=args.mirror_patches,
+        )
+        return 0
+
+    if args.report_only:
+        from benchmark.report import generate_report
+
+        return generate_report(workspace, run_id)
+
+    if args.evaluate_only:
+        from benchmark.evaluate import evaluate_predictions
+        from benchmark.report import generate_report
+
+        evaluate_predictions(workspace, run_id)
+        return generate_report(workspace, run_id)
+
+    # Distribute mode: check before heavy task-loading imports.
+    # Report/evaluate-only runs operate on local artifacts and should not poll
+    # the server or load the task pool first.
+    local_mode = args.local or args.subset or args.instance_ids
+    if not local_mode:
+        from benchmark.online.client import is_configured, whoami
+
+        if is_configured():
+            identity = whoami()
+            if identity:
+                log.info("Authenticated as: %s", identity)
+            return _run_distributed(args, workspace, run_id)
+
+    from benchmark.evaluate import evaluate_predictions
+    from benchmark.report import generate_report
+
+    # Run agents
+    import json
+
+    from benchmark.runner import BenchmarkInterrupted, run_benchmark
+    from benchmark.tasks import DATASET_MAP, load_tasks
+
+    # Resolve dataset and instance_ids from --subset if provided
+    instance_ids = args.instance_ids
+    dataset = DATASET_MAP[args.dataset]
+    if args.subset:
+        subset_data = json.loads(args.subset.read_text())
+        instance_ids = subset_data["instance_ids"]
+        dataset = subset_data.get("dataset", dataset)
+
+    # Local mode (distribute path handled earlier)
+    tasks = load_tasks(
+        dataset=dataset,
+        limit=args.limit,
+        instance_ids=instance_ids,
+        repo_filter=args.repo,
+        language=args.language,
+        offset=args.offset,
+    )
+
+    if not tasks:
+        log.error("No tasks matched the filters.")
+        return 1
+
+    log.info("Running %d tasks", len(tasks))
+
+    try:
+        run_benchmark(
+            tasks=tasks,
+            arms=arms,
+            workspace=workspace,
+            run_id=run_id,
+            timeout=args.timeout,
+            timeout_kodo=args.timeout_kodo,
+            parallel=args.parallel,
+            dataset=dataset,
+            seed=args.seed,
+        )
+    except BenchmarkInterrupted as exc:
+        return _print_interrupted(exc.completed_count)
+
+    if not args.skip_eval:
+        evaluate_predictions(workspace, run_id)
+
+    return generate_report(workspace, run_id)
+
+
+def _run_distributed(args: argparse.Namespace, workspace: Path, run_id: str) -> int:
+    """Poll central server for task assignments and run them in batches."""
+    from benchmark.online.client import fetch_assignments
+    from benchmark.runner import BenchmarkInterrupted, run_benchmark
+    from benchmark.tasks import DATASET_MAP, DATASET_PRO, load_tasks
+
+    # Backends: explicit --backends > explicit --arm > auto-detect
+    if args.backends:
+        backends = args.backends.split(",")
+    elif args.arm:
+        backends = args.arm
+    else:
+        backends = detect_backends()
+        log.info("Detected agents: %s", ", ".join(backends))
+
+    # Load only the requested dataset
+    all_datasets: dict[str, list[str]] = {}
+    all_tasks: dict[str, list] = {}  # instance_id -> task
+    ds_key = args.dataset
+    ds_name = DATASET_MAP[ds_key]
+    ds_tasks = load_tasks(dataset=ds_name)
+    all_datasets[ds_key] = [t.instance_id for t in ds_tasks]
+    for t in ds_tasks:
+        all_tasks[t.instance_id] = t
+    total_tasks = sum(len(v) for v in all_datasets.values())
+    log.info(
+        "Task pool: %d tasks (%s)",
+        total_tasks,
+        ", ".join(f"{k}: {len(v)}" for k, v in all_datasets.items()),
+    )
+
+    batch_size = args.limit or 20
+    total_completed = 0
+
+    try:
+        while True:
+            try:
+                assignments = fetch_assignments(
+                    backends=backends,
+                    datasets=all_datasets,
+                    limit=batch_size,
+                )
+            except Exception as exc:
+                log.error(
+                    "Failed to get assignments from %s: %s",
+                    os.environ.get("KODO_BENCH_URL", "(not set)"),
+                    exc,
+                )
+                return 1 if total_completed == 0 else 0
+
+            if not assignments:
+                if total_completed == 0:
+                    log.info("No tasks need evaluation — all covered!")
+                else:
+                    log.info("No more tasks. Completed %d total.", total_completed)
+                return 0
+
+            unique_ids = dict.fromkeys(a["instance_id"] for a in assignments)
+            tasks = [all_tasks[iid] for iid in unique_ids if iid in all_tasks]
+            arms = list({a["arm"] for a in assignments})
+            ds_keys = {a.get("dataset", "pro") for a in assignments}
+            dataset = DATASET_MAP.get(next(iter(ds_keys)), DATASET_PRO)
+            unique_tasks = len({a["instance_id"] for a in assignments})
+            log.info(
+                "Received %d tasks x %d agents (%s) from %s",
+                unique_tasks,
+                len(arms),
+                ", ".join(arms),
+                "/".join(ds_keys),
+            )
+
+            run_benchmark(
+                tasks=tasks,
+                arms=arms,
+                workspace=workspace,
+                run_id=run_id,
+                timeout=args.timeout,
+                timeout_kodo=args.timeout_kodo,
+                parallel=args.parallel,
+                dataset=dataset,
+                seed=args.seed,
+                assignments=assignments,
+            )
+            total_completed += len(assignments)
+            log.info(
+                "Batch done. %d completed so far, polling for more...", total_completed
+            )
+    except (KeyboardInterrupt, BenchmarkInterrupted) as exc:
+        n = exc.completed_count if isinstance(exc, BenchmarkInterrupted) else 0
+        return _print_interrupted(total_completed + n)
+
+
+def _print_interrupted(completed: int) -> int:
+    """Print a clean summary on Ctrl+C."""
+    print()  # newline after ^C
+    if completed > 1:
+        log.info(
+            "Interrupted. %d tasks completed and uploaded. Thanks for contributing!",
+            completed,
+        )
+    elif completed == 1:
+        log.info("Interrupted. 1 task completed and uploaded.")
+    else:
+        log.info("Interrupted. No tasks completed.")
+    log.info("Claimed tasks will be reassigned automatically.")
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        sys.exit(_print_interrupted(0))

benchmark/_util.py

@@ -0,0 +1,188 @@
+"""Shared helpers for the benchmark package."""
+
+from __future__ import annotations
+
+import json
+import logging
+import platform
+import re
+import shutil
+import subprocess
+import time
+from pathlib import Path
+
+log = logging.getLogger("benchmark")
+
+# Arm name sanitization — used for filenames and Docker container names.
+# Reversible: ":" → "--" (unlike the old ":" → "_" which was lossy).
+_UNSAFE_RE = re.compile(r"[^a-zA-Z0-9_.-]")
+
+
+def fmt_duration(seconds: int) -> str:
+    """Format seconds as human-readable duration: 7200 -> '2h', 300 -> '5m'."""
+    if seconds >= 3600:
+        h = seconds / 3600
+        return f"{h:.0f}h" if h == int(h) else f"{h:.1f}h"
+    if seconds >= 60:
+        return f"{seconds // 60}m"
+    return f"{seconds}s"
+
+
+def short_iid(instance_id: str) -> str:
+    """Shorten instance_id for display: 'django__django-13195' -> 'django/django#13195'."""
+    parts = instance_id.split("__", 1)
+    if len(parts) != 2:
+        return instance_id
+    owner = parts[0].replace("_", "-")
+    rest = parts[1]
+    dash_idx = rest.rfind("-")
+    if dash_idx > 0:
+        repo = rest[:dash_idx].replace("_", "-")
+        issue = rest[dash_idx + 1 :]
+        # Truncate long hashes (e.g. Go SWE-bench commit SHAs)
+        if len(issue) > 12:
+            issue = issue[:8]
+        return f"{owner}/{repo}#{issue}"
+    return f"{owner}/{rest}"
+
+
+def docker_safe(name: str) -> str:
+    """Replace chars invalid in Docker container names with underscores."""
+    return _UNSAFE_RE.sub("_", name)
+
+
+def load_json(path: Path) -> dict:
+    """Load a JSON file, returning {} on missing/corrupt files."""
+    if path.exists():
+        try:
+            return json.loads(path.read_text())
+        except (json.JSONDecodeError, OSError):
+            log.warning("Failed to parse %s", path)
+    return {}
+
+
+def load_jsonl(path: Path) -> list[dict]:
+    """Load a JSONL file line-by-line, skipping bad lines."""
+    results: list[dict] = []
+    if not path.exists():
+        return results
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                results.append(json.loads(line))
+            except json.JSONDecodeError:
+                log.warning("Skipping bad JSONL line in %s", path)
+    return results
+
+
+def iter_jsonl(path: Path):
+    """Iterate over JSONL lines without loading all into memory."""
+    if not path.exists():
+        return
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                yield json.loads(line)
+            except json.JSONDecodeError:
+                log.warning("Skipping bad JSONL line in %s", path)
+
+
+def setup_logging(verbose: bool = False) -> None:
+    """Configure logging for benchmark runs."""
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(
+        format="%(asctime)s %(levelname)-5s %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+        level=level,
+    )
+
+
+# CLI tool name → arm name(s).  kodo is always available (it's this project).
+_BACKEND_CLI_MAP: list[tuple[str, list[str]]] = [
+    ("claude", ["claude"]),
+    ("cursor-agent", ["cursor"]),
+    ("codex", ["codex"]),
+    ("gemini", ["gemini"]),
+]
+
+
+def ensure_docker_running(timeout: int = 60) -> bool:
+    """Check if Docker daemon is running; attempt to start it if not.
+
+    Returns True if Docker is available, False otherwise.
+    On macOS, tries OrbStack first, then Docker Desktop.
+    """
+    if _docker_is_ready():
+        return True
+
+    log.info("Docker daemon is not running. Attempting to start...")
+
+    if platform.system() != "Darwin":
+        log.warning(
+            "Docker is not running. Start it manually:\n  sudo systemctl start docker"
+        )
+        return False
+
+    if not _start_docker_macos():
+        log.error("Could not start Docker. Please start it manually.")
+        return False
+
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if _docker_is_ready():
+            log.info("Docker is now running.")
+            return True
+        time.sleep(2)
+
+    log.error("Docker did not become ready within %ds.", timeout)
+    return False
+
+
+def _docker_is_ready() -> bool:
+    """Return True if ``docker info`` succeeds."""
+    try:
+        result = subprocess.run(
+            ["docker", "info"],
+            capture_output=True,
+            timeout=10,
+        )
+        return result.returncode == 0
+    except (FileNotFoundError, subprocess.TimeoutExpired):
+        return False
+
+
+def _start_docker_macos() -> bool:
+    """Try to start Docker on macOS via OrbStack or Docker Desktop."""
+    if shutil.which("orbctl"):
+        log.info("Starting Docker via OrbStack...")
+        try:
+            subprocess.run(["orbctl", "start"], check=True, timeout=30)
+            return True
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
+            log.warning("OrbStack start failed, trying Docker Desktop...")
+
+    try:
+        subprocess.run(["open", "-a", "Docker"], check=True, timeout=10)
+        log.info("Starting Docker Desktop (this may take 30-60s)...")
+        return True
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return False
+
+
+def detect_backends() -> list[str]:
+    """Auto-detect which benchmark backends are available on this machine.
+
+    Checks PATH for each CLI tool.  ``kodo`` is always included since it's
+    the project itself (runs via ``uv run kodo``).
+    """
+    found: list[str] = ["kodo"]
+    for cli_name, arm_names in _BACKEND_CLI_MAP:
+        if shutil.which(cli_name):
+            found.extend(arm_names)
+    return found