agentbox-rl 0.2.0__py3-none-any.whl

agentbox/benchmark/report.py

@@ -0,0 +1,175 @@
+"""Build and render benchmark reports."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from collections.abc import Sequence
+from typing import Any
+
+from agentbox.benchmark.schema import (
+    BenchmarkReport,
+    ModelReport,
+    TaskResult,
+)
+from agentbox.metrics.aggregate import AggregateMetrics, aggregate_trajectories
+from agentbox.trajectory.schema import Trajectory
+from agentbox.types import FinalStatus
+
+
+def redact_model_config(cfg: dict[str, Any]) -> dict[str, Any]:
+    out = dict(cfg)
+    if "api_key" in out:
+        out["api_key"] = "***" if out["api_key"] else None
+    return out
+
+
+def build_task_results(
+    model_id: str,
+    trajs: Sequence[Trajectory],
+    *,
+    trajectory_paths: dict[str, list[str]] | None = None,
+) -> list[TaskResult]:
+    by_task: dict[str, list[Trajectory]] = defaultdict(list)
+    for t in trajs:
+        by_task[t.task_id].append(t)
+
+    results: list[TaskResult] = []
+    for task_id, group in sorted(by_task.items()):
+        n = len(group)
+        successes = sum(1 for x in group if x.final_status == FinalStatus.SUCCESS)
+        statuses: dict[str, int] = defaultdict(int)
+        seed_errors = 0
+        setup_fails = 0
+        for x in group:
+            statuses[x.final_status.value] += 1
+            err = (x.error or "").lower()
+            if x.final_status == FinalStatus.ERROR:
+                if "setup check" in err:
+                    setup_fails += 1
+                elif "seed" in err:
+                    seed_errors += 1
+        pass_at_k = 1.0 if successes > 0 else 0.0
+        results.append(
+            TaskResult(
+                task_id=task_id,
+                model_id=model_id,
+                trajectory_paths=(trajectory_paths or {}).get(task_id, []),
+                n=n,
+                successes=successes,
+                pass_at_1=successes / n if n else 0.0,
+                pass_at_k=pass_at_k,
+                mean_reward=sum(x.reward for x in group) / n if n else 0.0,
+                mean_steps=sum(x.metrics.steps for x in group) / n if n else 0.0,
+                mean_duration_s=sum(x.metrics.duration_s for x in group) / n if n else 0.0,
+                seed_errors=seed_errors,
+                setup_check_failures=setup_fails,
+                statuses=dict(statuses),
+            )
+        )
+    return results
+
+
+def build_model_report(
+    model_id: str,
+    model_config_public: dict[str, Any],
+    trajs: Sequence[Trajectory],
+    *,
+    wall_clock_s: float,
+    trajectory_paths: dict[str, list[str]] | None = None,
+) -> ModelReport:
+    by_task = build_task_results(
+        model_id, trajs, trajectory_paths=trajectory_paths
+    )
+    agg = aggregate_trajectories(trajs)
+    pass_at_k_mean = (
+        sum(t.pass_at_k or 0.0 for t in by_task) / len(by_task) if by_task else None
+    )
+    return ModelReport(
+        model_id=model_id,
+        model_config_public=redact_model_config(model_config_public),
+        aggregate=agg,
+        by_task=by_task,
+        wall_clock_s=wall_clock_s,
+        pass_at_k_mean=pass_at_k_mean,
+    )
+
+
+def build_leaderboard(models: Sequence[ModelReport]) -> list[dict[str, Any]]:
+    rows = []
+    for m in models:
+        rows.append(
+            {
+                "model_id": m.model_id,
+                "success_rate": m.aggregate.success_rate,
+                "mean_reward": m.aggregate.mean_reward,
+                "mean_steps": m.aggregate.mean_steps,
+                "mean_duration_s": m.aggregate.mean_duration_s,
+                "pass_at_k_mean": m.pass_at_k_mean,
+                "n": m.aggregate.n,
+                "wall_clock_s": m.wall_clock_s,
+            }
+        )
+    rows.sort(key=lambda r: (-r["success_rate"], -r["mean_reward"], r["mean_steps"]))
+    return rows
+
+
+def render_markdown(report: BenchmarkReport) -> str:
+    lines: list[str] = [
+        f"# Benchmark Report: {report.suite_id}",
+        "",
+        f"- **Suite version:** {report.suite_version}",
+        f"- **Content hash:** `{report.suite_content_hash or 'none'}`",
+        f"- **AgentBox:** {report.agentbox_version}",
+        f"- **Report id:** {report.report_id}",
+        f"- **Started:** {report.started_at.isoformat()}",
+        f"- **Finished:** {report.finished_at.isoformat()}",
+        "",
+        "## Leaderboard",
+        "",
+        "| Rank | Model | Success rate | Mean reward | Mean steps | Mean duration (s) | Pass@k | n |",
+        "| --- | --- | --- | --- | --- | --- | --- | --- |",
+    ]
+    for i, row in enumerate(report.leaderboard, start=1):
+        pak = row.get("pass_at_k_mean")
+        pak_s = f"{pak:.3f}" if pak is not None else "—"
+        lines.append(
+            f"| {i} | {row['model_id']} | {row['success_rate']:.3f} | "
+            f"{row['mean_reward']:.3f} | {row['mean_steps']:.2f} | "
+            f"{row['mean_duration_s']:.2f} | {pak_s} | {row['n']} |"
+        )
+
+    for m in report.models:
+        lines.extend(
+            [
+                "",
+                f"## Model: {m.model_id}",
+                "",
+                f"- Config: `{m.model_config_public}`",
+                f"- Success rate: **{m.aggregate.success_rate:.3f}**",
+                f"- Mean reward: {m.aggregate.mean_reward:.3f}",
+                f"- Mean steps: {m.aggregate.mean_steps:.2f}",
+                f"- Statuses: {m.aggregate.by_status}",
+                f"- Wall clock: {m.wall_clock_s:.1f}s",
+                "",
+                "| Task | n | Successes | Pass@1 | Pass@k | Mean reward | Mean steps |",
+                "| --- | --- | --- | --- | --- | --- | --- |",
+            ]
+        )
+        for t in m.by_task:
+            pak = f"{t.pass_at_k:.3f}" if t.pass_at_k is not None else "—"
+            lines.append(
+                f"| {t.task_id} | {t.n} | {t.successes} | {t.pass_at_1:.3f} | "
+                f"{pak} | {t.mean_reward:.3f} | {t.mean_steps:.2f} |"
+            )
+
+    lines.extend(
+        [
+            "",
+            "## Notes",
+            "",
+            f"- Pass@k definition: {report.pass_at_k_definition}",
+            "- API keys are redacted from this report.",
+            "",
+        ]
+    )
+    return "\n".join(lines)

agentbox/benchmark/runner.py

@@ -0,0 +1,131 @@
+"""Execute frozen benchmark suites against model endpoints."""
+
+from __future__ import annotations
+
+import logging
+import time
+import uuid
+from collections import defaultdict
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from agentbox.benchmark.loader import snapshot_suite
+from agentbox.benchmark.report import build_leaderboard, build_model_report
+from agentbox.benchmark.schema import (
+    BenchmarkReport,
+    BenchmarkRunConfig,
+    ModelUnderTest,
+)
+from agentbox.config import AgentConfig, ModelConfig, RolloutConfig
+from agentbox.runner.parallel import ParallelRunner
+from agentbox.trajectory.schema import Trajectory
+from agentbox.version import __version__
+
+logger = logging.getLogger("agentbox.benchmark")
+
+
+class BenchmarkRunner:
+    """Run a BenchmarkSuite against one or more OpenAI-compatible models."""
+
+    def __init__(self, config: BenchmarkRunConfig) -> None:
+        self.config = config
+
+    async def run(self) -> BenchmarkReport:
+        cfg = self.config
+        suite = cfg.suite
+        suite.verify_integrity(strict=cfg.strict_integrity)
+
+        out = Path(cfg.output_dir)
+        out.mkdir(parents=True, exist_ok=True)
+        snapshot_suite(suite, out / "suite_snapshot")
+
+        started = datetime.now(timezone.utc)
+        model_reports = []
+        for mut in cfg.models:
+            logger.info("benchmark model_id=%s start", mut.model_id)
+            mr = await self._run_model(mut)
+            model_reports.append(mr)
+
+        finished = datetime.now(timezone.utc)
+        report = BenchmarkReport(
+            report_id=str(uuid.uuid4()),
+            suite_id=suite.manifest.suite_id,
+            suite_version=suite.manifest.version,
+            suite_content_hash=suite.manifest.content_hash,
+            agentbox_version=__version__,
+            started_at=started,
+            finished_at=finished,
+            run_tags=dict(cfg.tags),
+            models=model_reports,
+            leaderboard=build_leaderboard(model_reports),
+        )
+        report.save_json(out / "report.json")
+        report.save_markdown(out / "REPORT.md")
+        return report
+
+    async def _run_model(self, mut: ModelUnderTest) -> Any:
+        cfg = self.config
+        suite = cfg.suite
+        mdir = Path(cfg.output_dir) / "models" / mut.model_id
+        traj_dir = mdir / "trajectories"
+        traj_dir.mkdir(parents=True, exist_ok=True)
+
+        # Fairness: suite agent tools/max_steps only; model cannot change tools
+        agent_cfg = suite.manifest.agent.model_copy(
+            deep=True,
+            update={"custom_tools": [], "drop_tools_prob": 0.0},
+        )
+        concurrency = cfg.concurrency or suite.manifest.concurrency
+        n_per = cfg.n_per_task or suite.manifest.n_per_task
+        seed = suite.manifest.seed
+
+        rollout_config = RolloutConfig(
+            sandbox=suite.manifest.sandbox,
+            model=mut.model,
+            agent=agent_cfg,
+            seed=seed,
+        )
+
+        path_map: dict[str, list[str]] = defaultdict(list)
+        trajs_acc: list[Trajectory] = []
+
+        def on_trajectory(traj: Trajectory) -> None:
+            trajs_acc.append(traj)
+            if cfg.save_trajectories:
+                fname = f"{traj.task_id}_{traj.run_id[:8]}.json"
+                path = traj_dir / fname
+                traj.save(path)
+                path_map[traj.task_id].append(str(path))
+
+        # Thread setup_checks into each Rollout via ParallelRunner extension
+        runner = ParallelRunner(
+            concurrency=concurrency,
+            config=rollout_config,
+            model=mut.model,
+            agent=agent_cfg,
+            sandbox=suite.manifest.sandbox,
+        )
+        setup_checks = list(suite.manifest.scoring.setup_checks)
+
+        t0 = time.monotonic()
+        trajs = await runner.run_tasks(
+            suite.tasks,
+            n_per_task=n_per,
+            progress=cfg.progress,
+            on_trajectory=on_trajectory,
+            setup_checks=setup_checks,
+        )
+        wall = time.monotonic() - t0
+
+        public = mut.model.model_dump(mode="json")
+        report = build_model_report(
+            mut.model_id,
+            public,
+            trajs,
+            wall_clock_s=wall,
+            trajectory_paths=dict(path_map),
+        )
+        report_path = mdir / "model_report.json"
+        report_path.write_text(report.model_dump_json(indent=2), encoding="utf-8")
+        return report

agentbox/benchmark/schema.py

@@ -0,0 +1,198 @@
+"""Benchmark suite, run, and report schemas."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from agentbox.config import AgentConfig, ModelConfig, SandboxConfig
+from agentbox.metrics.aggregate import AggregateMetrics
+from agentbox.tasks.schema import Task
+from agentbox.trajectory.schema import Trajectory
+from agentbox.types import ToolMode
+
+
+class SetupCheckSpec(BaseModel):
+    """Post-seed environment health check (not task success)."""
+
+    name: str
+    command: str
+    success_exit_code: int = 0
+    timeout_s: float = 60.0
+    required: bool = True
+
+
+class ScoringConfig(BaseModel):
+    """How a suite scores rollouts."""
+
+    primary: Literal["verifier"] = "verifier"
+    require_seed_ok: bool = True
+    setup_checks: list[SetupCheckSpec] = Field(default_factory=list)
+    track_steps: bool = True
+    track_duration: bool = True
+    enable_ruler: bool = False
+    ruler_group_size: int = 4
+
+
+def _default_bench_agent() -> AgentConfig:
+    return AgentConfig(
+        tools=ToolMode.STRUCTURED,
+        max_steps=40,
+        drop_tools_prob=0.0,
+        custom_tools=[],
+    )
+
+
+class BenchmarkSuiteManifest(BaseModel):
+    """suite.json manifest for a frozen evaluation pack."""
+
+    schema_version: Literal["1"] = "1"
+    suite_id: str
+    name: str
+    description: str = ""
+    version: str = "1.0.0"
+    tasks_path: str = "tasks"
+    sandbox: SandboxConfig = Field(default_factory=SandboxConfig)
+    agent: AgentConfig = Field(default_factory=_default_bench_agent)
+    n_per_task: int = 1
+    concurrency: int = 8
+    seed: int | None = 42
+    scoring: ScoringConfig = Field(default_factory=ScoringConfig)
+    content_hash: str | None = None
+    created_at: datetime | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="ignore")
+
+
+class BenchmarkSuite(BaseModel):
+    """In-memory suite: manifest + loaded tasks."""
+
+    root: Path | None = None
+    manifest: BenchmarkSuiteManifest
+    tasks: list[Task] = Field(default_factory=list)
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    def task_ids(self) -> list[str]:
+        return [t.task_id for t in self.tasks]
+
+    def freeze(self) -> BenchmarkSuite:
+        from agentbox.benchmark.hash import compute_suite_content_hash
+
+        m = self.manifest.model_copy(deep=True)
+        m.created_at = m.created_at or datetime.now(timezone.utc)
+        m.content_hash = compute_suite_content_hash(self.tasks, m)
+        return BenchmarkSuite(root=self.root, manifest=m, tasks=list(self.tasks))
+
+    def verify_integrity(self, *, strict: bool = False) -> bool:
+        from agentbox.benchmark.hash import compute_suite_content_hash
+        from agentbox.errors import SuiteIntegrityError
+
+        if not self.manifest.content_hash:
+            if strict:
+                raise SuiteIntegrityError("Suite has no content_hash; run freeze()")
+            return True
+        current = compute_suite_content_hash(self.tasks, self.manifest)
+        ok = current == self.manifest.content_hash
+        if not ok and strict:
+            raise SuiteIntegrityError(
+                f"Suite integrity check failed: expected {self.manifest.content_hash}, got {current}"
+            )
+        return ok
+
+
+class ModelUnderTest(BaseModel):
+    """One model endpoint under evaluation."""
+
+    model_id: str
+    model: ModelConfig
+
+
+class BenchmarkRunConfig(BaseModel):
+    """Configuration for executing a suite against one or more models."""
+
+    suite: BenchmarkSuite
+    models: list[ModelUnderTest]
+    concurrency: int | None = None
+    n_per_task: int | None = None
+    save_trajectories: bool = True
+    output_dir: Path
+    progress: bool = True
+    fail_fast: bool = False
+    strict_integrity: bool = False
+    tags: dict[str, str] = Field(default_factory=dict)
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+class TaskResult(BaseModel):
+    task_id: str
+    model_id: str
+    trajectory_paths: list[str] = Field(default_factory=list)
+    n: int = 0
+    successes: int = 0
+    pass_at_1: float = 0.0
+    pass_at_k: float | None = None
+    mean_reward: float = 0.0
+    mean_steps: float = 0.0
+    mean_duration_s: float = 0.0
+    seed_errors: int = 0
+    setup_check_failures: int = 0
+    statuses: dict[str, int] = Field(default_factory=dict)
+
+
+class ModelReport(BaseModel):
+    model_id: str
+    model_config_public: dict[str, Any]
+    aggregate: AggregateMetrics
+    by_task: list[TaskResult] = Field(default_factory=list)
+    wall_clock_s: float = 0.0
+    pass_at_k_mean: float | None = None
+
+
+class BenchmarkReport(BaseModel):
+    schema_version: Literal["1"] = "1"
+    report_id: str
+    suite_id: str
+    suite_version: str
+    suite_content_hash: str | None = None
+    agentbox_version: str
+    started_at: datetime
+    finished_at: datetime
+    run_tags: dict[str, str] = Field(default_factory=dict)
+    models: list[ModelReport] = Field(default_factory=list)
+    leaderboard: list[dict[str, Any]] = Field(default_factory=list)
+    pass_at_k_definition: str = (
+        "pass_at_k = fraction of tasks with >=1 success among n_per_task rollouts; "
+        "pass_at_1 = mean per-task success rate of first/all rollouts"
+    )
+
+    def save_json(self, path: str | Path) -> None:
+        path = Path(path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(self.model_dump_json(indent=2), encoding="utf-8")
+
+    def save_markdown(self, path: str | Path) -> None:
+        from agentbox.benchmark.report import render_markdown
+
+        path = Path(path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(render_markdown(self), encoding="utf-8")
+
+    @classmethod
+    def load(cls, path: str | Path) -> BenchmarkReport:
+        import json
+
+        data = json.loads(Path(path).read_text(encoding="utf-8"))
+        return cls.model_validate(data)
+
+
+class ComparisonResult(BaseModel):
+    suite_id: str
+    hash_match: bool
+    warnings: list[str] = Field(default_factory=list)
+    deltas: list[dict[str, Any]] = Field(default_factory=list)

agentbox/benchmark/setup_check.py

@@ -0,0 +1,57 @@
+"""Post-seed environment health checks (not task success scoring)."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from agentbox.benchmark.schema import SetupCheckSpec
+from agentbox.sandbox.types import ExecResult
+
+if TYPE_CHECKING:
+    from agentbox.sandbox.manager import SandboxManager
+    from agentbox.sandbox.types import Sandbox
+
+
+class SetupCheckResult(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    ok: bool
+    results: list[ExecResult] = Field(default_factory=list)
+    failed_check: str | None = None
+    error: str | None = None
+
+
+class SetupChecker:
+    """Run suite-level commands after seed, before the agent starts."""
+
+    def __init__(self, manager: SandboxManager) -> None:
+        self.manager = manager
+
+    async def run(
+        self,
+        sandbox: Sandbox,
+        checks: list[SetupCheckSpec],
+    ) -> SetupCheckResult:
+        if not checks:
+            return SetupCheckResult(ok=True)
+
+        results: list[ExecResult] = []
+        for check in checks:
+            result = await self.manager.exec(
+                sandbox,
+                check.command,
+                timeout_s=check.timeout_s,
+            )
+            results.append(result)
+            failed = result.timed_out or result.exit_code != check.success_exit_code
+            if failed and check.required:
+                err = result.stderr or result.stdout or "setup check failed"
+                return SetupCheckResult(
+                    ok=False,
+                    results=results,
+                    failed_check=check.name,
+                    error=f"setup check {check.name!r} failed: {err}",
+                )
+        return SetupCheckResult(ok=True, results=results)

agentbox/cli/__init__.py

@@ -0,0 +1 @@
+"""Command-line interface."""