trap-cli 0.0.1__py3-none-any.whl

trap/models/trap_yaml.py

@@ -0,0 +1,64 @@
+# Models for trap.yaml (solution author's config).
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class Profile(BaseModel):
+    # Self-reported engine identity, surfaced in the report. Both fields are
+    # multi-valued (a run may use several models/frameworks); a scalar is
+    # accepted as a convenience and normalised to a single-element list.
+    model: tuple[str, ...] = ()
+    framework: tuple[str, ...] = ()
+
+    @field_validator("model", "framework", mode="before")
+    @classmethod
+    def _as_list(cls, v: Any) -> Any:
+        if v is None:
+            return ()
+        if isinstance(v, str):
+            return (v,)
+        return v
+
+
+class TaskBinding(BaseModel):
+    # A task binding: which task this solution is run against. The map key is the
+    # binding's `alias` — the solution author's handle for this task, which also
+    # serves as the local run-dir name and the trapstreet task_id on submit. The
+    # value is just where the task lives. Solution-invariant settings sit at the
+    # top level of trap.yaml (see TrapConfig).
+    alias: str = ""
+    # required: local task dir XOR git+ URL (relative to trap.yaml), same polymorphic
+    # form as --solution. A git+ URL is cloned into clone_to.
+    source: str
+    # clone target for a git+ source (relative to trap.yaml, or absolute);
+    # omitted → hidden cache .trap/repos/<repo>. Only valid when source is a URL.
+    clone_to: Path | None = None
+
+
+class TrapConfig(BaseModel):
+    # trap.yaml is one solution's file: its invariant settings sit at the top
+    # level (flat, 1:1 with the YAML), and `tasks` is the collection of task
+    # bindings it is run against.
+    # Field order mirrors the canonical trap.yaml layout.
+    # optional leaderboard identity; None → server auto-assigns a serial name
+    name: str | None = None
+    # self-reported engine identity (model/framework); plumbed to report.json `profile`
+    profile: Profile = Field(default_factory=Profile)
+    # Prepares the solution checkout once after clone (e.g. `uv sync`, `npm install`).
+    # Solution-author owned. Auto-runs on a remote clone/update; else `tp run --setup-solution`.
+    setup_cmd: str | None = None
+    stdin: str | None = None  # optional: filename in inputs/{case_id}/ piped to the solution's stdin
+    cmd: str  # run via shlex.split, cwd = the trap.yaml directory
+    # env var carrying the run manifest (inputs_dir / outputs_dir); override if needed
+    manifest_envvar: str = "TRAP_MANIFEST"
+    # per-case wall-clock ceiling (seconds). A safety net against hangs/runaways, not a
+    # fair budget — duration is recorded faithfully, so set it generously; a timed-out
+    # case = "did not complete". Solution-author owned.
+    timeout: int = 600
+    # free-form escape hatch for author notes; tolerated but never written to the report
+    extra: dict[str, Any] = {}
+    tasks: dict[str, TaskBinding]

trap/models/traptask_yaml.py

@@ -0,0 +1,58 @@
+# Models for traptask.yaml (task author's config).
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class SubprocessConfig(BaseModel):
+    # How to invoke the judge/grader subprocess: the command (run via shlex.split,
+    # cwd = traptask.yaml's directory) plus the env var carrying its manifest.
+    cmd: str
+    manifest_envvar: str = "TRAPTASK_MANIFEST"
+    # Per-invocation wall-clock ceiling (seconds): a safety net against a hung or
+    # runaway actor, not a time budget. On timeout the actor is killed and recorded
+    # as an error on the run (exit 124) — it never crashes the run. Task-author owned,
+    # so it is identical for every solution that runs this task version. The subclasses
+    # set the per-actor default.
+    timeout: int
+
+
+class JudgeConfig(SubprocessConfig):
+    # Runs once per case and may itself call an LLM (LLM-as-judge) → generous default.
+    timeout: int = 300
+
+
+class GraderConfig(SubprocessConfig):
+    # Only aggregates the per-case results → fast, so a tighter default.
+    timeout: int = 120
+
+
+class DirsConfig(BaseModel):
+    # paths relative to traptask.yaml; outputs dir is a runtime tmpdir, not declared here
+    inputs: str = "inputs/"
+    expected: str = "expected/"
+
+
+class TraptaskCase(BaseModel):
+    id: str
+    description: str = ""  # free-form author note; trap neither consumes nor displays it
+    tags: tuple[str, ...] = ()
+    skip: bool = False
+
+
+class TraptaskConfig(BaseModel):
+    # Field order mirrors the canonical traptask.yaml layout.
+    # Optional human-readable title for the task; task-author owned. Read from the task
+    # repo (via the report's provenance.task), so it stays consistent across every
+    # solution that runs this task version.
+    name: str | None = None
+    # Prepares the checkout (e.g. `uv sync`); task-author owned so every solution gets
+    # an identical env. Auto-runs on a remote clone/update; else `tp run --setup-task`.
+    setup_cmd: str | None = None
+    dirs: DirsConfig = DirsConfig()
+    # Advisory contract: filenames (and/or `stdout`/`stderr`) the solution writes.
+    # Never enforced — the judge is the sole arbiter. Omit for dynamic outputs.
+    declared_outputs: tuple[str, ...] = ()
+    cases: tuple[TraptaskCase, ...]
+    judge: JudgeConfig | None = None  # None → skip per-case scoring
+    grader: GraderConfig | None = None  # None → skip overall aggregation

trap/report/__init__.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+import enum
+
+from trap.report.base import BaseRenderer
+from trap.report.handle import ReportHandle
+from trap.report.json import JsonRenderer
+from trap.report.rich import RichRenderer
+
+__all__ = ["BaseRenderer", "JsonRenderer", "OutputFormat", "ReportHandle", "RichRenderer", "renderer_factory"]
+
+
+class OutputFormat(enum.StrEnum):
+    rich = "rich"
+    json = "json"
+
+
+def renderer_factory(fmt: OutputFormat) -> BaseRenderer:
+    return {OutputFormat.rich: RichRenderer, OutputFormat.json: JsonRenderer}[fmt]()

trap/report/base.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from trap.models import ReportData
+
+
+class BaseRenderer:
+    def render(self, data: ReportData) -> None:
+        raise NotImplementedError

trap/report/handle.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from trap.models import CaseResult, Environment, Provenance, ReportData, TrapConfig
+
+_REPORT_FILENAME = "report.json"
+
+
+class ReportHandle:
+    def __init__(self, workspace: Path, task_alias: str, run: str) -> None:
+        self._workspace = workspace
+        self._task_alias = task_alias
+        self._run = run
+
+    @property
+    def run_dir(self) -> Path:
+        return self._workspace / self._task_alias / self._run
+
+    @property
+    def report_json_path(self) -> Path:
+        return self.run_dir / _REPORT_FILENAME
+
+    def save(
+        self,
+        trap_config: TrapConfig,
+        case_results: tuple[CaseResult, ...],
+        started_at_utc: datetime,
+        finished_at_utc: datetime,
+        provenance: Provenance,
+        grader_metrics: Any = None,
+        environment: Environment | None = None,
+    ) -> ReportData:
+        data = ReportData.from_run(
+            trap_config=trap_config,
+            provenance=provenance,
+            cases_results=case_results,
+            started_at_utc=started_at_utc,
+            finished_at_utc=finished_at_utc,
+            grader_metrics=grader_metrics,
+            environment=environment,
+        )
+        self.report_json_path.write_text(data.model_dump_json(indent=2))
+        return data
+
+    def assert_exists(self) -> None:
+        if not self.report_json_path.exists():
+            raise FileNotFoundError(f"no report found in {self.run_dir}")
+
+    def load(self) -> ReportData:
+        self.assert_exists()
+        return ReportData.model_validate_json(self.report_json_path.read_text())

trap/report/json.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+import sys
+
+from trap.models import ReportData
+from trap.report.base import BaseRenderer
+
+
+class JsonRenderer(BaseRenderer):
+    def render(self, data: ReportData) -> None:
+        sys.stdout.write(data.model_dump_json(indent=2) + "\n")

trap/report/rich.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import json
+
+from rich import box
+from rich.console import Console
+from rich.markup import escape
+from rich.panel import Panel
+from rich.table import Table
+from rich.text import Text
+
+from trap.models import CaseResult, ReportData
+from trap.report.base import BaseRenderer
+
+
+class RichRenderer(BaseRenderer):
+    def __init__(self, console: Console | None = None) -> None:
+        self.console = console or Console()
+
+    @staticmethod
+    def _get_metrics_keys(data: ReportData) -> set[str]:
+        return {key for result in data.cases_results if result.metrics for key in result.metrics}
+
+    @staticmethod
+    def _render_metric_cell(value: object) -> str:
+        if value is None:
+            return "[dim]—[/dim]"
+        if isinstance(value, bool):
+            return "[bold green]✓[/bold green]" if value else "[bold red]✗[/bold red]"
+        if isinstance(value, float) and 0.0 <= value <= 1.0:
+            pct = value * 100
+            if pct >= 100:
+                return f"[bold green]{pct:.0f}%[/bold green]"
+            if pct > 0:
+                return f"[bold yellow]{pct:.0f}%[/bold yellow]"
+            return f"[bold red]{pct:.0f}%[/bold red]"
+        # Bracket characters in user-supplied content (judge metric values,
+        # grader outputs, etc.) would otherwise be interpreted as Rich
+        # markup and crash with MarkupError on mismatched tags. e.g. a judge
+        # emitting a regex pattern like "[/\\-]" raises:
+        #     MarkupError: closing tag '[/\\-]' doesn't match any open tag
+        # Escape so brackets render as literal text.
+        return escape(str(value))
+
+    @staticmethod
+    def _render_exit(result: CaseResult) -> str:
+        # The raw exit code — not a verdict. "did it run" (this) is a separate axis from
+        # "is it correct" (the judge's score columns); 124 is the timeout sentinel.
+        if result.exit_code == 0:
+            return "[dim]0[/dim]"
+        return f"[red]{result.exit_code}[/red]"
+
+    @staticmethod
+    def _render_cost(usd: float) -> str:
+        if usd <= 0:
+            return "[dim]—[/dim]"
+        if usd < 0.001:
+            return f"[dim]${usd:.5f}[/dim]"
+        return f"[dim]${usd:.4f}[/dim]"
+
+    def _build_table(self, data: ReportData) -> Table:
+        metrics_keys = self._get_metrics_keys(data)
+        has_cost = any(c.cost is not None for c in data.cases_results)
+        table = Table(box=box.ROUNDED, show_header=True, header_style="bold", expand=True)
+        table.add_column("case")
+        table.add_column("exit", justify="right")
+        table.add_column("time", justify="right", style="dim")
+        for key in metrics_keys:
+            table.add_column(f"# {escape(key)}", justify="right", header_style="bold cyan")
+        if has_cost:
+            table.add_column("prompt_tok", justify="right", style="dim")
+            table.add_column("compl_tok", justify="right", style="dim")
+            table.add_column("cost", justify="right")
+        for result in data.cases_results:
+            row = [escape(result.case_id), self._render_exit(result), f"{result.duration:.3f}s"]
+            for key in metrics_keys:
+                value = result.metrics.get(key) if result.metrics else None
+                row.append(self._render_metric_cell(value))
+            if has_cost:
+                if result.cost:
+                    row.append(str(result.cost.prompt_tokens))
+                    row.append(str(result.cost.completion_tokens))
+                    row.append(self._render_cost(result.cost.cost_usd))
+                else:
+                    row.extend(["[dim]—[/dim]", "[dim]—[/dim]", "[dim]—[/dim]"])
+            table.add_row(*row)
+        return table
+
+    def _build_summary(self, data: ReportData) -> Panel:
+        # Neutral execution tally — trap reports facts, not a verdict. "non-zero exit"
+        # just counts cases whose solution did not exit 0 (correctness is the judge's
+        # score, not this). The grader's own output, if any, is shown verbatim below.
+        results = data.cases_results
+        n_total = len(results)
+        n_errored = sum(1 for r in results if r.exit_code != 0)
+        parts = [f"{n_total} case{'s' if n_total != 1 else ''}"]
+        if n_errored:
+            parts.append(f"[red]{n_errored} non-zero exit[/red]")
+
+        rows: list[tuple[str, Text | str]] = [
+            ("cases", Text.from_markup(" · ".join(parts) or "[dim]no cases[/dim]")),
+        ]
+        if data.grader_metrics is not None:
+            rows.append(("grader", escape(json.dumps(data.grader_metrics, ensure_ascii=False))))
+
+        grid = Table.grid(padding=(0, 2))
+        grid.add_column(style="dim", justify="right")
+        grid.add_column()
+        for key, value in rows:
+            grid.add_row(key, value)
+
+        return Panel(grid, title="Summary", expand=True)
+
+    def render(self, data: ReportData) -> None:
+        self.console.print(self._build_summary(data))
+        self.console.print(self._build_table(data))

trap/runner/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from trap.runner.task import TaskRunner
+
+__all__ = ["TaskRunner"]

trap/runner/capture.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class Capture:
+    """The stdout/stderr/meta files trap writes for one actor's run.
+
+    Each actor (solution, judge, grader) gets its own directory; the capture
+    files are unprefixed within it (so `solution/stdout`, `judge/stdout`).
+    """
+
+    stdout: Path
+    stderr: Path
+    meta: Path
+
+    @classmethod
+    def from_dir(cls, capture_dir: Path) -> Capture:
+        return cls(
+            stdout=capture_dir / "stdout",
+            stderr=capture_dir / "stderr",
+            meta=capture_dir / "meta.json",
+        )
+
+    def write(self, stdout: str, stderr: str, meta: dict[str, Any]) -> None:
+        """Create the actor directory and persist its run's stdout/stderr/meta."""
+        self.stdout.parent.mkdir(parents=True, exist_ok=True)
+        self.stdout.write_text(stdout)
+        self.stderr.write_text(stderr)
+        self.meta.write_text(json.dumps(meta))

trap/runner/grader.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+
+from trap.models import CaseResult, GraderConfig
+from trap.runner.capture import Capture
+from trap.runner.proc import CapturedSubprocess
+
+if TYPE_CHECKING:
+    from trap.runner.task import TaskRunner
+
+
+class GraderRunner:
+    def __init__(self, runner: TaskRunner, case_results: tuple[CaseResult, ...]) -> None:
+        assert runner.traptask_config.grader is not None
+        self.grader: GraderConfig = runner.traptask_config.grader
+        self.runner = runner
+        self.cases = case_results
+        # Run-level grader gets its own `grader/` directory next to report.json.
+        self.grader_dir = runner.run_dir / "grader"
+        self.capture = Capture.from_dir(self.grader_dir)
+
+    @property
+    def _manifest(self) -> str:
+        return json.dumps([c.model_dump() for c in self.cases])
+
+    def run(self) -> Any:
+        return CapturedSubprocess(
+            self.grader.cmd,
+            manifest_envvar=self.grader.manifest_envvar,
+            timeout=self.grader.timeout,
+            cwd=self.runner.traptask_dir,
+            manifest=self._manifest,
+            capture=self.capture,
+        ).run_metrics_or_error(runner_name="grader")

trap/runner/judge.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+
+from trap.models import JudgeConfig
+from trap.runner.proc import CapturedSubprocess
+
+if TYPE_CHECKING:
+    from trap.runner.layout import CaseLayout
+    from trap.runner.task import TaskRunner
+
+
+class JudgeRunner:
+    def __init__(self, runner: TaskRunner, case_id: str, layout: CaseLayout) -> None:
+        self.runner = runner
+        self.case_id = case_id
+        self.case_inputs_dir = runner.task_inputs_dir / case_id  # task-repo side
+        self.case_expected_dir = runner.task_expected_dir / case_id  # task-repo side
+        self.layout = layout  # workspace side
+
+        assert runner.traptask_config.judge is not None
+        self.judge: JudgeConfig = runner.traptask_config.judge
+
+    @property
+    def _manifest(self) -> str:
+        expected_dir = self.case_expected_dir
+        solution_capture = self.layout.solution_capture
+        return json.dumps(
+            {
+                "inputs_dir": str(self.case_inputs_dir.resolve()),
+                "expected_dir": str(expected_dir.resolve()) if expected_dir.exists() else None,
+                "outputs_dir": str(self.layout.outputs_dir.resolve()),
+                "run": {
+                    "stdout": str(solution_capture.stdout.resolve()),
+                    "stderr": str(solution_capture.stderr.resolve()),
+                    "meta": str(solution_capture.meta.resolve()),
+                },
+            }
+        )
+
+    def run(self) -> Any:
+        return CapturedSubprocess(
+            self.judge.cmd,
+            manifest_envvar=self.judge.manifest_envvar,
+            timeout=self.judge.timeout,
+            cwd=self.runner.traptask_dir,
+            manifest=self._manifest,
+            capture=self.layout.judge_capture,
+        ).run_metrics_or_error(runner_name="judge")

trap/runner/layout.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from trap.runner.capture import Capture
+
+
+@dataclass(frozen=True)
+class CaseLayout:
+    """Where one case's run artefacts live in the .trap workspace.
+
+    Structure: ``{run_dir}/{case_id}/{solution,judge}/...``. This is the single
+    source of truth for the per-case directory layout, so the runners derive
+    their paths from here instead of each re-joining the same strings.
+    """
+
+    case_dir: Path
+
+    @classmethod
+    def for_case(cls, run_dir: Path, case_id: str) -> CaseLayout:
+        return cls(run_dir / case_id)
+
+    @property
+    def outputs_dir(self) -> Path:
+        """Solution-written files only — the manifest's ``outputs_dir``."""
+        return self.case_dir / "solution" / "outputs"
+
+    @property
+    def solution_capture(self) -> Capture:
+        """The solution run's stdout/stderr/meta — the manifest's ``run``."""
+        return Capture.from_dir(self.case_dir / "solution")
+
+    @property
+    def judge_capture(self) -> Capture:
+        return Capture.from_dir(self.case_dir / "judge")

trap/runner/proc.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import json
+import os
+import shlex
+import subprocess
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from trap.runner.capture import Capture
+
+
+@dataclass(frozen=True)
+class ProcResult:
+    """The normalised outcome of one run. A timeout is folded in here as
+    ``exit_code == CapturedSubprocess.TIMEOUT_EXIT_CODE`` (124) with whatever partial
+    output we got, so callers never see a raw TimeoutExpired."""
+
+    stdout: str
+    stderr: str
+    exit_code: int
+    duration: float
+
+
+class CapturedSubprocess:
+    """A subprocess trap runs and whose streams it captures to the workspace — the
+    solution, a judge, or a grader. On top of ``subprocess.run`` it owns the shared
+    lifecycle: assemble the env, run the command (folding a timeout into exit 124 +
+    partial output), time it, and persist the capture.
+
+    Two consume methods, neither of which raises (failure is always in-band) — they
+    differ because the solution and the judge/grader play opposite roles:
+
+      - ``run()`` (solution) returns the raw ProcResult. The solution is the *subject
+        under test*, so a non-zero exit or a timeout is a legitimate outcome to measure
+        (a case may even expect exit 1) — not an error. It is handed back verbatim as
+        data for the judge to evaluate.
+      - ``run_metrics_or_error()`` (judge/grader) returns the actor's parsed JSON
+        metrics. A judge/grader is the *measuring apparatus*: if it times out, exits
+        non-zero, or emits non-JSON it produced no verdict at all, so that is folded
+        into an ``{"error": ...}`` metric (isolated, so one broken judge or grader never
+        crashes the run or loses the report).
+    """
+
+    # conventional exit code for "a subprocess was killed for exceeding its timeout"
+    TIMEOUT_EXIT_CODE = 124
+
+    def __init__(
+        self,
+        cmd: str,
+        *,
+        manifest_envvar: str,
+        timeout: int,
+        cwd: Path,
+        manifest: str,
+        capture: Capture,
+        stdin: str | None = None,
+        env_extra: dict[str, str] | None = None,
+    ) -> None:
+        self.cmd = cmd
+        self.manifest_envvar = manifest_envvar
+        self.timeout = timeout
+        self.cwd = cwd
+        self.manifest = manifest
+        self.capture = capture
+        self.stdin = stdin
+        self.env_extra = env_extra or {}
+
+    def run(self) -> ProcResult:
+        """Run and capture, returning the raw ProcResult. A timeout is folded in (exit
+        124) but never raised — for the solution a failed/timed-out run is a measured
+        outcome, so the caller reads ``exit_code`` rather than catching."""
+        env = {**os.environ, self.manifest_envvar: self.manifest, **self.env_extra}
+        t0 = time.monotonic()
+        try:
+            proc = subprocess.run(
+                shlex.split(self.cmd),
+                input=self.stdin,
+                capture_output=True,
+                text=True,
+                cwd=self.cwd,
+                timeout=self.timeout,
+                env=env,
+            )
+            stdout, stderr, exit_code = proc.stdout, proc.stderr, proc.returncode
+        except subprocess.TimeoutExpired as e:
+            stdout = self._as_text(e.stdout)
+            stderr = self._as_text(e.stderr) + f"\n[trap] timed out after {self.timeout}s"
+            exit_code = self.TIMEOUT_EXIT_CODE
+
+        duration = time.monotonic() - t0
+        self.capture.write(stdout, stderr, {"exit_code": exit_code, "duration": duration})
+        return ProcResult(stdout, stderr, exit_code, duration)
+
+    def run_metrics_or_error(self, *, runner_name: str) -> Any:
+        """Run the actor and return its parsed JSON metrics. A broken actor — timeout,
+        non-zero exit, or non-JSON output — is folded into an ``{"error": ...}`` metric
+        rather than raised, so one bad judge/grader never crashes the run or loses the
+        report (full detail is in the actor's stderr capture). ``runner_name`` ("judge" /
+        "grader") just labels the error."""
+        result = self.run()
+        if result.exit_code == self.TIMEOUT_EXIT_CODE:
+            return {"error": f"{runner_name} timed out after {self.timeout}s"}
+        if result.exit_code != 0:
+            return {"error": f"{runner_name} exited with status {result.exit_code}"}
+        try:
+            return json.loads(result.stdout)
+        except json.JSONDecodeError as e:
+            return {"error": f"{runner_name} produced invalid JSON: {e}"}
+
+    @staticmethod
+    def _as_text(out: object) -> str:
+        """Partial stdout/stderr off a TimeoutExpired may be a bytes-like buffer (or
+        absent) even with text=True; normalise whatever we got to a str."""
+        if isinstance(out, str):
+            return out
+        if isinstance(out, (bytes, bytearray, memoryview)):
+            return bytes(out).decode(errors="replace")
+        return ""