trapstreet-cli 0.1.0__py3-none-any.whl

trap/__init__.py

@@ -0,0 +1,6 @@
+try:
+    from trap._version import __version__
+except ImportError:
+    __version__ = "0.0.0+unknown"
+
+__all__ = ["__version__"]

trap/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

trap/cli.py

@@ -0,0 +1,326 @@
+from __future__ import annotations
+
+import http.server
+import json as _json
+import os
+import socket
+import socketserver
+import threading
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+import webbrowser
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import typer
+from rich.console import Console
+
+from trap.loader import TrapLoader, TrapTaskLoader
+from trap.report import OutputFormat, ReportSaver, renderer_factory
+from trap.runner import TaskRunner
+
+app = typer.Typer(help="AI prompt / agent / workflow / testing framework.")
+console = Console()
+
+AUTH_FILE = Path.home() / ".config" / "trapstreet" / "auth.json"
+DEFAULT_SERVER = "https://trapstreet.run"
+
+
+# -----------------------------------------------------------------------------
+# auth helpers
+
+
+def _load_auth_file() -> dict[str, str]:
+    """Read ~/.config/trapstreet/auth.json or return empty dict."""
+    if not AUTH_FILE.exists():
+        return {}
+    try:
+        return _json.loads(AUTH_FILE.read_text())
+    except (OSError, _json.JSONDecodeError):
+        return {}
+
+
+def _save_auth_file(server: str, api_key: str, runner: str | None) -> Path:
+    AUTH_FILE.parent.mkdir(parents=True, exist_ok=True)
+    payload = {"server": server, "api_key": api_key}
+    if runner:
+        payload["runner"] = runner
+    AUTH_FILE.write_text(_json.dumps(payload, indent=2) + "\n")
+    AUTH_FILE.chmod(0o600)
+    return AUTH_FILE
+
+
+def _resolve_api_key(flag_or_env: str | None) -> str:
+    """Auth precedence: --api-key / TRAPSTREET_API_KEY > auth.json > error."""
+    if flag_or_env:
+        return flag_or_env
+    api_key = _load_auth_file().get("api_key")
+    if api_key:
+        return api_key
+    console.print(
+        "[red]not logged in[/red]. Run [bold]tp login[/bold] "
+        "or set [bold]TRAPSTREET_API_KEY[/bold] / pass [bold]--api-key[/bold]."
+    )
+    raise SystemExit(2)
+
+
+# -----------------------------------------------------------------------------
+# commands
+
+
+@app.command()
+def run(
+    task: str | None = typer.Argument(None),
+    trap_yaml_path: Path = typer.Option(Path("trap.yaml"), "--config", "-c"),
+    tags: list[str] = typer.Option([], "--tag", "-t"),
+    output: OutputFormat = typer.Option(OutputFormat.rich, "--output", "-o"),
+    fail_fast: bool = typer.Option(False, "--fail-fast"),
+    workspace: Path = typer.Option(Path(".trap"), "--workspace", "-w"),
+) -> None:
+    """Run a task against a solution."""
+    trap_yaml_loader = TrapLoader(trap_yaml_path)
+    task_obj = trap_yaml_loader.resolve_task(task)
+
+    task_yaml_loader = TrapTaskLoader.from_task(task_obj, trap_yaml_loader.trap_dir)
+    active_cases = task_yaml_loader.cases_with_tags(tags)
+
+    ts = datetime.now().isoformat(timespec="seconds")
+    trap_run_dir = workspace.resolve() / task_obj.name / ts
+
+    runner = TaskRunner(
+        task_obj=task_obj,
+        trap_dir=trap_yaml_loader.trap_dir,
+        traptask_obj=task_yaml_loader.traptask,
+        traptask_dir=task_yaml_loader.task_dir,
+        task_outputs_dir=trap_run_dir,
+    )
+    case_results, grader_metrics = runner.run(active_cases, fail_fast=fail_fast)
+
+    report_data = ReportSaver.save(
+        trap_run_dir, case_results, task_obj, grader_metrics=grader_metrics
+    )
+    renderer_factory(output).render(report_data)
+
+    case_failed = any(cr.exit_code != 0 for cr in case_results)
+    grader_failed = grader_metrics is not None and grader_metrics.get("passed") is False
+    raise SystemExit(case_failed or grader_failed)
+
+
+@app.command()
+def report(
+    task: str | None = typer.Argument(None),
+    run: str = typer.Argument("latest"),
+    trap_yaml_path: Path = typer.Option(Path("trap.yaml"), "--config", "-c"),
+    output: OutputFormat = typer.Option(OutputFormat.rich, "--output", "-o"),
+    workspace: Path = typer.Option(Path(".trap"), "--workspace", "-w"),
+) -> None:
+    """Display a report for a task (defaults to latest run)."""
+    trap_yaml_loader = TrapLoader(trap_yaml_path)
+    task_name = trap_yaml_loader.resolve_task(task).name
+    run_dir = workspace.resolve() / task_name / run
+    report_data = ReportSaver.load(run_dir)
+    renderer_factory(output).render(report_data)
+
+
+@app.command()
+def login(
+    server: str = typer.Option(
+        DEFAULT_SERVER,
+        "--server",
+        envvar="TRAPSTREET_URL",
+        help="Trapstreet server URL.",
+    ),
+    timeout: int = typer.Option(
+        300, "--timeout", help="Seconds to wait for browser approval."
+    ),
+) -> None:
+    """Open the browser to authorize this machine; save api_key locally.
+
+    Starts a temporary HTTP server on localhost, opens
+    <server>/cli/authorize?return=http://localhost:<port>/callback in your
+    browser, and waits for the redirect back with the api_key.
+
+    The token is saved to ~/.config/trapstreet/auth.json (mode 600).
+    Subsequent `tp submit` calls read from there automatically — no env
+    var needed (but TRAPSTREET_API_KEY still works as an override).
+    """
+    # Bind to an arbitrary free port on the loopback interface.
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
+        probe.bind(("127.0.0.1", 0))
+        port = probe.getsockname()[1]
+
+    captured: dict[str, str] = {}
+
+    class _CallbackHandler(http.server.BaseHTTPRequestHandler):
+        # Silence default access log
+        def log_message(self, *_args: Any) -> None:  # noqa: ARG002
+            return
+
+        def do_GET(self) -> None:  # noqa: N802
+            qs = urllib.parse.urlparse(self.path).query
+            params = urllib.parse.parse_qs(qs)
+            api_key = (params.get("api_key") or [None])[0]
+            runner_name = (params.get("runner") or [None])[0]
+
+            if api_key:
+                captured["api_key"] = api_key
+                if runner_name:
+                    captured["runner"] = runner_name
+                self.send_response(200)
+                self.send_header("content-type", "text/html; charset=utf-8")
+                self.end_headers()
+                self.wfile.write(
+                    b"<!doctype html><meta charset=utf-8>"
+                    b"<title>logged in</title>"
+                    b"<style>body{font-family:ui-monospace,monospace;"
+                    b"background:#0a0a0a;color:#ededed;padding:3em;}"
+                    b"h1{color:#ff5f1f}</style>"
+                    b"<h1>logged in</h1>"
+                    b"<p>You can close this tab.</p>"
+                )
+            else:
+                self.send_response(400)
+                self.send_header("content-type", "text/plain")
+                self.end_headers()
+                self.wfile.write(b"missing api_key in query string")
+
+    server_obj = socketserver.TCPServer(("127.0.0.1", port), _CallbackHandler)
+    thread = threading.Thread(target=server_obj.serve_forever, daemon=True)
+    thread.start()
+
+    return_url = f"http://localhost:{port}/callback"
+    auth_url = (
+        f"{server.rstrip('/')}/cli/authorize"
+        f"?return={urllib.parse.quote(return_url, safe='')}"
+    )
+
+    console.print(f"opening [link={auth_url}]{auth_url}[/link]")
+    try:
+        webbrowser.open(auth_url)
+    except Exception:  # noqa: BLE001
+        # Couldn't open; user can copy/paste
+        console.print(
+            "[yellow]could not open browser automatically — "
+            "copy the URL above into a browser[/yellow]"
+        )
+
+    deadline = time.time() + timeout
+    while time.time() < deadline and "api_key" not in captured:
+        time.sleep(0.2)
+
+    server_obj.shutdown()
+    server_obj.server_close()
+
+    if "api_key" not in captured:
+        console.print(
+            f"[red]timed out after {timeout}s[/red] waiting for browser approval"
+        )
+        raise SystemExit(2)
+
+    path = _save_auth_file(server, captured["api_key"], captured.get("runner"))
+    runner_hint = (
+        f" · runner [bold]{captured.get('runner')}[/bold]"
+        if captured.get("runner")
+        else ""
+    )
+    console.print(f"[green]✓ logged in[/green]{runner_hint} · token saved to {path}")
+
+
+@app.command()
+def logout() -> None:
+    """Delete the locally-stored api_key."""
+    if AUTH_FILE.exists():
+        AUTH_FILE.unlink()
+        console.print(f"[green]✓[/green] removed {AUTH_FILE}")
+    else:
+        console.print(f"already logged out — no file at {AUTH_FILE}")
+
+
+@app.command()
+def submit(
+    task: str | None = typer.Argument(
+        None,
+        help="Task name (defaults to first task in trap.yaml). "
+        "Used as both the local run dir and the trapstreet task_id.",
+    ),
+    trap_yaml_path: Path = typer.Option(Path("trap.yaml"), "--config", "-c"),
+    workspace: Path = typer.Option(Path(".trap"), "--workspace", "-w"),
+    run: str = typer.Option(
+        "latest", "--run", "-r", help="Which run to upload (default: latest)."
+    ),
+    server: str = typer.Option(
+        DEFAULT_SERVER,
+        "--server",
+        envvar="TRAPSTREET_URL",
+        help="Trapstreet server URL.",
+    ),
+    api_key: str | None = typer.Option(
+        None,
+        "--api-key",
+        envvar="TRAPSTREET_API_KEY",
+        help="Runner api_key. Falls back to TRAPSTREET_API_KEY env, "
+        "then ~/.config/trapstreet/auth.json (see `tp login`).",
+    ),
+) -> None:
+    """Upload the latest report.json to trapstreet."""
+    # Resolve api_key from flag > env > auth.json > error.
+    api_key = _resolve_api_key(api_key)
+
+    # If --server wasn't overridden but auth.json has one, prefer file's server.
+    if server == DEFAULT_SERVER and not os.environ.get("TRAPSTREET_URL"):
+        file_server = _load_auth_file().get("server")
+        if file_server:
+            server = file_server
+
+    trap_yaml_loader = TrapLoader(trap_yaml_path)
+    task_name = trap_yaml_loader.resolve_task(task).name
+
+    report_path = workspace.resolve() / task_name / run / "report.json"
+    if not report_path.exists():
+        console.print(
+            f"[red]error[/red]: no report at {report_path}. "
+            "Run [bold]tp run[/bold] first."
+        )
+        raise SystemExit(2)
+
+    payload = report_path.read_bytes()
+    url = f"{server.rstrip('/')}/api/submit/{task_name}"
+    req = urllib.request.Request(
+        url,
+        data=payload,
+        headers={
+            "authorization": f"Bearer {api_key}",
+            "content-type": "application/json",
+        },
+        method="POST",
+    )
+
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            body: Any = _json.loads(resp.read())
+    except urllib.error.HTTPError as e:
+        msg = e.read().decode("utf-8", errors="replace")
+        console.print(f"[red]http {e.code}[/red]: {msg}")
+        raise SystemExit(2)
+    except urllib.error.URLError as e:
+        console.print(f"[red]connection error[/red]: {e.reason}")
+        raise SystemExit(2)
+
+    run_obj = body.get("run") or {}
+    run_id = run_obj.get("id", "?")
+    score = run_obj.get("total_score")
+    passed = run_obj.get("passed")
+    view_url = body.get("view_url", "")
+    status = "[green]✓ passed[/green]" if passed else "[red]✗ failed[/red]"
+    console.print(f"{status} [bold]{run_id}[/bold] · score [cyan]{score}[/cyan]")
+    if view_url:
+        console.print(f"  → [link={view_url}]{view_url}[/link]")
+
+
+@app.command()
+def init() -> None:
+    """Generate annotated trap.yaml + traptask.yaml scaffold."""
+    console.print("not yet")

trap/loader.py

@@ -0,0 +1,75 @@
+# Loads trap.yaml and traptask.yaml into their respective loader classes.
+from __future__ import annotations
+
+from collections.abc import Iterable
+from pathlib import Path
+
+import yaml
+from pydantic import TypeAdapter
+
+from trap.models import Task, TrapTask, TrapTaskCase
+
+_tasks_adapter: TypeAdapter[dict[str, Task]] = TypeAdapter(dict[str, Task])
+
+
+class TrapLoader:
+    """Loads trap.yaml (solution author's config)."""
+
+    def __init__(self, trap_yaml_path: Path) -> None:
+        self.trap_dir: Path = trap_yaml_path.resolve().parent
+        data = yaml.safe_load(trap_yaml_path.read_text())
+        raw = _tasks_adapter.validate_python(data["tasks"])
+        self.tasks: dict[str, Task] = {
+            name: task.model_copy(update={"name": name}) for name, task in raw.items()
+        }
+
+    def select_task(self, name: str) -> Task:
+        """Return task by name."""
+        if name not in self.tasks:
+            raise KeyError(f"task {name!r} not found in trap.yaml")
+        return self.tasks[name]
+
+    def resolve_task(self, name: str | None) -> Task:
+        """Return named task, or the first task if name is None."""
+        return self.select_task(name or next(iter(self.tasks)))
+
+
+class TrapTaskLoader:
+    """Loads traptask.yaml (task author's config) and resolves runtime paths."""
+
+    def __init__(self, traptask_yaml_path: Path) -> None:
+        self.task_dir: Path = traptask_yaml_path.resolve().parent
+        if traptask_yaml_path.exists():
+            self.traptask = TrapTask.model_validate(yaml.safe_load(traptask_yaml_path.read_text()))
+        else:
+            self.traptask = self._discover(self.task_dir)
+        self.inputs_dir: Path = (self.task_dir / self.traptask.dirs.inputs).resolve()
+        self.expected_dir: Path = (self.task_dir / self.traptask.dirs.expected).resolve()
+
+    @staticmethod
+    def _discover(task_dir: Path) -> TrapTask:
+        """Auto-build TrapTask by scanning inputs/ when traptask.yaml is absent."""
+        inputs_dir = task_dir / "inputs"
+        if not inputs_dir.is_dir():
+            raise FileNotFoundError(f"no traptask.yaml and no inputs/ directory found in {task_dir}")
+        case_ids = sorted(p.name for p in inputs_dir.iterdir() if p.is_dir())
+        if not case_ids:
+            raise FileNotFoundError(f"inputs/ in {task_dir} has no case subdirectories")
+        return TrapTask(cases=tuple(TrapTaskCase(id=case_id) for case_id in case_ids))
+
+    @classmethod
+    def from_task(cls, task: Task, trap_dir: Path) -> TrapTaskLoader:
+        """Resolve traptask.yaml from a Task's directory path and the trap.yaml directory."""
+        traptask_dir = (trap_dir / task.traptask).resolve()
+        return cls(traptask_dir / "traptask.yaml")
+
+    @property
+    def cases(self) -> tuple[TrapTaskCase, ...]:
+        """Return all non-skipped cases."""
+        return tuple(c for c in self.traptask.cases if not c.skip)
+
+    def cases_with_tags(self, tags: Iterable[str] | None = None) -> tuple[TrapTaskCase, ...]:
+        """Return non-skipped cases matching any of the specified tags, or all cases if tags is empty/None."""
+        if not (tag_set := set(tags or ())):
+            return self.cases
+        return tuple(c for c in self.cases if not tag_set.isdisjoint(c.tags))

trap/models/__init__.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from .config import InputsBinding, Task
+from .report import Counts, ReportData
+from .results import CaseResult
+from .task import DirsConfig, SubprocessCmd, TrapTask, TrapTaskCase
+
+__all__ = [
+    "CaseResult",
+    "Counts",
+    "DirsConfig",
+    "InputsBinding",
+    "ReportData",
+    "SubprocessCmd",
+    "Task",
+    "TrapTask",
+    "TrapTaskCase",
+]

trap/models/config.py

@@ -0,0 +1,27 @@
+# Models for trap.yaml (solution author's config).
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class InputsBinding(BaseModel):
+    stdin: str | None = None  # filename in inputs/{case_id}/ piped as subprocess stdin
+    # declared input keys; runner validates these stems exist in inputs/{case_id}/ before running
+    files: tuple[str, ...] = ()
+    # all files in inputs/{case_id}/ are also exposed via INPUTS env var at runtime
+    # TODO: args: list[str] = []   — pass inputs as CLI positional/named arguments
+    # TODO: env: dict[str, str] = {}  — inject inputs as environment variables
+
+
+class Task(BaseModel):
+    name: str = ""
+    description: str = ""
+    cmd: str
+    traptask: str  # path to traptask directory (relative to trap.yaml); trap looks for traptask.yaml inside
+    inputs: InputsBinding | None = None
+    # output filenames; solution writes each to the path given by outputs_envvar[name] at runtime
+    file_outputs: tuple[str, ...] = ()
+    timeout: int = 30
+    # env var names injected by the runner; override if the solution already uses these names
+    inputs_envvar: str = "INPUTS"
+    outputs_envvar: str = "OUTPUTS"

trap/models/report.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel
+
+from trap.models.config import Task
+from trap.models.results import CaseResult
+
+
+class Counts(BaseModel):
+    passed: int
+    failed: int
+    skipped: int
+
+
+class ReportData(BaseModel):
+    task: Task
+    cases: tuple[CaseResult, ...]
+    run_counts: Counts
+    grader_metrics: Any = None
+
+    @classmethod
+    def from_run(
+        cls,
+        cases: tuple[CaseResult, ...],
+        task: Task,
+        grader_metrics: Any = None,
+    ) -> ReportData:
+        return cls(
+            task=task,
+            cases=cases,
+            grader_metrics=grader_metrics,
+            run_counts=Counts(
+                passed=sum(1 for r in cases if not r.skipped and r.exit_code == 0),
+                failed=sum(1 for r in cases if not r.skipped and r.exit_code != 0),
+                skipped=sum(1 for r in cases if r.skipped),
+            ),
+        )

trap/models/results.py

@@ -0,0 +1,15 @@
+# Runtime result models produced by judge and grader subprocesses.
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class CaseResult(BaseModel):
+    case_id: str
+    exit_code: int = 0
+    duration: float = 0.0  # seconds
+    # any JSON-serializable value; trap does not interpret this, grader does
+    metrics: Any
+    skipped: bool = False

trap/models/task.py

@@ -0,0 +1,30 @@
+# Models for traptask.yaml (task author's config).
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class SubprocessCmd(BaseModel):
+    # cmd is relative to traptask.yaml's directory and run via shlex.split
+    cmd: str
+    payload_envvar: str = "TRAPTASK_PAYLOAD"
+
+
+class DirsConfig(BaseModel):
+    # paths relative to traptask.yaml; outputs dir is a runtime tmpdir, not declared here
+    inputs: str = "inputs/"
+    expected: str = "expected/"
+
+
+class TrapTaskCase(BaseModel):
+    id: str
+    description: str = ""
+    tags: tuple[str, ...] = ()
+    skip: bool = False
+
+
+class TrapTask(BaseModel):
+    dirs: DirsConfig = DirsConfig()
+    cases: tuple[TrapTaskCase, ...]
+    judge: SubprocessCmd | None = None  # None → skip per-case scoring
+    grader: SubprocessCmd | None = None  # None → skip overall aggregation

trap/report/__init__.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+import enum
+
+from trap.report.base import BaseRenderer
+from trap.report.json import JsonRenderer
+from trap.report.rich import RichRenderer
+from trap.report.saver import ReportSaver
+
+__all__ = ["BaseRenderer", "JsonRenderer", "OutputFormat", "ReportSaver", "RichRenderer", "renderer_factory"]
+
+
+class OutputFormat(enum.StrEnum):
+    rich = "rich"
+    json = "json"
+
+
+def renderer_factory(fmt: OutputFormat) -> BaseRenderer:
+    match fmt:
+        case OutputFormat.rich:
+            return RichRenderer()
+        case OutputFormat.json:
+            return JsonRenderer()

trap/report/base.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from trap.models import ReportData
+
+
+class BaseRenderer:
+    def render(self, data: ReportData) -> None:
+        raise NotImplementedError

trap/report/json.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+import sys
+
+from trap.models import ReportData
+from trap.report.base import BaseRenderer
+
+
+class JsonRenderer(BaseRenderer):
+    def render(self, data: ReportData) -> None:
+        sys.stdout.write(data.model_dump_json(indent=2) + "\n")