trap-cli 0.0.1__tar.gz → 0.0.3__tar.gz

{trap_cli-0.0.1 → trap_cli-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trap-cli
-Version: 0.0.1
+Version: 0.0.3
 Summary: tp — non-invasive CLI testing framework for AI workflows. Submits results to trapstreet.run.
 Project-URL: Homepage, https://trapstreet.run
 Project-URL: Repository, https://github.com/trapstreet/trap

{trap_cli-0.0.1 → trap_cli-0.0.3}/src/trap/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.0.1'
-__version_tuple__ = version_tuple = (0, 0, 1)
+__version__ = version = '0.0.3'
+__version_tuple__ = version_tuple = (0, 0, 3)
 
 __commit_id__ = commit_id = None

{trap_cli-0.0.1 → trap_cli-0.0.3}/src/trap/auth/client.py

@@ -45,7 +45,8 @@ class ApiClient:
 
     def submit(self, report_path: Path) -> dict[str, Any]:
         # Content-addressed ingest: the task identity travels inside the report
-        # (provenance.task.{repo,commit}), not the URL — so no task_id path segment.
+        # (provenance.task.{repo,commit,subdirectory}), not the URL — so no task_id
+        # path segment.
         try:
             resp = self._client.post("/api/submit", content=report_path.read_bytes())
             resp.raise_for_status()

{trap_cli-0.0.1 → trap_cli-0.0.3}/src/trap/cli/__init__.py

@@ -23,6 +23,8 @@ from trap.runner import TaskRunner
 app = typer.Typer(help="AI prompt / agent / workflow / testing framework.")
 app.add_typer(auth_app, name="auth")
 console = Console()
+# Warnings go to stderr so `-o json` stdout stays machine-parseable.
+err_console = Console(stderr=True)
 
 
 def _die(msg: object) -> typer.Exit:
@@ -42,7 +44,7 @@ def _confirm_remote(url: str, *, trust: bool) -> None:
     runs); refuses (rather than running silently) when there's no TTY to confirm at."""
     if trust:
         return
-    console.print(
+    err_console.print(
         "[yellow]⚠  about to download and RUN code from a remote source:[/yellow]\n"
         f"     [bold]{url}[/bold]\n"
         "   trap will execute its setup command, the solution, and any judge/grader\n"
@@ -57,6 +59,39 @@ def _confirm_remote(url: str, *, trust: bool) -> None:
         raise typer.Exit(code=1)
 
 
+def _confirm_unanchored(provenance: Provenance, *, allow: bool) -> None:
+    """Gate run/submit when a side has no anchored git provenance.
+
+    trapstreet accepts such uploads but its leaderboard hides runs it cannot pin to
+    a commit — make the user acknowledge that up front instead of discovering it on
+    the site. The reason travels in the provenance's `issue` field, so `tp submit`
+    (which reads the saved report rather than re-probing the checkouts) names it
+    too. `allow` (--allow-unanchored / TRAP_ALLOW_UNANCHORED, for CI) keeps the
+    warning but skips the prompt; with no TTY and no `allow` it refuses.
+    """
+    sides = {"solution": provenance.solution, "task": provenance.task}
+    missing = {name: side.issue for name, side in sides.items() if not side.repo}
+    if not missing:
+        return
+    for name, reason in missing.items():
+        suffix = f" ({reason})" if reason else ""
+        err_console.print(f"[yellow]⚠  {name} has no git provenance{suffix}[/yellow]")
+    err_console.print(
+        "[yellow]   trapstreet accepts the submission, but the leaderboard hides runs that "
+        "aren't anchored to a commit on a remote — run from a clean, committed checkout "
+        "with an origin remote to make it rankable.[/yellow]"
+    )
+    if allow:
+        return
+    if not sys.stdin.isatty():
+        raise _die(
+            "unanchored provenance needs confirmation; pass --allow-unanchored "
+            "(or set TRAP_ALLOW_UNANCHORED=1) to proceed non-interactively"
+        )
+    if not typer.confirm("Continue anyway?", default=False):
+        raise typer.Exit(code=1)
+
+
 @app.command()
 def run(
     task: Annotated[str | None, typer.Argument()] = None,
@@ -76,6 +111,14 @@ def run(
             "(git+ URL) solution/task. Also settable via TRAP_TRUST_REMOTE=1.",
         ),
     ] = False,
+    allow_unanchored: Annotated[
+        bool,
+        typer.Option(
+            "--allow-unanchored",
+            help="Skip the confirmation when the run has no git provenance (the "
+            "leaderboard hides such runs). Also settable via TRAP_ALLOW_UNANCHORED=1.",
+        ),
+    ] = False,
     tags: Annotated[list[str] | None, typer.Option("--tag", "-t")] = None,
     output: Annotated[OutputFormat, typer.Option("--output", "-o")] = OutputFormat.rich,
     fail_fast: Annotated[bool, typer.Option("--fail-fast")] = False,
@@ -138,6 +181,16 @@ def run(
     except (GitOpsError, ConfigError, subprocess.CalledProcessError) as e:
         raise _die(e) from None
 
+    # Record git provenance (repo + commit) of both checkouts — solution and task —
+    # so the run is reproducible; an unanchored side carries an `issue` naming why.
+    # Probed once, before any case runs: an unanchored checkout produces a report the
+    # leaderboard will hide, so the user must acknowledge (or pre-authorise) that.
+    provenance = Provenance(
+        solution=LocalRepo.provenance_of(trap_yaml_loader.trap_dir),
+        task=LocalRepo.provenance_of(traptask_yaml_loader.traptask_dir),
+    )
+    _confirm_unanchored(provenance, allow=allow_unanchored or _env_truthy("TRAP_ALLOW_UNANCHORED"))
+
     active_cases = traptask_yaml_loader.cases_with_tags(tags or [])
 
     started_at_local = datetime.now()
@@ -162,13 +215,6 @@ def run(
         )
     finished_at_utc = datetime.now(UTC)
 
-    # Record git provenance (repo + commit) of both checkouts — solution and task —
-    # so the run is reproducible. Each side is empty for a dirty/remote-less/local tree.
-    provenance = Provenance(
-        solution=LocalRepo.provenance_of(trap_yaml_loader.trap_dir),
-        task=LocalRepo.provenance_of(traptask_yaml_loader.traptask_dir),
-    )
-
     # Capture the host machine environment (CPU/RAM/OS/Python) unless disabled.
     # Detection is best-effort and must never abort a completed run.
     environment_info = None
@@ -232,6 +278,14 @@ def submit(
     ] = None,
     workspace: Annotated[Path, typer.Option("--workspace", "-w")] = Path(".trap"),
     run: Annotated[str, typer.Option("--run", "-r", help="Which run to upload.")] = "latest",
+    allow_unanchored: Annotated[
+        bool,
+        typer.Option(
+            "--allow-unanchored",
+            help="Skip the confirmation when the run has no git provenance (the "
+            "leaderboard hides such runs). Also settable via TRAP_ALLOW_UNANCHORED=1.",
+        ),
+    ] = False,
 ) -> None:
     """Upload a report.json to trapstreet.
 
@@ -254,9 +308,14 @@ def submit(
         raise _die(e) from None
     report_handle = ReportHandle(workspace.resolve(), task_alias, run)
     try:
-        report_handle.assert_exists()
+        report_data = report_handle.load()
     except FileNotFoundError:
         raise _die(f"no report at {report_handle.report_json_path}. Run [bold]tp run[/bold] first.") from None
+    # Repeat the unanchored-provenance gate at upload time — the report records
+    # what `tp run` saw, so the checkouts aren't re-probed here.
+    _confirm_unanchored(
+        report_data.provenance, allow=allow_unanchored or _env_truthy("TRAP_ALLOW_UNANCHORED")
+    )
     report_path = report_handle.report_json_path
 
     client = ApiClient(resolved_server, resolved_key)

trap_cli-0.0.3/src/trap/git_ops/local.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import git
+
+from trap.git_ops.url import ParsedGitUrl
+from trap.models.provenance import GitProvenance
+
+
+class LocalRepo:
+    """An existing on-disk git checkout — read-only inspection.
+
+    Distinct from `RemoteRepo` (which clones a declared URL into a root): this wraps
+    a `git.Repo` already on disk — whether trap cloned it or the user pointed at
+    a local solution. Both clone-sync validation and report provenance go through
+    here, so the "open repo + read origin/commit/dirty" logic lives in one place.
+    """
+
+    def __init__(self, repo: git.Repo, path: Path) -> None:
+        self.repo = repo
+        # The path this checkout was opened at (may sit below the repo root when
+        # opened with search_parent).
+        self.path = path
+
+    @classmethod
+    def open(cls, path: Path, *, search_parent: bool = False) -> LocalRepo | None:
+        """Open the git checkout at `path`, or None if it isn't a git repo."""
+        try:
+            return cls(git.Repo(path, search_parent_directories=search_parent), path)
+        except (git.InvalidGitRepositoryError, git.NoSuchPathError):
+            return None
+
+    @property
+    def origin_normalised_url(self) -> str | None:
+        """`origin` remote as a canonical https URL, or None if there's no origin."""
+        try:
+            return ParsedGitUrl.from_full_url(self.repo.remotes.origin.url).normalised_url
+        except AttributeError:
+            return None
+
+    def provenance_issue(self) -> str | None:
+        """Why this checkout cannot be anchored to {repo, commit}, or None if it can.
+
+        The run isn't reproducible from remote+commit alone when the repo has no
+        origin, no commit, or uncommitted tracked-file changes. Best-effort: a probe
+        failure degrades to a reason rather than raising. Untracked files (run
+        outputs under .trap/, .venv, …) don't count as dirty.
+        """
+        try:
+            if self.origin_normalised_url is None:
+                return "no origin remote"
+            if not self.repo.head.is_valid():
+                return "no commit to anchor to"
+            if self.repo.is_dirty():
+                return "uncommitted changes"
+            return None
+        except Exception:
+            return "git probe failed"
+
+    @property
+    def subdirectory(self) -> str | None:
+        """Opened path relative to the repo root, None at the root (or in a bare repo)."""
+        root = self.repo.working_tree_dir
+        if root is None:
+            return None
+        rel = self.path.resolve().relative_to(Path(root).resolve())
+        return rel.as_posix() if rel != Path(".") else None
+
+    def provenance(self) -> GitProvenance:
+        """{repo, commit, subdirectory} for a clean checkout with an origin, else empty
+        with `issue` naming why — we claim nothing about a run that isn't reproducible."""
+        issue = self.provenance_issue()
+        if issue is not None:
+            return GitProvenance(issue=issue)
+        return GitProvenance(
+            repo=self.origin_normalised_url,
+            commit=self.repo.head.commit.hexsha,
+            subdirectory=self.subdirectory,
+        )
+
+    @classmethod
+    def provenance_of(cls, path: Path) -> GitProvenance:
+        """Provenance ({repo, commit}) of the checkout at `path`."""
+        local_repo = cls.open(path, search_parent=True)
+        return local_repo.provenance() if local_repo else GitProvenance(issue="not a git repo")

{trap_cli-0.0.1 → trap_cli-0.0.3}/src/trap/models/provenance.py

@@ -4,11 +4,17 @@ from pydantic import BaseModel, Field
 
 
 class GitProvenance(BaseModel):
-    """Git origin of one checkout: {repo, commit}. Both None when the tree isn't a
-    clean, remote-backed git repo (see LocalRepo.provenance)."""
+    """Git origin of one checkout: {repo, commit, subdirectory}. repo and commit are
+    None when the tree isn't a clean, remote-backed git repo — `issue` then names why
+    (see LocalRepo.provenance_issue). Anchored checkouts carry no issue.
+
+    `subdirectory` locates the checkout within the repo (None at the root), so two
+    solutions or tasks living in different subdirectories of one repo stay distinct."""
 
     repo: str | None = None
     commit: str | None = None
+    subdirectory: str | None = None
+    issue: str | None = None
 
 
 class Provenance(BaseModel):

{trap_cli-0.0.1 → trap_cli-0.0.3}/src/trap/models/report.py

@@ -28,6 +28,9 @@ class ReportData(BaseModel):
     started_at_utc: str
     finished_at_utc: str
 
+    # Leaderboard identity the solution author chose in trap.yaml (`name`); None →
+    # the server auto-assigns one.
+    solution_name: str | None = None
     # Engine identity (model/framework). Self-reported from trap.yaml today, but the
     # website consumes it from the report — never from trap.yaml directly — precisely
     # because the source may change: a future version is expected to derive this by
@@ -56,6 +59,7 @@ class ReportData(BaseModel):
             grader_metrics=grader_metrics,
             started_at_utc=started_at_utc.isoformat(timespec="seconds"),
             finished_at_utc=finished_at_utc.isoformat(timespec="seconds"),
+            solution_name=trap_config.name,
             profile=trap_config.profile,
             environment=environment,
         )

trap_cli-0.0.1/src/trap/git_ops/local.py

@@ -1,60 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-
-import git
-
-from trap.git_ops.url import ParsedGitUrl
-from trap.models.provenance import GitProvenance
-
-
-class LocalRepo:
-    """An existing on-disk git checkout — read-only inspection.
-
-    Distinct from `RemoteRepo` (which clones a declared URL into a root): this wraps
-    a `git.Repo` already on disk — whether trap cloned it or the user pointed at
-    a local solution. Both clone-sync validation and report provenance go through
-    here, so the "open repo + read origin/commit/dirty" logic lives in one place.
-    """
-
-    def __init__(self, repo: git.Repo) -> None:
-        self.repo = repo
-
-    @classmethod
-    def open(cls, path: Path, *, search_parent: bool = False) -> LocalRepo | None:
-        """Open the git checkout at `path`, or None if it isn't a git repo."""
-        try:
-            return cls(git.Repo(path, search_parent_directories=search_parent))
-        except (git.InvalidGitRepositoryError, git.NoSuchPathError):
-            return None
-
-    @property
-    def origin_normalised_url(self) -> str | None:
-        """`origin` remote as a canonical https URL, or None if there's no origin."""
-        try:
-            return ParsedGitUrl.from_full_url(self.repo.remotes.origin.url).normalised_url
-        except AttributeError:
-            return None
-
-    def provenance(self) -> GitProvenance:
-        """{repo, commit} for a clean checkout with an origin, else empty.
-
-        Empty for a dirty tree (tracked-file changes), a remote-less repo, or any
-        git error — the run isn't reproducible from remote+commit alone, so we
-        claim nothing. Best-effort: a probe failure degrades to empty rather than
-        aborting an otherwise-complete run. Untracked files (run outputs under
-        .trap/, .venv, …) don't count as dirty.
-        """
-        try:
-            url = self.origin_normalised_url
-            if url is None or not self.repo.head.is_valid() or self.repo.is_dirty():
-                return GitProvenance()
-            return GitProvenance(repo=url, commit=self.repo.head.commit.hexsha)
-        except Exception:
-            return GitProvenance()
-
-    @classmethod
-    def provenance_of(cls, path: Path) -> GitProvenance:
-        """Provenance ({repo, commit}) of the checkout at `path`, empty if not a git repo."""
-        local_repo = cls.open(path, search_parent=True)
-        return local_repo.provenance() if local_repo else GitProvenance()