trap-cli 0.0.1__py3-none-any.whl

trap/runner/solution.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+from trap.cost import CostProxy
+from trap.models import CaseResult
+from trap.models.cost import CaseCost
+from trap.runner.proc import CapturedSubprocess
+
+if TYPE_CHECKING:
+    from trap.runner.layout import CaseLayout
+    from trap.runner.task import TaskRunner
+
+
+class SolutionRunner:
+    def __init__(self, runner: TaskRunner, case_id: str, layout: CaseLayout) -> None:
+        self.runner = runner
+        self.case_id = case_id
+        self.case_inputs_dir = runner.task_inputs_dir / case_id  # task-repo side
+        self.layout = layout  # workspace side
+
+    @property
+    def _stdin(self) -> str:
+        stdin = self.runner.trap_config.stdin
+        if stdin:
+            return (self.case_inputs_dir / stdin).read_text()
+        return ""
+
+    @property
+    def _manifest(self) -> str:
+        return json.dumps(
+            {
+                "inputs_dir": str(self.case_inputs_dir.resolve()),
+                "outputs_dir": str(self.layout.outputs_dir.resolve()),
+            }
+        )
+
+    def run(self) -> CaseResult:
+        self.layout.outputs_dir.mkdir(parents=True, exist_ok=True)
+
+        proxy: CostProxy | None = None
+        proxy_env: dict[str, str] = {}
+        if self.runner.cost_enabled:
+            try:
+                proxy = CostProxy()
+                proxy.start()
+                proxy_env = proxy.env_overrides
+            except Exception:
+                pass
+
+        # A timeout is folded into the result (exit 124, partial output) rather than
+        # raised — for the solution it counts as "did not complete", never a crash.
+        config = self.runner.trap_config
+        proc = CapturedSubprocess(
+            config.cmd,
+            manifest_envvar=config.manifest_envvar,
+            timeout=config.timeout,
+            cwd=self.runner.trap_dir,
+            manifest=self._manifest,
+            capture=self.layout.solution_capture,
+            stdin=self._stdin,
+            env_extra=proxy_env,
+        )
+        case_cost: CaseCost | None = None
+        try:
+            result = proc.run()
+        finally:
+            if proxy is not None:
+                partial = proxy.stop()
+                if partial.calls > 0:
+                    case_cost = partial
+
+        return CaseResult(
+            case_id=self.case_id,
+            exit_code=result.exit_code,
+            duration=result.duration,
+            metrics=None,
+            cost=case_cost,
+        )

trap/runner/task.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable, Iterator
+from datetime import datetime
+from functools import cached_property
+from pathlib import Path
+from typing import Any
+
+from trap.models import CaseResult, TrapConfig, TraptaskCase, TraptaskConfig
+from trap.runner.grader import GraderRunner
+from trap.runner.judge import JudgeRunner
+from trap.runner.layout import CaseLayout
+from trap.runner.solution import SolutionRunner
+
+
+class TaskRunner:
+    def __init__(
+        self,
+        trap_config: TrapConfig,
+        trap_dir: Path,
+        traptask_dir: Path,
+        traptask_config: TraptaskConfig,
+        run_dir: Path,
+        cost_enabled: bool = True,
+    ) -> None:
+        self.trap_config = trap_config
+        self.trap_dir = trap_dir
+        self.traptask_config = traptask_config
+        self.traptask_dir = traptask_dir
+        self.run_dir = run_dir
+        self.cost_enabled = cost_enabled
+
+    @cached_property
+    def task_inputs_dir(self) -> Path:
+        """The task's inputs/ dir (traptask_dir / dirs.inputs), resolved once on first use."""
+        return (self.traptask_dir / self.traptask_config.dirs.inputs).resolve()
+
+    @cached_property
+    def task_expected_dir(self) -> Path:
+        """The task's expected/ dir (traptask_dir / dirs.expected), resolved once on first use."""
+        return (self.traptask_dir / self.traptask_config.dirs.expected).resolve()
+
+    def _update_latest(self) -> None:
+        latest = self.run_dir.parent / "latest"
+        if latest.is_symlink():
+            latest.unlink()
+        elif latest.exists():
+            # Path exists but isn't a symlink — likely an interrupted prior run
+            # or a sync tool that materialized the symlink as a real directory.
+            # Move it aside (non-destructive) so subsequent runs self-heal.
+            suffix = datetime.now().isoformat(timespec="microseconds")
+            latest.rename(latest.with_name(f"latest.broken.{suffix}"))
+        latest.symlink_to(self.run_dir.name)
+
+    def _iter_cases(
+        self,
+        cases: Iterable[TraptaskCase],
+        *,
+        fail_fast: bool = False,
+        on_case_start: Callable[[str], None] | None = None,
+        on_case_done: Callable[[CaseResult], None] | None = None,
+    ) -> Iterator[CaseResult]:
+        # TODO: parallelize case runs, but judge cases sequentially in the same order as case runs
+        for case in cases:
+            if on_case_start is not None:
+                on_case_start(case.id)
+            layout = CaseLayout.for_case(self.run_dir, case.id)
+            case_result = SolutionRunner(self, case.id, layout).run()
+            if self.traptask_config.judge is not None:
+                # A broken judge is folded into an error metric by the runner (never
+                # crashes the run); we just attach whatever it returns.
+                metrics = JudgeRunner(self, case.id, layout).run()
+                case_result = case_result.model_copy(update={"metrics": metrics})
+            if on_case_done is not None:
+                on_case_done(case_result)
+            yield case_result
+            if fail_fast and case_result.exit_code != 0:
+                break
+
+    def run(
+        self,
+        cases: Iterable[TraptaskCase],
+        *,
+        fail_fast: bool = False,
+        on_case_start: Callable[[str], None] | None = None,
+        on_case_done: Callable[[CaseResult], None] | None = None,
+    ) -> tuple[tuple[CaseResult, ...], Any]:
+
+        case_results = tuple(
+            self._iter_cases(
+                cases, fail_fast=fail_fast, on_case_start=on_case_start, on_case_done=on_case_done
+            )
+        )
+
+        grader_metrics = None
+        if self.traptask_config.grader is not None:
+            # A broken grader is folded into an error metric by the runner (the run
+            # still completes and the report still saves).
+            grader_metrics = GraderRunner(self, case_results).run()
+
+        self._update_latest()
+        return case_results, grader_metrics

trap_cli-0.0.1.dist-info/METADATA

@@ -0,0 +1,88 @@
+Metadata-Version: 2.4
+Name: trap-cli
+Version: 0.0.1
+Summary: tp — non-invasive CLI testing framework for AI workflows. Submits results to trapstreet.run.
+Project-URL: Homepage, https://trapstreet.run
+Project-URL: Repository, https://github.com/trapstreet/trap
+Project-URL: Issues, https://github.com/trapstreet/trap/issues
+Author-email: "trapstreet.run" <founder@trapstreet.run>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agent,ai,benchmark,eval,llm,trapstreet,workflow
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.13
+Requires-Dist: gitpython>=3.1
+Requires-Dist: httpx>=0.27
+Requires-Dist: psutil>=7.2.2
+Requires-Dist: py-cpuinfo>=9.0.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0
+Requires-Dist: tokencost>=0.1.26
+Requires-Dist: typer>=0.12
+Description-Content-Type: text/markdown
+
+# trap
+
+[![CI](https://github.com/trapstreet/trap/actions/workflows/ci.yml/badge.svg)](https://github.com/trapstreet/trap/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/trap-cli)](https://pypi.org/project/trap-cli/)
+[![Python](https://img.shields.io/pypi/pyversions/trap-cli)](https://pypi.org/project/trap-cli/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+
+**Non-invasive CLI testing framework for AI prompts, agents, and workflows.**
+
+trap treats any solution as a black box — it invokes it as a subprocess, captures outputs, then optionally scores them through a language-agnostic judge and grader. The solution doesn't need to import trap or know it exists.
+
+## Install
+
+```bash
+# requires uv — https://docs.astral.sh/uv/getting-started/installation/
+
+# from PyPI
+uv tool install trap-cli
+
+# from git (latest main)
+uv tool install "git+https://github.com/trapstreet/trap.git"
+```
+
+The command is `tp`.
+
+## How it works
+
+trap runs your solution as a subprocess with one env var (`TRAP_MANIFEST` — an input dir and
+an output dir), captures what it writes, then optionally scores it through a judge (per case)
+and a grader (overall). See the [full flow and IO contract](docs/index.md).
+
+Two roles, two directories, one IO contract:
+
+- **Solution author** — writes `trap.yaml` and the solution code
+- **Task author** — writes `traptask.yaml`, `inputs/`, `expected/`, and optional judge/grader scripts
+
+## Quick start
+
+```bash
+# from examples/echo/solution/
+tp run           # run all cases
+tp run -t smoke  # run only cases tagged `smoke`
+```
+
+## Documentation
+
+- [Quick start](docs/quickstart.md)
+- [Writing a solution](docs/guides/writing-solution.md)
+- [Writing a task](docs/guides/writing-task.md)
+- [CLI reference](docs/reference/cli.md)
+- [IO contract](docs/reference/io-contract.md)
+
+## License
+
+MIT

trap_cli-0.0.1.dist-info/RECORD

@@ -0,0 +1,54 @@
+trap/__init__.py,sha256=UA2VQ2W2CY2JiBDDO7nQeYJUJuniitUWzKzFVXKGzBI,184
+trap/_version.py,sha256=8OsTLsIVB9D0HdPTmt5rVwyVUBe9xTVGkRslXicxzkM,520
+trap/auth/__init__.py,sha256=5bkDjtwT8GwDe_yof0i5RK35mmqDKw60UQjkVc1J0Ns,394
+trap/auth/client.py,sha256=6p-kJmOstolFbZ0CUgxrISxOprUawvNdhtDtBEFfC2Y,2059
+trap/auth/login.py,sha256=GfK2vxWZu3Zbj6U4QS0Wr1nmMSsYk5vbZpqdapj5UTc,1283
+trap/auth/oauth.py,sha256=yDqiwV4PX0xRzMta1P2o0bHpIvTAtgqhc_5EyOAKebE,2726
+trap/auth/store.py,sha256=GAw1HdxaQWPJG7ybHx1FriB71Y2ZnS81OKA5TwGG49s,1010
+trap/cli/__init__.py,sha256=VuLDC4t9uOhsqBUwXc8vtavyO44xagjKJbG2X-WLC-k,10728
+trap/cli/_auth.py,sha256=pg-0UonfqUOpVrzyPzMjTvu1v3wxNi4s-GL6r3dBwrE,3615
+trap/cost/__init__.py,sha256=dZgUNIwI_1TJGNXOy7LSRx0rWwGnVDDW2dl2mkEKcX0,54
+trap/cost/calculator.py,sha256=jo4CQnqEr_UTn4aNjv7n7UUnIPzplfs3leZ4nhRLep4,1710
+trap/cost/providers.py,sha256=BeRIwjsEDi7ASxQ9CIBwOKme9IAPF1MAjcnhat7PbFM,4965
+trap/cost/proxy.py,sha256=LJPqcYkHrWJBMuzC0P-8ozfm2nOI5wJV-3O1Mq8JtLg,7246
+trap/display/__init__.py,sha256=3AFDEPoPvDRFeyn94IAWdw6DZIHNCuAGx7982Ez_Uto,152
+trap/display/progress.py,sha256=FCM1BuEb-z5QwvCwlJ9xnUgZVEmNmAIRLrdVgGl9s68,1954
+trap/display/submit.py,sha256=rA12JI9zIRxybRTEzZd3BMfEsVzCZHY9VbdcgZlYOgY,719
+trap/environment/__init__.py,sha256=TYHi2fnhRAZ3Am-GJo5-J89XY5QpKZEKYRHX1Er7j9c,77
+trap/environment/detector.py,sha256=QJLaQSWtaQEc2Vk5ew2-AEPulamEjXybY0iAB2mUwwc,2632
+trap/git_ops/__init__.py,sha256=eA9RWw-vFK4B2x4NAkwXgckrEDpDgb64gtOigJQ1pAg,293
+trap/git_ops/base.py,sha256=hcbcjU6gvl3XtITvwwa5hpbjylfh2Q2rE2w5jungUkQ,252
+trap/git_ops/local.py,sha256=2ANpyeJ7mOnIlqFLw4398WoVAYHCeRQ4RnKbrdm7d8k,2470
+trap/git_ops/remote.py,sha256=L_tBQVgQ4ReMKWJ6fUEAtoKHGg1m5_did6DuYw_I0co,2596
+trap/git_ops/rev.py,sha256=IxnvU_NsiO9YI2f9XAz3MONF2jL14hXMQnVkek59Y0g,4402
+trap/git_ops/url.py,sha256=6NqpWixb6v2e8ultqwcmTn-yfs0QpPOkoZNSkJiK8Eg,3861
+trap/loader/__init__.py,sha256=cFfdpu4_LR4wSPEk9Q5f5hr-fmyzcMjyk0S_DV3dO9Q,167
+trap/loader/errors.py,sha256=rGe4VT_06mVp_CgVKj-JbvLAEQIOXaJFlM3it7fYdbs,325
+trap/loader/trap_yaml.py,sha256=-IVHh8pAVT2IhznddUJlHOiFZkDXcBf5eXZCyCBUge0,4218
+trap/loader/traptask_yaml.py,sha256=3ITDiJ5fVYZgVjVh_udwH3RFxlxcWtN9394v4Fezp7A,4168
+trap/models/__init__.py,sha256=RDpYjr_E7s0ynFajfdPSqaneCtRX03hPQ6nU2y4rGPI,755
+trap/models/cost.py,sha256=K-FciMrWB2tXu6PEFVM-ZKfk8EZnR3_MS_hIpaxpC4Y,831
+trap/models/environment.py,sha256=1C2e3Dp-XsUZLH2fgVrIpn3REBauDtpLaHw-VTEzceE,747
+trap/models/provenance.py,sha256=fPUWa2Si5rT08vA5LtOZ5cvPRg7ZvVb-zEM-QMfHakE,705
+trap/models/report.py,sha256=ojI8zXMSxtsN1Lh1T1dtWS1WKD9GQLOAFG7u_DOIVg0,2374
+trap/models/results.py,sha256=_Q8PT8gOFZNxMarEJyFML8LIbmxbDMyVmcuWD_z0c_A,432
+trap/models/trap_yaml.py,sha256=xRYmwjknYJmMqDk8Y6S6kNxpcQrPZPRBI26xsjlHc14,2977
+trap/models/traptask_yaml.py,sha256=htD7G6zJcpOPFH8LcrRejr1tmS8hG52VNyj5PIImMPQ,2404
+trap/report/__init__.py,sha256=snW1TEmWZYdbGXkvlXMF0b14rOXYXNolP6jJxzry5Nc,547
+trap/report/base.py,sha256=UGXSHiD3Te3_0UyzkMADpSdw8Rs7yFFGuz5KhfEPgiY,175
+trap/report/handle.py,sha256=Piitl2yOjgMwMtJMD8jrrwD7BLc5ht0SPYa4YvV75XQ,1676
+trap/report/json.py,sha256=9RUtQFGjyAOei-q8C7qi95-XrlogoA3gfMjC0KTaOLY,273
+trap/report/rich.py,sha256=CbSd3i0hBJSOtYSXJiqj-HK7-Lzz89syg_hgfmTwiHA,5023
+trap/runner/__init__.py,sha256=2VleaPrx206KgFUdBEe2CjCIcNrTZMOCBkZ8mij3O4A,102
+trap/runner/capture.py,sha256=pMHsGUZvGS-l9Ngi9XPcZlBa-64p_eTBaRXViO1wGVY,1017
+trap/runner/grader.py,sha256=zaRK_tBtzBtLAMI54moA4_yxaS5My1tJWKK743PXzqA,1241
+trap/runner/judge.py,sha256=jnzwNp-IPYBCNTb7iCmXoKjUaVJd4DDOIVUp0Giv5Ng,1844
+trap/runner/layout.py,sha256=em-q1hZHSbAYaJWwBNmU-GhmFgbIt20Bvsm6kyvFmDA,1118
+trap/runner/proc.py,sha256=GaAWsxdIpvNmeupwk9MVKkHhG-zDdzo3AVqp6X_zbuk,4972
+trap/runner/solution.py,sha256=AWG0d2hr4yS9yFfpSF0ycTTKJTDOwsQg6Q4Q2eQA1-g,2498
+trap/runner/task.py,sha256=2j6TInZhHKCANG4XogyIgydw2BQjdZxJ0agssVgF49Y,4033
+trap_cli-0.0.1.dist-info/METADATA,sha256=LYN_Yn8E-gFgFjpotNcE9TmSaYvhvIRHIGUPW48-qgg,3188
+trap_cli-0.0.1.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+trap_cli-0.0.1.dist-info/entry_points.txt,sha256=wLIOVpx-cjscwnuIRPTuuXLyXs7K31r68xA7Gwt-Bmk,36
+trap_cli-0.0.1.dist-info/licenses/LICENSE,sha256=ZZ7IeDjFVJJOC6qoVFoHWcysqbip79BElVZvAb5ScTY,1071
+trap_cli-0.0.1.dist-info/RECORD,,

trap_cli-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

trap_cli-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+tp = trap.cli:app

trap_cli-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 trapstreet.run
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.