trap-cli 0.0.1__tar.gz

trap_cli-0.0.1/.gitignore

@@ -0,0 +1,45 @@
+LOCAL*
+.env
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.so
+
+# Distribution / packaging
+dist/
+build/
+*.egg-info/
+.eggs/
+
+# uv / virtual envs
+.venv/
+uv.lock~
+
+# hatch-vcs generated
+src/trap/_version.py
+
+# trap runtime workspace
+.trap/
+
+# MkDocs generated site
+site/
+
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.deepeval
+
+# Ruff
+.ruff_cache/
+
+# macOS
+.DS_Store
+
+# Editors
+.idea/
+.vscode/
+*.swp

trap_cli-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 trapstreet.run
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

trap_cli-0.0.1/PKG-INFO

@@ -0,0 +1,88 @@
+Metadata-Version: 2.4
+Name: trap-cli
+Version: 0.0.1
+Summary: tp — non-invasive CLI testing framework for AI workflows. Submits results to trapstreet.run.
+Project-URL: Homepage, https://trapstreet.run
+Project-URL: Repository, https://github.com/trapstreet/trap
+Project-URL: Issues, https://github.com/trapstreet/trap/issues
+Author-email: "trapstreet.run" <founder@trapstreet.run>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agent,ai,benchmark,eval,llm,trapstreet,workflow
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.13
+Requires-Dist: gitpython>=3.1
+Requires-Dist: httpx>=0.27
+Requires-Dist: psutil>=7.2.2
+Requires-Dist: py-cpuinfo>=9.0.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0
+Requires-Dist: tokencost>=0.1.26
+Requires-Dist: typer>=0.12
+Description-Content-Type: text/markdown
+
+# trap
+
+[![CI](https://github.com/trapstreet/trap/actions/workflows/ci.yml/badge.svg)](https://github.com/trapstreet/trap/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/trap-cli)](https://pypi.org/project/trap-cli/)
+[![Python](https://img.shields.io/pypi/pyversions/trap-cli)](https://pypi.org/project/trap-cli/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+
+**Non-invasive CLI testing framework for AI prompts, agents, and workflows.**
+
+trap treats any solution as a black box — it invokes it as a subprocess, captures outputs, then optionally scores them through a language-agnostic judge and grader. The solution doesn't need to import trap or know it exists.
+
+## Install
+
+```bash
+# requires uv — https://docs.astral.sh/uv/getting-started/installation/
+
+# from PyPI
+uv tool install trap-cli
+
+# from git (latest main)
+uv tool install "git+https://github.com/trapstreet/trap.git"
+```
+
+The command is `tp`.
+
+## How it works
+
+trap runs your solution as a subprocess with one env var (`TRAP_MANIFEST` — an input dir and
+an output dir), captures what it writes, then optionally scores it through a judge (per case)
+and a grader (overall). See the [full flow and IO contract](docs/index.md).
+
+Two roles, two directories, one IO contract:
+
+- **Solution author** — writes `trap.yaml` and the solution code
+- **Task author** — writes `traptask.yaml`, `inputs/`, `expected/`, and optional judge/grader scripts
+
+## Quick start
+
+```bash
+# from examples/echo/solution/
+tp run           # run all cases
+tp run -t smoke  # run only cases tagged `smoke`
+```
+
+## Documentation
+
+- [Quick start](docs/quickstart.md)
+- [Writing a solution](docs/guides/writing-solution.md)
+- [Writing a task](docs/guides/writing-task.md)
+- [CLI reference](docs/reference/cli.md)
+- [IO contract](docs/reference/io-contract.md)
+
+## License
+
+MIT

trap_cli-0.0.1/README.md

@@ -0,0 +1,55 @@
+# trap
+
+[![CI](https://github.com/trapstreet/trap/actions/workflows/ci.yml/badge.svg)](https://github.com/trapstreet/trap/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/trap-cli)](https://pypi.org/project/trap-cli/)
+[![Python](https://img.shields.io/pypi/pyversions/trap-cli)](https://pypi.org/project/trap-cli/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+
+**Non-invasive CLI testing framework for AI prompts, agents, and workflows.**
+
+trap treats any solution as a black box — it invokes it as a subprocess, captures outputs, then optionally scores them through a language-agnostic judge and grader. The solution doesn't need to import trap or know it exists.
+
+## Install
+
+```bash
+# requires uv — https://docs.astral.sh/uv/getting-started/installation/
+
+# from PyPI
+uv tool install trap-cli
+
+# from git (latest main)
+uv tool install "git+https://github.com/trapstreet/trap.git"
+```
+
+The command is `tp`.
+
+## How it works
+
+trap runs your solution as a subprocess with one env var (`TRAP_MANIFEST` — an input dir and
+an output dir), captures what it writes, then optionally scores it through a judge (per case)
+and a grader (overall). See the [full flow and IO contract](docs/index.md).
+
+Two roles, two directories, one IO contract:
+
+- **Solution author** — writes `trap.yaml` and the solution code
+- **Task author** — writes `traptask.yaml`, `inputs/`, `expected/`, and optional judge/grader scripts
+
+## Quick start
+
+```bash
+# from examples/echo/solution/
+tp run           # run all cases
+tp run -t smoke  # run only cases tagged `smoke`
+```
+
+## Documentation
+
+- [Quick start](docs/quickstart.md)
+- [Writing a solution](docs/guides/writing-solution.md)
+- [Writing a task](docs/guides/writing-task.md)
+- [CLI reference](docs/reference/cli.md)
+- [IO contract](docs/reference/io-contract.md)
+
+## License
+
+MIT

trap_cli-0.0.1/examples/echo/task/README.md

@@ -0,0 +1,39 @@
+# Echo Task
+
+Your solution receives a JSON object via stdin and must echo the `message` field to stdout.
+
+## Input
+
+A JSON object delivered via stdin:
+
+```json
+{"message": "Hello, World!"}
+```
+
+## Output
+
+The value of `message`, printed to stdout (with a trailing newline).
+
+```
+Hello, World!
+```
+
+## Error handling
+
+If the `message` field is missing, print an error to stderr and exit with code 1.
+
+## Evaluation
+
+Each case declares one or more assertions in its `expected.json` file.
+The judge (`judge.py`) checks each declared assertion against the solution's stdout and exit code:
+
+
+| Assertion     | Passes when                                          |
+|---------------|------------------------------------------------------|
+| `contains`    | stdout contains the substring (case-insensitive)     |
+| `not_contains`| stdout does not contain the substring                |
+| `exact`       | stdout matches exactly (after strip)                 |
+| `regex`       | stdout matches the regex pattern                     |
+| `exit_code`   | process exit code equals the value                   |
+
+All declared assertions must pass for a case to pass.

trap_cli-0.0.1/pyproject.toml

@@ -0,0 +1,107 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "trap-cli"
+dynamic = ["version"]
+description = "tp — non-invasive CLI testing framework for AI workflows. Submits results to trapstreet.run."
+readme = "README.md"
+license = "MIT"
+authors = [{ name = "trapstreet.run", email = "founder@trapstreet.run" }]
+keywords = ["eval", "ai", "llm", "benchmark", "trapstreet", "agent", "workflow"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Software Development :: Testing",
+]
+requires-python = ">=3.13"
+dependencies = [
+    "typer>=0.12",
+    "pydantic>=2.0",
+    "pyyaml>=6.0",
+    "rich>=13.0",
+    "httpx>=0.27",
+    "tokencost>=0.1.26",
+    "gitpython>=3.1",
+    "psutil>=7.2.2",
+    "py-cpuinfo>=9.0.0",
+]
+
+[project.urls]
+Homepage = "https://trapstreet.run"
+Repository = "https://github.com/trapstreet/trap"
+Issues = "https://github.com/trapstreet/trap/issues"
+
+[project.scripts]
+tp = "trap.cli:app"
+
+[tool.hatch.version]
+source = "vcs"
+
+# Version comes from git tags shaped like `vX.Y.Z`. Options are forwarded
+# to setuptools_scm; `fallback_version` keeps untagged builds working.
+[tool.hatch.version.raw-options]
+tag_regex = "^v(?P<version>.+)$"
+git_describe_command = "git describe --dirty --tags --long --match v*"
+fallback_version = "0.0.0.dev0"
+
+[tool.hatch.build.hooks.vcs]
+version-file = "src/trap/_version.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/trap"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src/trap", "README.md", "pyproject.toml"]
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0",
+    "pytest-cov>=5.0",
+    "ruff>=0.9",
+]
+
+[tool.uv.workspace]
+exclude = ["examples/*"]
+
+[tool.pyright]
+# Type-check the package; tests intentionally feed pydantic models loose input
+# (scalars/dicts its validators coerce), which strict typing would flag.
+include = ["src"]
+
+[tool.ruff]
+line-length = 110
+target-version = "py313"
+extend-exclude = ["src/trap/_version.py"]  # generated by hatch-vcs
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "C4", "RUF"]
+
+[tool.ruff.lint.per-file-ignores]
+"src/trap/_version.py" = ["RUF022"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.coverage.run]
+source = ["trap"]
+branch = true
+omit = ["src/trap/_version.py"]
+
+[tool.coverage.report]
+show_missing = true
+fail_under = 100
+exclude_also = [
+    "if TYPE_CHECKING:",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+    "@(abc\\.)?abstractmethod",
+]

trap_cli-0.0.1/src/trap/__init__.py

@@ -0,0 +1,6 @@
+try:
+    from trap._version import __version__
+except ImportError:  # pragma: no cover - only when built without hatch-vcs
+    __version__ = "0.0.0+unknown"
+
+__all__ = ["__version__"]

trap_cli-0.0.1/src/trap/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.0.1'
+__version_tuple__ = version_tuple = (0, 0, 1)
+
+__commit_id__ = commit_id = None

trap_cli-0.0.1/src/trap/auth/__init__.py

@@ -0,0 +1,15 @@
+from trap.auth.client import ApiClient, ApiError
+from trap.auth.login import BrowserProvider, TokenProvider
+from trap.auth.oauth import OAuthCallbackServer
+from trap.auth.store import DEFAULT_SERVER, AuthData, AuthStore
+
+__all__ = [
+    "DEFAULT_SERVER",
+    "ApiClient",
+    "ApiError",
+    "AuthData",
+    "AuthStore",
+    "BrowserProvider",
+    "OAuthCallbackServer",
+    "TokenProvider",
+]

trap_cli-0.0.1/src/trap/auth/client.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from functools import cached_property
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+
+class ApiError(Exception):
+    """A trapstreet API call failed — bad status, unreachable server, or invalid token.
+    Carries a user-facing message; the CLI maps it to a clean error (no traceback)."""
+
+
+class ApiClient:
+    """Authenticated HTTP client for the trapstreet API."""
+
+    def __init__(self, server: str, api_key: str, timeout: int = 30) -> None:
+        self._server = server.rstrip("/")
+        self._api_key = api_key
+        self._timeout = timeout
+
+    @cached_property
+    def _client(self) -> httpx.Client:
+        return httpx.Client(
+            base_url=self._server,
+            headers={
+                "authorization": f"Bearer {self._api_key}",
+                "content-type": "application/json",
+            },
+            timeout=self._timeout,
+        )
+
+    def get_me(self) -> dict[str, Any]:
+        try:
+            resp = self._client.get("/api/me", timeout=10)
+            resp.raise_for_status()
+            return resp.json()
+        except httpx.HTTPStatusError as e:
+            if e.response.status_code == 401:
+                raise ApiError("token is invalid") from None
+            raise ApiError(f"server error ({e.response.status_code})") from None
+        except httpx.RequestError:
+            raise ApiError("server unreachable") from None
+
+    def submit(self, report_path: Path) -> dict[str, Any]:
+        # Content-addressed ingest: the task identity travels inside the report
+        # (provenance.task.{repo,commit}), not the URL — so no task_id path segment.
+        try:
+            resp = self._client.post("/api/submit", content=report_path.read_bytes())
+            resp.raise_for_status()
+            return resp.json()
+        except httpx.HTTPStatusError as e:
+            raise ApiError(f"http {e.response.status_code}: {e.response.text}") from None
+        except httpx.RequestError as e:
+            raise ApiError(f"connection error: {e}") from None

trap_cli-0.0.1/src/trap/auth/login.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from trap.auth.oauth import OAuthCallbackServer
+from trap.auth.store import DEFAULT_SERVER, AuthData
+
+
+class TokenProvider:
+    def __init__(self, server: str, token: str) -> None:
+        self._server = server
+        self._token = token
+
+    @property
+    def pre_message(self) -> str:
+        return "authenticating with api_key"
+
+    def acquire(self) -> AuthData:
+        if not self._token:
+            raise ValueError("empty api_key")
+        return AuthData(server=self._server, api_key=self._token)
+
+
+class BrowserProvider:
+    def __init__(self, server: str, timeout: int) -> None:
+        if server != DEFAULT_SERVER:
+            raise ValueError(
+                f"browser login is only supported on {DEFAULT_SERVER}. Use --with-token for custom servers."
+            )
+        self._cb = OAuthCallbackServer(server)
+        self._timeout = timeout
+
+    @property
+    def pre_message(self) -> str:
+        return f"opening [link={self._cb.auth_url}]{self._cb.auth_url}[/link]"
+
+    def acquire(self) -> AuthData:
+        if not self._cb.run(self._timeout):
+            raise TimeoutError(f"timed out after {self._timeout}s waiting for browser approval")
+        assert self._cb.auth_data is not None
+        return self._cb.auth_data

trap_cli-0.0.1/src/trap/auth/oauth.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import http.server
+import socketserver
+import threading
+import urllib.parse
+import webbrowser
+from typing import Any
+
+from trap.auth.store import AuthData
+
+
+class _CallbackHandler(http.server.BaseHTTPRequestHandler):
+    _SUCCESS_HTML = (
+        b"<!doctype html><meta charset=utf-8>"
+        b"<title>logged in</title>"
+        b"<style>body{font-family:ui-monospace,monospace;"
+        b"background:#0a0a0a;color:#ededed;padding:3em;}"
+        b"h1{color:#ff5f1f}</style>"
+        b"<h1>logged in</h1>"
+        b"<p>You can close this tab.</p>"
+    )
+
+    def log_message(self, format: str, *args: Any) -> None:
+        return
+
+    def _respond(self, status: int, content_type: str, body: bytes) -> None:
+        self.send_response(status)
+        self.send_header("content-type", content_type)
+        self.end_headers()
+        self.wfile.write(body)
+
+    def do_GET(self) -> None:
+        params = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query)
+
+        api_key = (params.get("api_key") or [None])[0]
+        if not api_key:
+            self._respond(400, "text/plain", b"missing api_key in query string")
+            return
+
+        solution = (params.get("solution") or [None])[0]
+        srv: OAuthCallbackServer = self.server  # type: ignore[assignment]
+        srv._receive(AuthData(server=srv._server_url, api_key=api_key, solution=solution))
+        self._respond(200, "text/html; charset=utf-8", self._SUCCESS_HTML)
+
+
+class OAuthCallbackServer(socketserver.TCPServer):
+    """Local HTTP server that receives the OAuth callback with api_key."""
+
+    def __init__(self, server_url: str) -> None:
+        super().__init__(("127.0.0.1", 0), _CallbackHandler)
+        self._server_url = server_url.rstrip("/")
+        self._auth_data: AuthData | None = None
+        self._received = threading.Event()
+
+    def _receive(self, data: AuthData) -> None:
+        self._auth_data = data
+        self._received.set()
+
+    @property
+    def auth_data(self) -> AuthData | None:
+        return self._auth_data
+
+    @property
+    def port(self) -> int:
+        return self.server_address[1]
+
+    @property
+    def auth_url(self) -> str:
+        return f"{self._server_url}/cli/authorize?return=http://localhost:{self.port}/callback"
+
+    def run(self, timeout: int) -> bool:
+        """Start server, open browser, wait for callback, shut down. Returns True if api_key received."""
+        threading.Thread(target=self.serve_forever, daemon=True).start()
+        try:
+            webbrowser.open(self.auth_url)
+        except Exception:
+            pass
+        received = self._received.wait(timeout)
+        self.shutdown()
+        self.server_close()
+        return received

trap_cli-0.0.1/src/trap/auth/store.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import json as _json
+from pathlib import Path
+
+from pydantic import BaseModel, ValidationError
+
+DEFAULT_SERVER = "https://trapstreet.run"
+
+
+class AuthData(BaseModel):
+    server: str
+    api_key: str
+    solution: str | None = None
+
+
+class AuthStore:
+    PATH = Path.home() / ".config" / "trapstreet" / "auth.json"
+
+    def load(self) -> AuthData | None:
+        if not self.exists:
+            return None
+        try:
+            return AuthData.model_validate(_json.loads(self.PATH.read_text()))
+        except (OSError, _json.JSONDecodeError, ValidationError):
+            return None
+
+    def save(self, data: AuthData) -> Path:
+        self.PATH.parent.mkdir(parents=True, exist_ok=True)
+        self.PATH.write_text(data.model_dump_json(exclude_none=True, indent=2) + "\n")
+        self.PATH.chmod(0o600)
+        return self.PATH
+
+    @property
+    def exists(self) -> bool:
+        return self.PATH.exists()
+
+    def delete(self) -> None:
+        self.PATH.unlink()