agent-eval-rpc 0.21.0__tar.gz

agent_eval_rpc-0.21.0/.gitignore

@@ -0,0 +1,8 @@
+.venv/
+__pycache__/
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+build/
+dist/
+*.pyc

agent_eval_rpc-0.21.0/PKG-INFO

@@ -0,0 +1,198 @@
+Metadata-Version: 2.4
+Name: agent-eval-rpc
+Version: 0.21.0
+Summary: Python RPC client for @tangle-network/agent-eval — judge content against rubrics over HTTP or stdio RPC. Eval logic runs in the Node runtime; this package is a thin wire client.
+Project-URL: Homepage, https://github.com/tangle-network/agent-eval
+Project-URL: Issues, https://github.com/tangle-network/agent-eval/issues
+Project-URL: Documentation, https://github.com/tangle-network/agent-eval/blob/main/clients/python/README.md
+Author: Tangle Network
+License: MIT
+Keywords: agent,evaluation,judge,llm,rubric
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: pydantic>=2.6
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# agent-eval-rpc — Python client
+
+Python client for [`@tangle-network/agent-eval`](https://github.com/tangle-network/agent-eval) — a content/code judging framework written in TypeScript. This package is a **thin transport adapter**: every judgement runs in the Node runtime, marshalled over HTTP or stdio RPC. Two languages, one implementation. No drift.
+
+## What you get
+
+A function-call interface to score any string against a rubric:
+
+```python
+from agent_eval_rpc import Client
+
+client = Client()  # auto-detects HTTP server, falls back to subprocess
+result = client.judge(
+    content="We just launched zero-copy IO between agents and their workdir",
+    rubric_name="anti-slop",
+)
+
+print(result.composite)         # 0.0..1.0 — single number to gate on
+print(result.dimensions)        # {"buyer_quality": 0.7, "voice": 0.8, "signal": 0.9}
+print(result.failure_modes)     # [] or ["ai-cadence", "marketing-tone", ...]
+print(result.wins)              # ["specific-component", "earned-detail", ...]
+print(result.rationale)         # "The post names a real architectural detail..."
+```
+
+That's the entire surface for content judging.
+
+## Install
+
+```sh
+cd clients/python
+pip install -e .
+```
+
+To use it, **one of**:
+
+- `npm install -g @tangle-network/agent-eval` — gives you the `agent-eval` binary, used by the subprocess transport (works offline, slower per call due to Node startup ~500ms).
+- Run a server: `agent-eval serve --port 5005` — gives you HTTP transport (~10ms per call once up).
+
+The Python client picks whichever is available. Force one with `Client(transport="http")` or `Client(transport="subprocess")`.
+
+## Why the architecture works this way
+
+The TypeScript package is the source of truth for evaluation logic. We don't reimplement rubrics, scoring, or judges in Python — we marshal JSON to the canonical runtime over a versioned wire protocol (defined as Zod schemas, exported as OpenAPI, mirrored in this package as pydantic models).
+
+Adding a new method to the API means: define a Zod schema in `src/wire/schemas.ts`, write the handler in `src/wire/handlers.ts`, and the Python client picks it up on the next regeneration. **There is no separate Python implementation to maintain.**
+
+This is the same pattern as the Anthropic SDK, Stripe SDK, and gRPC: one canonical implementation, language-specific transport clients.
+
+## API
+
+### `Client`
+
+```python
+Client(
+    base_url: str | None = None,        # AGENT_EVAL_URL or http://127.0.0.1:5005
+    cli_path: str | None = None,        # AGENT_EVAL_CLI or 'agent-eval'
+    transport: "auto" | "http" | "subprocess" = "auto",
+    timeout_s: float = 120.0,
+)
+```
+
+### `client.judge(...)`
+
+Score a piece of content against a rubric.
+
+```python
+def judge(
+    *,
+    content: str,                                  # the text being judged
+    rubric_name: str | None = None,                # OR
+    rubric: Rubric | dict | None = None,           # an inline rubric definition
+    context: dict | None = None,                   # free-form metadata for the judge
+    model: str | None = None,                      # override the judge LLM
+) -> JudgeResult
+```
+
+**Either** `rubric_name` (use a built-in like `"anti-slop"`) **or** `rubric` (an inline definition with your own dimensions/prompt). Not both.
+
+**Returns** `JudgeResult`:
+- `composite: float` — weighted score in 0..1. The single number to gate on.
+- `dimensions: dict[str, float]` — per-axis scores (e.g. `{"buyer_quality": 0.7}`).
+- `failure_modes: list[str]` — ids of negative patterns detected.
+- `wins: list[str]` — ids of positive patterns detected.
+- `rationale: str` — plain-English explanation.
+- `rubric_version: str` — stable hash of the rubric used. Compare scores only when this matches.
+- `model: str` — LLM that produced the judgement.
+- `duration_ms: int` — wall-clock latency.
+
+### `client.list_rubrics()`
+
+Return every rubric the server has registered, with their dimensions and stable `rubric_version`.
+
+```python
+rubrics = client.list_rubrics()
+for r in rubrics.rubrics:
+    print(r.name, r.description, r.rubric_version)
+```
+
+### `client.version()`
+
+Return server + wire-protocol version. Match your `pip install` version to `version`; check `wire_version` for compatibility.
+
+```python
+v = client.version()
+assert v.version.startswith("0.20")
+assert v.wire_version == "1.0.0"
+```
+
+## Defining a custom rubric
+
+Built-in `anti-slop` is tuned for technical-buyer audiences. For different scoring, pass a `Rubric` inline:
+
+```python
+from agent_eval_rpc import Client, Rubric, RubricDimension, FailureMode
+
+rubric = Rubric(
+    name="my-rubric",
+    description="Does this commit message explain WHY, not just what?",
+    systemPrompt="You score commit messages. Score 0..1 on whether the WHY is clear...",
+    dimensions=[
+        RubricDimension(id="explains_why", description="Does the message say *why*?", weight=1.0),
+    ],
+    failureModes=[
+        FailureMode(id="what-not-why", description="States the change but not the reason"),
+    ],
+)
+
+result = client.judge(content="bumped the version", rubric=rubric)
+```
+
+## Errors
+
+| Exception | When |
+|---|---|
+| `ValidationError` | Server (or pydantic) rejected the request as malformed. Fix your inputs. |
+| `RubricNotFoundError` | Unknown `rubric_name`. Call `list_rubrics()` to see what's registered. |
+| `TransportError` | HTTP unreachable or subprocess failed. Retry or check the server. |
+| `AgentEvalError` | Base class — catches everything above. |
+
+All errors carry `.code` and `.details` (the structured payload from the server).
+
+## Versioning
+
+This package is **version-locked** to the npm package. `agent-eval-rpc==0.21.0` ↔ `@tangle-network/agent-eval@0.21.0`. CI verifies the npm package, Python package, runtime `__version__`, and release tag all agree before publish. If one registry publish fails after the other succeeds, retry the failed publish from the same tag or supersede with the next patch release.
+
+`wire_version` is separate. It bumps only on breaking schema changes. Package versions can differ across releases as long as `wire_version` is the same.
+
+## Development
+
+```sh
+# install in editable mode
+pip install -e ".[dev]"
+
+# unit tests (no Node required)
+pytest tests/test_models.py
+
+# integration tests against the bundled CLI
+cd ../.. && pnpm build         # build the agent-eval CLI in repo root
+cd clients/python && pytest    # runs subprocess tests against dist/cli.js
+```
+
+## Adding a new method
+
+When the TS side adds a new endpoint (say `evaluateScenario`):
+1. Update `src/wire/schemas.ts` with `EvaluateScenarioRequestSchema` and `EvaluateScenarioResponseSchema`.
+2. Add a handler in `src/wire/handlers.ts`, route in `src/wire/server.ts`, and case in `src/wire/rpc.ts`.
+3. In this client, add the matching pydantic model in `models.py` and method on `Client`. The pattern is mechanical — copy the shape from `judge`.
+4. Test in both languages. Bump versions together.
+
+A future iteration moves step 3 to `datamodel-code-generator -i openapi.json` so it's mechanical-and-automatic instead of mechanical-by-hand. Until the surface grows past ~10 endpoints, hand-written models are more readable.

agent_eval_rpc-0.21.0/README.md

@@ -0,0 +1,170 @@
+# agent-eval-rpc — Python client
+
+Python client for [`@tangle-network/agent-eval`](https://github.com/tangle-network/agent-eval) — a content/code judging framework written in TypeScript. This package is a **thin transport adapter**: every judgement runs in the Node runtime, marshalled over HTTP or stdio RPC. Two languages, one implementation. No drift.
+
+## What you get
+
+A function-call interface to score any string against a rubric:
+
+```python
+from agent_eval_rpc import Client
+
+client = Client()  # auto-detects HTTP server, falls back to subprocess
+result = client.judge(
+    content="We just launched zero-copy IO between agents and their workdir",
+    rubric_name="anti-slop",
+)
+
+print(result.composite)         # 0.0..1.0 — single number to gate on
+print(result.dimensions)        # {"buyer_quality": 0.7, "voice": 0.8, "signal": 0.9}
+print(result.failure_modes)     # [] or ["ai-cadence", "marketing-tone", ...]
+print(result.wins)              # ["specific-component", "earned-detail", ...]
+print(result.rationale)         # "The post names a real architectural detail..."
+```
+
+That's the entire surface for content judging.
+
+## Install
+
+```sh
+cd clients/python
+pip install -e .
+```
+
+To use it, **one of**:
+
+- `npm install -g @tangle-network/agent-eval` — gives you the `agent-eval` binary, used by the subprocess transport (works offline, slower per call due to Node startup ~500ms).
+- Run a server: `agent-eval serve --port 5005` — gives you HTTP transport (~10ms per call once up).
+
+The Python client picks whichever is available. Force one with `Client(transport="http")` or `Client(transport="subprocess")`.
+
+## Why the architecture works this way
+
+The TypeScript package is the source of truth for evaluation logic. We don't reimplement rubrics, scoring, or judges in Python — we marshal JSON to the canonical runtime over a versioned wire protocol (defined as Zod schemas, exported as OpenAPI, mirrored in this package as pydantic models).
+
+Adding a new method to the API means: define a Zod schema in `src/wire/schemas.ts`, write the handler in `src/wire/handlers.ts`, and the Python client picks it up on the next regeneration. **There is no separate Python implementation to maintain.**
+
+This is the same pattern as the Anthropic SDK, Stripe SDK, and gRPC: one canonical implementation, language-specific transport clients.
+
+## API
+
+### `Client`
+
+```python
+Client(
+    base_url: str | None = None,        # AGENT_EVAL_URL or http://127.0.0.1:5005
+    cli_path: str | None = None,        # AGENT_EVAL_CLI or 'agent-eval'
+    transport: "auto" | "http" | "subprocess" = "auto",
+    timeout_s: float = 120.0,
+)
+```
+
+### `client.judge(...)`
+
+Score a piece of content against a rubric.
+
+```python
+def judge(
+    *,
+    content: str,                                  # the text being judged
+    rubric_name: str | None = None,                # OR
+    rubric: Rubric | dict | None = None,           # an inline rubric definition
+    context: dict | None = None,                   # free-form metadata for the judge
+    model: str | None = None,                      # override the judge LLM
+) -> JudgeResult
+```
+
+**Either** `rubric_name` (use a built-in like `"anti-slop"`) **or** `rubric` (an inline definition with your own dimensions/prompt). Not both.
+
+**Returns** `JudgeResult`:
+- `composite: float` — weighted score in 0..1. The single number to gate on.
+- `dimensions: dict[str, float]` — per-axis scores (e.g. `{"buyer_quality": 0.7}`).
+- `failure_modes: list[str]` — ids of negative patterns detected.
+- `wins: list[str]` — ids of positive patterns detected.
+- `rationale: str` — plain-English explanation.
+- `rubric_version: str` — stable hash of the rubric used. Compare scores only when this matches.
+- `model: str` — LLM that produced the judgement.
+- `duration_ms: int` — wall-clock latency.
+
+### `client.list_rubrics()`
+
+Return every rubric the server has registered, with their dimensions and stable `rubric_version`.
+
+```python
+rubrics = client.list_rubrics()
+for r in rubrics.rubrics:
+    print(r.name, r.description, r.rubric_version)
+```
+
+### `client.version()`
+
+Return server + wire-protocol version. Match your `pip install` version to `version`; check `wire_version` for compatibility.
+
+```python
+v = client.version()
+assert v.version.startswith("0.20")
+assert v.wire_version == "1.0.0"
+```
+
+## Defining a custom rubric
+
+Built-in `anti-slop` is tuned for technical-buyer audiences. For different scoring, pass a `Rubric` inline:
+
+```python
+from agent_eval_rpc import Client, Rubric, RubricDimension, FailureMode
+
+rubric = Rubric(
+    name="my-rubric",
+    description="Does this commit message explain WHY, not just what?",
+    systemPrompt="You score commit messages. Score 0..1 on whether the WHY is clear...",
+    dimensions=[
+        RubricDimension(id="explains_why", description="Does the message say *why*?", weight=1.0),
+    ],
+    failureModes=[
+        FailureMode(id="what-not-why", description="States the change but not the reason"),
+    ],
+)
+
+result = client.judge(content="bumped the version", rubric=rubric)
+```
+
+## Errors
+
+| Exception | When |
+|---|---|
+| `ValidationError` | Server (or pydantic) rejected the request as malformed. Fix your inputs. |
+| `RubricNotFoundError` | Unknown `rubric_name`. Call `list_rubrics()` to see what's registered. |
+| `TransportError` | HTTP unreachable or subprocess failed. Retry or check the server. |
+| `AgentEvalError` | Base class — catches everything above. |
+
+All errors carry `.code` and `.details` (the structured payload from the server).
+
+## Versioning
+
+This package is **version-locked** to the npm package. `agent-eval-rpc==0.21.0` ↔ `@tangle-network/agent-eval@0.21.0`. CI verifies the npm package, Python package, runtime `__version__`, and release tag all agree before publish. If one registry publish fails after the other succeeds, retry the failed publish from the same tag or supersede with the next patch release.
+
+`wire_version` is separate. It bumps only on breaking schema changes. Package versions can differ across releases as long as `wire_version` is the same.
+
+## Development
+
+```sh
+# install in editable mode
+pip install -e ".[dev]"
+
+# unit tests (no Node required)
+pytest tests/test_models.py
+
+# integration tests against the bundled CLI
+cd ../.. && pnpm build         # build the agent-eval CLI in repo root
+cd clients/python && pytest    # runs subprocess tests against dist/cli.js
+```
+
+## Adding a new method
+
+When the TS side adds a new endpoint (say `evaluateScenario`):
+1. Update `src/wire/schemas.ts` with `EvaluateScenarioRequestSchema` and `EvaluateScenarioResponseSchema`.
+2. Add a handler in `src/wire/handlers.ts`, route in `src/wire/server.ts`, and case in `src/wire/rpc.ts`.
+3. In this client, add the matching pydantic model in `models.py` and method on `Client`. The pattern is mechanical — copy the shape from `judge`.
+4. Test in both languages. Bump versions together.
+
+A future iteration moves step 3 to `datamodel-code-generator -i openapi.json` so it's mechanical-and-automatic instead of mechanical-by-hand. Until the surface grows past ~10 endpoints, hand-written models are more readable.

agent_eval_rpc-0.21.0/pyproject.toml

@@ -0,0 +1,54 @@
+[build-system]
+requires = ["hatchling>=1.21"]
+build-backend = "hatchling.build"
+
+[project]
+name = "agent-eval-rpc"
+version = "0.21.0"
+description = "Python RPC client for @tangle-network/agent-eval — judge content against rubrics over HTTP or stdio RPC. Eval logic runs in the Node runtime; this package is a thin wire client."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [{ name = "Tangle Network" }]
+keywords = ["evaluation", "llm", "rubric", "agent", "judge"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Software Development :: Quality Assurance",
+]
+dependencies = [
+  "httpx>=0.27",
+  "pydantic>=2.6",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.0",
+  "pytest-asyncio>=0.23",
+  "ruff>=0.6",
+]
+
+[project.urls]
+Homepage = "https://github.com/tangle-network/agent-eval"
+Issues = "https://github.com/tangle-network/agent-eval/issues"
+Documentation = "https://github.com/tangle-network/agent-eval/blob/main/clients/python/README.md"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/agent_eval_rpc"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "B", "UP"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-ra -q"

agent_eval_rpc-0.21.0/src/agent_eval_rpc/__init__.py

@@ -0,0 +1,67 @@
+"""agent-eval-rpc — Python RPC client for @tangle-network/agent-eval.
+
+The TypeScript package is the source of truth for evaluation logic. This
+client is a thin transport adapter — every judgement runs in the Node
+runtime, marshalled over HTTP or stdio RPC. Two languages, one
+implementation.
+
+The package distributes as ``agent-eval-rpc`` on PyPI and imports as
+``agent_eval_rpc`` to make the wire-client nature explicit; the rubric
+logic lives upstream in ``@tangle-network/agent-eval`` on npm.
+
+Quickstart
+----------
+
+    from agent_eval_rpc import Client
+
+    client = Client()  # auto-detects HTTP server, falls back to subprocess
+    result = client.judge(content="our scaffold supports zero-copy IO", rubric_name="anti-slop")
+    print(result.composite, result.failure_modes)
+
+Or as a one-shot using the bundled `agent-eval` CLI:
+
+    result = client.judge(content="…", rubric={"name": "custom", ...})
+
+See README.md for the full guide.
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from .client import Client
+from .errors import (
+    AgentEvalError,
+    RubricNotFoundError,
+    TransportError,
+    ValidationError,
+)
+from .models import (
+    FailureMode,
+    JudgeRequest,
+    JudgeResult,
+    ListRubricsResponse,
+    Rubric,
+    RubricDimension,
+    RubricInfo,
+    VersionResponse,
+)
+
+try:
+    __version__ = version("agent-eval-rpc")
+except PackageNotFoundError:
+    __version__ = "0.21.0"
+
+__all__ = [
+    "Client",
+    "AgentEvalError",
+    "TransportError",
+    "RubricNotFoundError",
+    "ValidationError",
+    "JudgeRequest",
+    "JudgeResult",
+    "Rubric",
+    "RubricDimension",
+    "FailureMode",
+    "RubricInfo",
+    "ListRubricsResponse",
+    "VersionResponse",
+]

agent_eval_rpc-0.21.0/src/agent_eval_rpc/client.py

@@ -0,0 +1,215 @@
+"""Client — the public entry point.
+
+Two transports, one API:
+
+- HTTP (default if reachable): talks to a running `agent-eval serve`.
+  Best for live agent paths and high-frequency calls.
+- Subprocess (fallback / explicit): shells out to `agent-eval rpc <method>`.
+  Best for batch / cron — no service to manage.
+
+Auto-detection: if `base_url` reaches a running server in `auto_probe_timeout`
+seconds, HTTP wins. Otherwise the client falls back to subprocess. Force one
+transport with `transport="http"` or `transport="subprocess"`.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+from typing import Any, Literal
+
+import httpx
+
+from .errors import AgentEvalError, TransportError, from_error_body
+from .models import (
+    JudgeRequest,
+    JudgeResult,
+    ListRubricsResponse,
+    Rubric,
+    VersionResponse,
+)
+
+Transport = Literal["http", "subprocess", "auto"]
+
+DEFAULT_BASE_URL = "http://127.0.0.1:5005"
+DEFAULT_CLI = "agent-eval"
+DEFAULT_TIMEOUT_S = 120.0
+
+
+class Client:
+    """Synchronous client for agent-eval.
+
+    Parameters
+    ----------
+    base_url:
+        Where to find the HTTP server. Defaults to AGENT_EVAL_URL env var
+        or http://127.0.0.1:5005.
+    cli_path:
+        Name or absolute path of the `agent-eval` binary used by the
+        subprocess transport. Defaults to AGENT_EVAL_CLI or 'agent-eval'.
+    transport:
+        'auto' (default), 'http', or 'subprocess'.
+    timeout_s:
+        Per-call timeout, default 120 seconds. Judges are allowed up to
+        ~60s server-side, so 120s is comfortably above that.
+    auto_probe_timeout:
+        How long to wait for the HTTP /healthz check during auto-detect.
+    """
+
+    def __init__(
+        self,
+        base_url: str | None = None,
+        *,
+        cli_path: str | None = None,
+        transport: Transport = "auto",
+        timeout_s: float = DEFAULT_TIMEOUT_S,
+        auto_probe_timeout: float = 1.0,
+    ) -> None:
+        self.base_url = (base_url or os.environ.get("AGENT_EVAL_URL") or DEFAULT_BASE_URL).rstrip("/")
+        self.cli_path = cli_path or os.environ.get("AGENT_EVAL_CLI") or DEFAULT_CLI
+        self.timeout_s = timeout_s
+        self._transport = self._resolve_transport(transport, auto_probe_timeout)
+
+    # ── Public methods ──────────────────────────────────────────────
+
+    def judge(
+        self,
+        *,
+        content: str,
+        rubric_name: str | None = None,
+        rubric: Rubric | dict[str, Any] | None = None,
+        context: dict[str, Any] | None = None,
+        model: str | None = None,
+    ) -> JudgeResult:
+        """Score `content` against a rubric and return a typed result."""
+        # Validate locally so the user sees a Python-side error before the
+        # transport even fires. The server validates again as defense in depth.
+        rubric_value: Rubric | None
+        if isinstance(rubric, dict):
+            rubric_value = Rubric.model_validate(rubric)
+        else:
+            rubric_value = rubric
+        request = JudgeRequest(
+            rubric_name=rubric_name,
+            rubric=rubric_value,
+            content=content,
+            context=context,
+            model=model,
+        )
+        body = self._call("judge", request.model_dump(by_alias=True, exclude_none=True))
+        return JudgeResult.model_validate(body)
+
+    def list_rubrics(self) -> ListRubricsResponse:
+        body = self._call("listRubrics", {})
+        return ListRubricsResponse.model_validate(body)
+
+    def version(self) -> VersionResponse:
+        body = self._call("version", {})
+        return VersionResponse.model_validate(body)
+
+    @property
+    def transport(self) -> Literal["http", "subprocess"]:
+        return self._transport
+
+    # ── Transport dispatch ──────────────────────────────────────────
+
+    def _resolve_transport(
+        self, requested: Transport, probe_timeout: float
+    ) -> Literal["http", "subprocess"]:
+        if requested == "http":
+            return "http"
+        if requested == "subprocess":
+            return "subprocess"
+        # auto: probe HTTP first
+        try:
+            with httpx.Client(timeout=probe_timeout) as c:
+                r = c.get(f"{self.base_url}/healthz")
+                if r.status_code == 200:
+                    return "http"
+        except (httpx.HTTPError, OSError):
+            pass
+        if shutil.which(self.cli_path) is None:
+            raise TransportError(
+                f"No HTTP server at {self.base_url} and no `{self.cli_path}` binary on PATH. "
+                "Either run `agent-eval serve` or `npm i -g @tangle-network/agent-eval`."
+            )
+        return "subprocess"
+
+    def _call(self, method: str, params: dict[str, Any]) -> Any:
+        if self._transport == "http":
+            return self._http_call(method, params)
+        return self._subprocess_call(method, params)
+
+    def _http_call(self, method: str, params: dict[str, Any]) -> Any:
+        path = _http_path_for(method)
+        try:
+            with httpx.Client(timeout=self.timeout_s, base_url=self.base_url) as c:
+                if path.method == "GET":
+                    r = c.get(path.url)
+                else:
+                    r = c.post(path.url, json=params)
+        except httpx.HTTPError as e:
+            raise TransportError(f"HTTP transport failed: {e}") from e
+        if r.status_code >= 400:
+            try:
+                raise from_error_body(r.status_code, r.json())
+            except (ValueError, json.JSONDecodeError):
+                raise TransportError(f"HTTP {r.status_code}: {r.text[:500]}")
+        try:
+            return r.json()
+        except json.JSONDecodeError as e:
+            raise TransportError(f"Server returned non-JSON body: {e}") from e
+
+    def _subprocess_call(self, method: str, params: dict[str, Any]) -> Any:
+        try:
+            proc = subprocess.run(
+                [self.cli_path, "rpc", method],
+                input=json.dumps(params),
+                capture_output=True,
+                text=True,
+                timeout=self.timeout_s,
+                check=False,
+            )
+        except (FileNotFoundError, subprocess.TimeoutExpired) as e:
+            raise TransportError(f"Subprocess transport failed: {e}") from e
+        if not proc.stdout:
+            raise TransportError(
+                f"agent-eval rpc {method} produced no output. stderr: {proc.stderr[:500]}"
+            )
+        try:
+            envelope = json.loads(proc.stdout.strip().splitlines()[-1])
+        except json.JSONDecodeError as e:
+            raise TransportError(f"agent-eval rpc returned non-JSON: {proc.stdout[:500]}") from e
+        if "error" in envelope:
+            # Map to the right exception class — same as HTTP path.
+            raise from_error_body(proc.returncode or 500, envelope)
+        if "result" not in envelope:
+            raise TransportError(f"Malformed RPC envelope: {envelope}")
+        return envelope["result"]
+
+
+# ── Method → HTTP path mapping ──────────────────────────────────────
+
+
+class _HttpPath:
+    __slots__ = ("method", "url")
+
+    def __init__(self, method: str, url: str) -> None:
+        self.method = method
+        self.url = url
+
+
+_PATHS = {
+    "judge": _HttpPath("POST", "/v1/judge"),
+    "listRubrics": _HttpPath("GET", "/v1/rubrics"),
+    "version": _HttpPath("GET", "/v1/version"),
+}
+
+
+def _http_path_for(method: str) -> _HttpPath:
+    try:
+        return _PATHS[method]
+    except KeyError as e:
+        raise AgentEvalError(f"Unknown method: {method}") from e

agent_eval_rpc-0.21.0/src/agent_eval_rpc/errors.py

@@ -0,0 +1,52 @@
+"""Exception hierarchy.
+
+All errors raised by this client subclass `AgentEvalError`. Catch the
+specific ones (`RubricNotFoundError`, `ValidationError`) for cases that
+are fixable in caller code; let `TransportError` bubble or retry it.
+"""
+
+from __future__ import annotations
+
+
+class AgentEvalError(Exception):
+    """Base class for every error raised by this client."""
+
+    def __init__(self, message: str, *, code: str | None = None, details: object = None) -> None:
+        super().__init__(message)
+        self.code = code
+        self.details = details
+
+
+class TransportError(AgentEvalError):
+    """The HTTP request or subprocess invocation failed at the transport layer.
+
+    Distinct from server-side errors (which arrive as 4xx with a typed
+    body — those map to other subclasses). TransportError = the request
+    couldn't be made or the response couldn't be parsed.
+    """
+
+
+class ValidationError(AgentEvalError):
+    """Server rejected the request as malformed (HTTP 400 with code='validation_error')."""
+
+
+class RubricNotFoundError(AgentEvalError):
+    """Server has no rubric by that name (HTTP 404 with code='rubric_not_found')."""
+
+
+def from_error_body(status: int, body: object) -> AgentEvalError:
+    """Map a server error envelope to the right exception class."""
+    code = None
+    message = "Unknown error"
+    details = None
+    if isinstance(body, dict):
+        err = body.get("error")
+        if isinstance(err, dict):
+            code = err.get("code")
+            message = err.get("message", message)
+            details = err.get("details")
+    if code == "rubric_not_found":
+        return RubricNotFoundError(message, code=code, details=details)
+    if code == "validation_error":
+        return ValidationError(message, code=code, details=details)
+    return AgentEvalError(f"HTTP {status}: {message}", code=code, details=details)

agent_eval_rpc-0.21.0/src/agent_eval_rpc/models.py

@@ -0,0 +1,124 @@
+"""Data models that mirror the wire-protocol Zod schemas.
+
+These pydantic models are kept in sync by hand for now — the surface is
+small (six classes). When the wire surface grows past ~10 endpoints,
+swap this file for `datamodel-code-generator -i openapi.json -o models.py`.
+
+Every field name and type matches `src/wire/schemas.ts` exactly. If you
+change one without changing the other, the dual-publish CI will fail.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class _StrictModel(BaseModel):
+    """Reject unknown fields — drift between TS and Python should fail loudly."""
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RubricDimension(_StrictModel):
+    """A scoring axis within a rubric.
+
+    Composite scores combine each dimension by `weight`. The `min`/`max`
+    bounds are used to normalize raw scores into 0..1 before weighting.
+    """
+
+    id: str = Field(..., description="Stable id like 'buyer_quality'.")
+    description: str = Field(..., description="One-line plain-English meaning.")
+    weight: float = Field(1.0, ge=0, description="Relative weight in composite. 0 disables.")
+    min: float = 0.0
+    max: float = 1.0
+
+
+class FailureMode(_StrictModel):
+    """A negative pattern the judge looks for. Detected ones appear in result.failure_modes."""
+
+    id: str
+    description: str
+
+
+class Rubric(_StrictModel):
+    """A complete rubric — what's being scored and how.
+
+    Pass this inline to `Client.judge(rubric=...)` or register a built-in
+    rubric server-side and use `Client.judge(rubric_name=...)`.
+    """
+
+    name: str
+    description: str
+    system_prompt: str = Field(..., alias="systemPrompt")
+    dimensions: list[RubricDimension]
+    failure_modes: list[FailureMode] = Field(default_factory=list, alias="failureModes")
+    wins: list[FailureMode] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+
+class JudgeRequest(_StrictModel):
+    """Input to /v1/judge. Provide either rubric_name or rubric (not both)."""
+
+    rubric_name: str | None = Field(None, alias="rubricName")
+    rubric: Rubric | None = None
+    content: str = Field(..., min_length=1, description="The text being judged.")
+    context: dict[str, Any] | None = Field(
+        None,
+        description="Free-form metadata surfaced to the judging LLM.",
+    )
+    model: str | None = Field(None, description="Override the judge model.")
+
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+    @model_validator(mode="after")
+    def _exactly_one_rubric(self) -> JudgeRequest:
+        if (self.rubric_name is None) == (self.rubric is None):
+            raise ValueError("Provide exactly one of `rubric_name` or `rubric`.")
+        return self
+
+
+class JudgeResult(_StrictModel):
+    """Output of /v1/judge. The `composite` is the 0..1 score to gate on."""
+
+    composite: float = Field(..., ge=0, le=1)
+    dimensions: dict[str, float]
+    failure_modes: list[str] = Field(default_factory=list, alias="failureModes")
+    wins: list[str] = Field(default_factory=list)
+    rationale: str
+    rubric_version: str = Field(..., alias="rubricVersion")
+    model: str
+    duration_ms: int = Field(..., alias="durationMs")
+
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+
+class RubricInfo(_StrictModel):
+    """One entry in /v1/rubrics."""
+
+    name: str
+    description: str
+    dimensions: list[dict[str, Any]]
+    failure_modes: list[str] = Field(default_factory=list, alias="failureModes")
+    rubric_version: str = Field(..., alias="rubricVersion")
+
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+
+class ListRubricsResponse(_StrictModel):
+    """Response from /v1/rubrics."""
+
+    rubrics: list[RubricInfo]
+
+
+class VersionResponse(_StrictModel):
+    """Response from /v1/version. Match `version` to your installed pip package."""
+
+    package: str
+    version: str
+    wire_version: str = Field(..., alias="wireVersion")
+    api_surface: list[str] = Field(..., alias="apiSurface")
+
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)

agent_eval_rpc-0.21.0/tests/test_models.py

@@ -0,0 +1,67 @@
+"""Schema mirror tests — defend against TS/Python drift.
+
+Each test names the regression it would catch. The invariant: anything
+the TypeScript JudgeRequest accepts/rejects, the Python JudgeRequest
+must accept/reject the same way.
+"""
+from __future__ import annotations
+
+import pytest
+
+from agent_eval_rpc.models import JudgeRequest, Rubric, RubricDimension
+
+
+MIN_RUBRIC = Rubric(
+    name="r",
+    description="d",
+    systemPrompt="p",
+    dimensions=[RubricDimension(id="a", description="b")],
+)
+
+
+def test_judge_request_accepts_rubric_name_alone() -> None:
+    JudgeRequest(rubric_name="anti-slop", content="hello")
+
+
+def test_judge_request_accepts_inline_rubric_alone() -> None:
+    JudgeRequest(rubric=MIN_RUBRIC, content="hello")
+
+
+def test_judge_request_rejects_both_rubric_name_and_rubric() -> None:
+    """Regression: ambiguous selection — server must not have to choose."""
+    with pytest.raises(ValueError, match="exactly one"):
+        JudgeRequest(rubric_name="anti-slop", rubric=MIN_RUBRIC, content="hello")
+
+
+def test_judge_request_rejects_neither_rubric_name_nor_rubric() -> None:
+    """Regression: silently dispatching to default rubric hides bugs."""
+    with pytest.raises(ValueError, match="exactly one"):
+        JudgeRequest(content="hello")
+
+
+def test_judge_request_rejects_empty_content() -> None:
+    """Regression: empty content scored high because LLMs are agreeable."""
+    with pytest.raises(ValueError):
+        JudgeRequest(rubric_name="anti-slop", content="")
+
+
+def test_rubric_dimension_defaults() -> None:
+    d = RubricDimension(id="x", description="y")
+    assert d.weight == 1.0
+    assert d.min == 0.0
+    assert d.max == 1.0
+
+
+def test_rubric_round_trip_preserves_camelCase_aliases() -> None:
+    """Wire format uses systemPrompt/failureModes; Python uses snake_case.
+    Round-trip via .model_dump(by_alias=True) must preserve the wire shape."""
+    r = Rubric(
+        name="r",
+        description="d",
+        systemPrompt="p",
+        dimensions=[RubricDimension(id="a", description="b")],
+    )
+    payload = r.model_dump(by_alias=True)
+    assert "systemPrompt" in payload
+    assert "failureModes" in payload
+    Rubric.model_validate(payload)  # accepts its own output

agent_eval_rpc-0.21.0/tests/test_subprocess.py

@@ -0,0 +1,87 @@
+"""Integration tests against the real `agent-eval rpc` binary.
+
+These run end-to-end against the bundled CLI in this repo's `dist/`.
+We exercise every method that doesn't need a live LLM:
+  - version
+  - listRubrics
+  - judge with no rubric (validation error path)
+  - judge with bad rubric_name (rubric_not_found path)
+
+Live judge calls (which DO hit an LLM) live in test_live_judge.py and
+are gated by the AGENT_EVAL_LIVE=1 env var.
+"""
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+import pytest
+
+from agent_eval_rpc import Client, RubricNotFoundError, ValidationError
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+CLI_DIST = REPO_ROOT / "dist" / "cli.js"
+
+pytestmark = pytest.mark.skipif(
+    not CLI_DIST.exists(),
+    reason="run `pnpm build` in agent-eval root before these tests",
+)
+
+
+def _client() -> Client:
+    """Subprocess client that invokes the bundled CLI directly via node."""
+
+    class _NodeWrappedClient(Client):
+        def _subprocess_call(self, method: str, params):  # type: ignore[override]
+            import json
+            import subprocess
+
+            proc = subprocess.run(
+                ["node", str(CLI_DIST), "rpc", method],
+                input=json.dumps(params),
+                capture_output=True,
+                text=True,
+                timeout=self.timeout_s,
+                check=False,
+            )
+            if not proc.stdout:
+                raise RuntimeError(f"no stdout. stderr: {proc.stderr}")
+            envelope = json.loads(proc.stdout.strip().splitlines()[-1])
+            if "error" in envelope:
+                from agent_eval_rpc.errors import from_error_body
+                raise from_error_body(proc.returncode or 500, envelope)
+            return envelope["result"]
+
+    c = _NodeWrappedClient(transport="subprocess")
+    # Override the resolved transport to bypass shutil.which check
+    c._transport = "subprocess"  # type: ignore[assignment]
+    return c
+
+
+def test_version_via_subprocess() -> None:
+    c = _client()
+    v = c.version()
+    assert v.package == "@tangle-network/agent-eval"
+    assert v.version
+    assert "judge" in v.api_surface
+
+
+def test_list_rubrics_includes_anti_slop() -> None:
+    c = _client()
+    rubrics = c.list_rubrics()
+    names = [r.name for r in rubrics.rubrics]
+    assert "anti-slop" in names
+
+
+def test_judge_unknown_rubric_name_raises_RubricNotFoundError() -> None:
+    """Regression: server returns 404; client must raise the typed error, not bubble TransportError."""
+    c = _client()
+    with pytest.raises(RubricNotFoundError):
+        c.judge(content="hello world", rubric_name="no-such-rubric-xyz")
+
+
+def test_judge_empty_content_raises_ValidationError() -> None:
+    """Regression: pydantic should catch this before subprocess fires."""
+    c = _client()
+    with pytest.raises((ValidationError, ValueError)):
+        c.judge(content="", rubric_name="anti-slop")