iris-eval 0.1.0__py3-none-any.whl

iris_eval/__init__.py

@@ -0,0 +1,61 @@
+"""iris-eval — the Python client for Iris, the open-source agent-evaluation MCP server.
+
+    from iris_eval import IrisClient
+
+    iris = IrisClient()                     # finds the server: IRIS_URL, or the runtime.json a running server wrote
+    logged = iris.log_trace("support-bot", input="Was the refund approved?", output="Yes, it posts within five days.")
+    evaluation = iris.evaluate_output("The refund was approved.", input="Was the refund approved?", agent_name="support-bot")
+    evaluation["verdict"]["state"]          # "pass" | "fail" | "unknown"
+
+A thin client over the HTTP API (docs/sdk-spec.md rules it so): the rules, the
+composer and the storage live in the server; this package speaks to them. The
+pytest plugin (``iris`` fixture, ``assert_iris``) rides on it.
+"""
+
+from .client import (
+    DEFAULT_TIMEOUT,
+    AsyncIrisClient,
+    IrisClient,
+    IrisConnectionError,
+    IrisError,
+)
+from .discovery import ServerLocation, find_server
+from .types import (
+    Capabilities,
+    Evaluation,
+    Health,
+    LoggedTrace,
+    RuleResult,
+    TokenUsage,
+    ToolCall,
+    Trace,
+    TraceDetail,
+    TracePage,
+    Verdict,
+    VerdictState,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "AsyncIrisClient",
+    "Capabilities",
+    "DEFAULT_TIMEOUT",
+    "Evaluation",
+    "Health",
+    "IrisClient",
+    "IrisConnectionError",
+    "IrisError",
+    "LoggedTrace",
+    "RuleResult",
+    "ServerLocation",
+    "TokenUsage",
+    "ToolCall",
+    "Trace",
+    "TraceDetail",
+    "TracePage",
+    "Verdict",
+    "VerdictState",
+    "__version__",
+    "find_server",
+]

iris_eval/client.py

@@ -0,0 +1,506 @@
+"""The client: one method per door, the server's own sentence on every refusal.
+
+``IrisClient`` and ``AsyncIrisClient`` have the same methods; the async one
+awaits. Both are context managers and close their connection pool. A
+non-2xx answer raises ``IrisError`` carrying the server's ``error`` sentence,
+the status and, when the server sent them, the validation ``details``; a
+server that cannot be reached raises ``IrisConnectionError`` naming the URL
+and how to start one.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Mapping
+
+import httpx
+
+from .discovery import find_server
+from .types import Capabilities, Evaluation, Health, LoggedTrace, TraceDetail, TracePage
+
+DEFAULT_TIMEOUT = 10.0
+API = "/api/v1"
+NO_SERVER = (
+    "No Iris server: set IRIS_URL (for example http://127.0.0.1:6920), or start one where this runs — "
+    "`npx -y @iris-eval/mcp-server --dashboard` — and the client finds its port in runtime.json."
+)
+
+
+class IrisError(Exception):
+    """The server refused: its own sentence, the HTTP status, and the validation details when it sent them."""
+
+    def __init__(self, message: str, *, status: int, details: Any = None, method: str = "", path: str = ""):
+        super().__init__(message)
+        self.message = message
+        self.status = status
+        self.details = details
+        self.method = method
+        self.path = path
+
+    def __str__(self) -> str:  # pragma: no cover - trivial
+        where = f" ({self.method} {self.path} → {self.status})" if self.method else f" ({self.status})"
+        return f"{self.message}{where}"
+
+
+class IrisConnectionError(IrisError):
+    """No server answered at the URL."""
+
+    def __init__(self, base_url: str, cause: Exception):
+        super().__init__(f"Could not reach the Iris server at {base_url}: {cause}. {NO_SERVER}", status=0)
+        self.base_url = base_url
+        self.cause = cause
+
+
+def _error_from(res: httpx.Response) -> IrisError:
+    message = f"HTTP {res.status_code}"
+    details = None
+    try:
+        body = res.json()
+        if isinstance(body, dict):
+            if isinstance(body.get("error"), str):
+                message = body["error"]
+            elif isinstance(body.get("message"), str):
+                message = body["message"]
+            details = body.get("details")
+    except ValueError:
+        text = res.text.strip()
+        if text:
+            message = text[:300]
+    return IrisError(message, status=res.status_code, details=details, method=res.request.method, path=res.request.url.path)
+
+
+def _trace_body(
+    agent_name: str,
+    *,
+    input: str | None,
+    output: str | None,
+    framework: str | None,
+    tool_calls: list[Mapping[str, Any]] | None,
+    latency_ms: float | None,
+    token_usage: Mapping[str, Any] | None,
+    cost_usd: float | None,
+    metadata: Mapping[str, Any] | None,
+    tools: list[Mapping[str, Any]] | None,
+    run: str | None,
+    case_key: str | None,
+    session_id: str | None,
+    spans: list[Mapping[str, Any]] | None,
+    timestamp: str | None,
+    evaluate: bool,
+    eval_type: str | None,
+) -> dict[str, Any]:
+    body: dict[str, Any] = {"agent_name": agent_name}
+    for key, value in (
+        ("input", input),
+        ("output", output),
+        ("framework", framework),
+        ("tool_calls", tool_calls),
+        ("latency_ms", latency_ms),
+        ("token_usage", token_usage),
+        ("cost_usd", cost_usd),
+        ("metadata", metadata),
+        ("tools", tools),
+        ("run", run),
+        ("case_key", case_key),
+        ("session_id", session_id),
+        ("spans", spans),
+        ("timestamp", timestamp),
+        ("eval_type", eval_type),
+    ):
+        if value is not None:
+            body[key] = value
+    if evaluate:
+        body["evaluate"] = True
+    return body
+
+
+def _query(
+    *,
+    agent_name: str | None,
+    framework: str | None,
+    session: str | None,
+    since: str | None,
+    until: str | None,
+    min_score: float | None,
+    max_score: float | None,
+    limit: int | None,
+    offset: int | None,
+    sort_by: str | None,
+    sort_order: str | None,
+) -> dict[str, Any]:
+    params: dict[str, Any] = {}
+    for key, value in (
+        ("agent_name", agent_name),
+        ("framework", framework),
+        ("session", session),
+        ("since", since),
+        ("until", until),
+        ("min_score", min_score),
+        ("max_score", max_score),
+        ("limit", limit),
+        ("offset", offset),
+        ("sort_by", sort_by),
+        ("sort_order", sort_order),
+    ):
+        if value is not None:
+            params[key] = value
+    return params
+
+
+class _Base:
+    def __init__(self, base_url: str | None, api_key: str | None, timeout: float, user_agent: str):
+        located = base_url.rstrip("/") if base_url else None
+        if located is None:
+            found = find_server()
+            if found is None:
+                raise IrisConnectionError("(no URL)", RuntimeError("nothing names a server"))
+            located = found.base_url
+        self.base_url = located
+        headers = {"accept": "application/json", "user-agent": user_agent}
+        if api_key:
+            headers["authorization"] = f"Bearer {api_key}"
+        self._headers = headers
+        self._timeout = timeout
+
+
+class IrisClient(_Base):
+    """The synchronous client.
+
+    ``base_url``: ``http://host:port`` of the dashboard; omitted, the server is
+    found from ``IRIS_URL`` or ``runtime.json``. ``api_key``: sent as
+    ``Authorization: Bearer`` — required by a server bound beyond loopback.
+    ``transport``: an ``httpx`` transport, for tests.
+    """
+
+    def __init__(
+        self,
+        base_url: str | None = None,
+        *,
+        api_key: str | None = None,
+        timeout: float = DEFAULT_TIMEOUT,
+        transport: httpx.BaseTransport | None = None,
+    ):
+        from . import __version__
+
+        super().__init__(base_url, api_key, timeout, f"iris-eval-python/{__version__}")
+        self._http = httpx.Client(base_url=self.base_url, headers=self._headers, timeout=timeout, transport=transport)
+
+    def __enter__(self) -> "IrisClient":
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+    def close(self) -> None:
+        self._http.close()
+
+    def _request(self, method: str, path: str, *, json: Any = None, params: Mapping[str, Any] | None = None) -> Any:
+        try:
+            res = self._http.request(method, API + path, json=json, params=params)
+        except httpx.HTTPError as err:
+            raise IrisConnectionError(self.base_url, err) from err
+        if res.status_code >= 400:
+            raise _error_from(res)
+        return res.json()
+
+    def log_trace(
+        self,
+        agent_name: str,
+        *,
+        input: str | None = None,
+        output: str | None = None,
+        framework: str | None = None,
+        tool_calls: list[Mapping[str, Any]] | None = None,
+        latency_ms: float | None = None,
+        token_usage: Mapping[str, Any] | None = None,
+        cost_usd: float | None = None,
+        metadata: Mapping[str, Any] | None = None,
+        tools: list[Mapping[str, Any]] | None = None,
+        run: str | None = None,
+        case_key: str | None = None,
+        session_id: str | None = None,
+        spans: list[Mapping[str, Any]] | None = None,
+        timestamp: str | None = None,
+        evaluate: bool = False,
+        eval_type: str | None = None,
+    ) -> LoggedTrace:
+        """Store a trace (``POST /api/v1/traces``); with ``evaluate=True`` the answer carries its evaluation too."""
+        body = _trace_body(
+            agent_name,
+            input=input,
+            output=output,
+            framework=framework,
+            tool_calls=tool_calls,
+            latency_ms=latency_ms,
+            token_usage=token_usage,
+            cost_usd=cost_usd,
+            metadata=metadata,
+            tools=tools,
+            run=run,
+            case_key=case_key,
+            session_id=session_id,
+            spans=spans,
+            timestamp=timestamp,
+            evaluate=evaluate,
+            eval_type=eval_type,
+        )
+        return self._request("POST", "/traces", json=body)
+
+    def evaluate_output(
+        self,
+        output: str,
+        *,
+        input: str | None = None,
+        agent_name: str = "python",
+        eval_type: str | None = None,
+        tool_calls: list[Mapping[str, Any]] | None = None,
+        tools: list[Mapping[str, Any]] | None = None,
+        cost_usd: float | None = None,
+        token_usage: Mapping[str, Any] | None = None,
+        run: str | None = None,
+        case_key: str | None = None,
+        session_id: str | None = None,
+        metadata: Mapping[str, Any] | None = None,
+    ) -> Evaluation:
+        """Evaluate an output under the server's rules and get the verdict.
+
+        Over HTTP the evaluate door is the ingest door: the output is stored
+        as a trace of ``agent_name`` (so the dashboard shows it, and a run or
+        case key makes it comparable) and evaluated in the same call — the
+        same object the ``evaluate_output`` MCP tool returns. ``eval_type``
+        names one bundle; omitted, every bundle runs.
+        """
+        logged = self.log_trace(
+            agent_name,
+            input=input,
+            output=output,
+            tool_calls=tool_calls,
+            tools=tools,
+            cost_usd=cost_usd,
+            token_usage=token_usage,
+            run=run,
+            case_key=case_key,
+            session_id=session_id,
+            metadata=metadata,
+            evaluate=True,
+            eval_type=eval_type,
+        )
+        evaluation = logged.get("evaluation")
+        if not isinstance(evaluation, dict):
+            raise IrisError("The server stored the trace but answered with no evaluation", status=500, method="POST", path=API + "/traces")
+        return evaluation
+
+    def get_traces(
+        self,
+        *,
+        agent_name: str | None = None,
+        framework: str | None = None,
+        session: str | None = None,
+        since: str | None = None,
+        until: str | None = None,
+        min_score: float | None = None,
+        max_score: float | None = None,
+        limit: int | None = None,
+        offset: int | None = None,
+        sort_by: str | None = None,
+        sort_order: str | None = None,
+        **extra: Any,
+    ) -> TracePage:
+        """A page of traces (``GET /api/v1/traces``) — every filter the route reads.
+
+        Any other keyword is sent as a query parameter as it is, so a filter the
+        server gains later needs no new client; one the server does not read is
+        a 400 naming it (``IrisError``), never silently ignored.
+        """
+        params = _query(
+            agent_name=agent_name,
+            framework=framework,
+            session=session,
+            since=since,
+            until=until,
+            min_score=min_score,
+            max_score=max_score,
+            limit=limit,
+            offset=offset,
+            sort_by=sort_by,
+            sort_order=sort_order,
+        )
+        params.update({k: v for k, v in extra.items() if v is not None})
+        return self._request("GET", "/traces", params=params)
+
+    def get_trace(self, trace_id: str) -> TraceDetail:
+        """One trace with its spans and evaluations (``GET /api/v1/traces/:id``); a 404 raises ``IrisError``."""
+        return self._request("GET", f"/traces/{trace_id}")
+
+    def health(self) -> Health:
+        """``GET /api/v1/health`` — open, unkeyed; ``status`` is ``ok`` or ``degraded`` (a 503 is still an answer)."""
+        try:
+            res = self._http.get(API + "/health")
+        except httpx.HTTPError as err:
+            raise IrisConnectionError(self.base_url, err) from err
+        if res.status_code not in (200, 503):
+            raise _error_from(res)
+        return res.json()
+
+    def capabilities(self) -> Capabilities:
+        """``GET /api/v1/capabilities`` — what this server can do."""
+        return self._request("GET", "/capabilities")
+
+
+class AsyncIrisClient(_Base):
+    """The asynchronous client — the same methods, awaited."""
+
+    def __init__(
+        self,
+        base_url: str | None = None,
+        *,
+        api_key: str | None = None,
+        timeout: float = DEFAULT_TIMEOUT,
+        transport: httpx.AsyncBaseTransport | None = None,
+    ):
+        from . import __version__
+
+        super().__init__(base_url, api_key, timeout, f"iris-eval-python/{__version__}")
+        self._http = httpx.AsyncClient(base_url=self.base_url, headers=self._headers, timeout=timeout, transport=transport)
+
+    async def __aenter__(self) -> "AsyncIrisClient":
+        return self
+
+    async def __aexit__(self, *exc: object) -> None:
+        await self.aclose()
+
+    async def aclose(self) -> None:
+        await self._http.aclose()
+
+    async def _request(self, method: str, path: str, *, json: Any = None, params: Mapping[str, Any] | None = None) -> Any:
+        try:
+            res = await self._http.request(method, API + path, json=json, params=params)
+        except httpx.HTTPError as err:
+            raise IrisConnectionError(self.base_url, err) from err
+        if res.status_code >= 400:
+            raise _error_from(res)
+        return res.json()
+
+    async def log_trace(
+        self,
+        agent_name: str,
+        *,
+        input: str | None = None,
+        output: str | None = None,
+        framework: str | None = None,
+        tool_calls: list[Mapping[str, Any]] | None = None,
+        latency_ms: float | None = None,
+        token_usage: Mapping[str, Any] | None = None,
+        cost_usd: float | None = None,
+        metadata: Mapping[str, Any] | None = None,
+        tools: list[Mapping[str, Any]] | None = None,
+        run: str | None = None,
+        case_key: str | None = None,
+        session_id: str | None = None,
+        spans: list[Mapping[str, Any]] | None = None,
+        timestamp: str | None = None,
+        evaluate: bool = False,
+        eval_type: str | None = None,
+    ) -> LoggedTrace:
+        body = _trace_body(
+            agent_name,
+            input=input,
+            output=output,
+            framework=framework,
+            tool_calls=tool_calls,
+            latency_ms=latency_ms,
+            token_usage=token_usage,
+            cost_usd=cost_usd,
+            metadata=metadata,
+            tools=tools,
+            run=run,
+            case_key=case_key,
+            session_id=session_id,
+            spans=spans,
+            timestamp=timestamp,
+            evaluate=evaluate,
+            eval_type=eval_type,
+        )
+        return await self._request("POST", "/traces", json=body)
+
+    async def evaluate_output(
+        self,
+        output: str,
+        *,
+        input: str | None = None,
+        agent_name: str = "python",
+        eval_type: str | None = None,
+        tool_calls: list[Mapping[str, Any]] | None = None,
+        tools: list[Mapping[str, Any]] | None = None,
+        cost_usd: float | None = None,
+        token_usage: Mapping[str, Any] | None = None,
+        run: str | None = None,
+        case_key: str | None = None,
+        session_id: str | None = None,
+        metadata: Mapping[str, Any] | None = None,
+    ) -> Evaluation:
+        logged = await self.log_trace(
+            agent_name,
+            input=input,
+            output=output,
+            tool_calls=tool_calls,
+            tools=tools,
+            cost_usd=cost_usd,
+            token_usage=token_usage,
+            run=run,
+            case_key=case_key,
+            session_id=session_id,
+            metadata=metadata,
+            evaluate=True,
+            eval_type=eval_type,
+        )
+        evaluation = logged.get("evaluation")
+        if not isinstance(evaluation, dict):
+            raise IrisError("The server stored the trace but answered with no evaluation", status=500, method="POST", path=API + "/traces")
+        return evaluation
+
+    async def get_traces(
+        self,
+        *,
+        agent_name: str | None = None,
+        framework: str | None = None,
+        session: str | None = None,
+        since: str | None = None,
+        until: str | None = None,
+        min_score: float | None = None,
+        max_score: float | None = None,
+        limit: int | None = None,
+        offset: int | None = None,
+        sort_by: str | None = None,
+        sort_order: str | None = None,
+        **extra: Any,
+    ) -> TracePage:
+        params = _query(
+            agent_name=agent_name,
+            framework=framework,
+            session=session,
+            since=since,
+            until=until,
+            min_score=min_score,
+            max_score=max_score,
+            limit=limit,
+            offset=offset,
+            sort_by=sort_by,
+            sort_order=sort_order,
+        )
+        params.update({k: v for k, v in extra.items() if v is not None})
+        return await self._request("GET", "/traces", params=params)
+
+    async def get_trace(self, trace_id: str) -> TraceDetail:
+        return await self._request("GET", f"/traces/{trace_id}")
+
+    async def health(self) -> Health:
+        try:
+            res = await self._http.get(API + "/health")
+        except httpx.HTTPError as err:
+            raise IrisConnectionError(self.base_url, err) from err
+        if res.status_code not in (200, 503):
+            raise _error_from(res)
+        return res.json()
+
+    async def capabilities(self) -> Capabilities:
+        return await self._request("GET", "/capabilities")

iris_eval/discovery.py

@@ -0,0 +1,78 @@
+"""Where the server is.
+
+In order: ``IRIS_URL`` (the whole base, ``http://host:port``); the
+``runtime.json`` a running dashboard writes under ``IRIS_HOME`` (or
+``~/.iris``) with the port it actually bound; nothing. The file may be
+stale after an unclean exit, so a location read from it is verified with
+``GET /api/v1/health`` before it is trusted — the contract the server's own
+comment states.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+import httpx
+
+DEFAULT_HEALTH_TIMEOUT = 2.0
+
+
+@dataclass(frozen=True)
+class ServerLocation:
+    """A base URL and where it came from: ``env``, ``runtime.json`` or ``default``."""
+
+    base_url: str
+    source: str
+
+
+def iris_home() -> Path:
+    """``IRIS_HOME``, or ``~/.iris`` — the same rule the server applies."""
+    home = os.environ.get("IRIS_HOME")
+    return Path(home).expanduser() if home else Path.home() / ".iris"
+
+
+def runtime_file() -> Path:
+    return iris_home() / "runtime.json"
+
+
+def read_runtime_port(path: Path | None = None) -> int | None:
+    """The dashboard port a running server recorded, or None."""
+    p = path or runtime_file()
+    try:
+        data = json.loads(p.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+    port = data.get("dashboardPort") if isinstance(data, dict) else None
+    return port if isinstance(port, int) and 0 < port < 65536 else None
+
+
+def is_healthy(base_url: str, *, timeout: float = DEFAULT_HEALTH_TIMEOUT, transport: httpx.BaseTransport | None = None) -> bool:
+    """``GET /api/v1/health`` answers — 200 (ok) or 503 (degraded) both mean a server is there; a connection error means none."""
+    try:
+        with httpx.Client(base_url=base_url, timeout=timeout, transport=transport) as client:
+            res = client.get("/api/v1/health")
+        return res.status_code in (200, 503)
+    except httpx.HTTPError:
+        return False
+
+
+def find_server(*, verify: bool = True, transport: httpx.BaseTransport | None = None) -> ServerLocation | None:
+    """The server's base URL, or None when nothing names one.
+
+    ``IRIS_URL`` wins as written (a URL the operator set is not second-guessed).
+    A port read from ``runtime.json`` is only returned when the health route
+    answers on it, unless ``verify`` is False.
+    """
+    env = os.environ.get("IRIS_URL")
+    if env:
+        return ServerLocation(env.rstrip("/"), "env")
+    port = read_runtime_port()
+    if port is None:
+        return None
+    base = f"http://127.0.0.1:{port}"
+    if verify and not is_healthy(base, transport=transport):
+        return None
+    return ServerLocation(base, "runtime.json")

iris_eval/pytest_plugin.py

@@ -0,0 +1,106 @@
+"""The pytest plugin: an ``iris`` fixture and ``assert_iris``.
+
+Registered by the ``pytest11`` entry point when ``iris-eval`` is installed,
+so a test file needs no ``conftest``::
+
+    from iris_eval.pytest_plugin import assert_iris
+
+    def test_refund_answer(iris):
+        evaluation = assert_iris("The refund was approved and posts within five days.",
+                                 input="Was the refund approved?", agent_name="support-bot", client=iris)
+        assert evaluation["verdict"]["basis"] == "clean"
+
+The fixture finds the server the way the client does (``IRIS_URL``, then
+``runtime.json``). With none, the test is **skipped** with a sentence that
+says how to start one — unless ``IRIS_REQUIRE=1``, when it fails instead
+(the setting for a CI job that must not pass green because no server ran).
+``assert_iris`` evaluates the output and asserts on ``verdict.state``, the
+composed verdict — never on the score alone.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping
+
+import pytest
+
+from .client import IrisClient, IrisConnectionError, IrisError
+from .discovery import find_server
+from .types import Evaluation, VerdictState
+
+SKIP_REASON = (
+    "no Iris server: set IRIS_URL, or start one — `npx -y @iris-eval/mcp-server --dashboard` — "
+    "and the client reads its port from runtime.json; set IRIS_REQUIRE=1 to fail instead of skipping"
+)
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    group = parser.getgroup("iris", "Iris agent evaluation")
+    group.addoption("--iris-url", action="store", default=None, help="The Iris server's base URL (overrides IRIS_URL and runtime.json).")
+    group.addoption("--iris-api-key", action="store", default=None, help="The API key (overrides IRIS_API_KEY).")
+
+
+@pytest.fixture(scope="session")
+def iris(request: pytest.FixtureRequest) -> IrisClient:
+    """A client for the Iris server this session can reach; skipped (or failed with IRIS_REQUIRE=1) when there is none."""
+    url = request.config.getoption("--iris-url") or None
+    key = request.config.getoption("--iris-api-key") or os.environ.get("IRIS_API_KEY") or None
+    required = os.environ.get("IRIS_REQUIRE", "").strip() in ("1", "true", "yes")
+    if url is None and find_server() is None:
+        if required:
+            pytest.fail(SKIP_REASON, pytrace=False)
+        pytest.skip(SKIP_REASON)
+    try:
+        client = IrisClient(url, api_key=key)
+        client.health()
+    except (IrisConnectionError, IrisError) as err:
+        if required:
+            pytest.fail(f"the Iris server did not answer: {err}", pytrace=False)
+        pytest.skip(f"the Iris server did not answer: {err}")
+    request.addfinalizer(client.close)
+    return client
+
+
+def assert_iris(
+    output: str,
+    *,
+    input: str | None = None,
+    agent_name: str = "pytest",
+    eval_type: str | None = None,
+    expect: VerdictState = "pass",
+    client: IrisClient | None = None,
+    **trace: Any,
+) -> Evaluation:
+    """Evaluate ``output`` and assert the verdict's state is ``expect`` (``pass`` by default).
+
+    The message names the basis and the rules that decided, so a failing
+    test reads like the dashboard: ``verdict fail on detector_veto by no_pii``.
+    Extra keyword arguments (``tool_calls``, ``tools``, ``cost_usd``,
+    ``token_usage``, ``run``, ``case_key``, ``session_id``, ``metadata``) go on
+    the trace. Returns the evaluation for further assertions.
+    """
+    own = client is None
+    c = client or IrisClient()
+    try:
+        evaluation = c.evaluate_output(output, input=input, agent_name=agent_name, eval_type=eval_type, **_trace_fields(trace))
+    finally:
+        if own:
+            c.close()
+    verdict = evaluation.get("verdict") or {}
+    state = verdict.get("state")
+    if state != expect:
+        by = ", ".join(verdict.get("by") or []) or "—"
+        raise AssertionError(
+            f"Iris verdict {state} on {verdict.get('basis')} by {by} (expected {expect}); "
+            f"score {evaluation.get('score')}; evaluation {evaluation.get('id')} on trace {evaluation.get('trace_id')}"
+        )
+    return evaluation
+
+
+def _trace_fields(fields: Mapping[str, Any]) -> dict[str, Any]:
+    allowed = {"tool_calls", "tools", "cost_usd", "token_usage", "run", "case_key", "session_id", "metadata"}
+    unknown = sorted(set(fields) - allowed)
+    if unknown:
+        raise TypeError(f"assert_iris got unexpected keyword(s): {', '.join(unknown)} — the trace fields are {', '.join(sorted(allowed))}")
+    return dict(fields)

iris_eval/types.py

@@ -0,0 +1,120 @@
+"""The shapes the server answers with, as typed dictionaries.
+
+They mirror the HTTP API (docs/api-reference.md) key for key, so a reader
+of the API reference reads these, and a key the server adds tomorrow is
+carried through untouched (``total=False`` everywhere: the server owns the
+contract, the client only names what it knows).
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal, TypedDict
+
+VerdictState = Literal["pass", "fail", "unknown"]
+
+
+class Verdict(TypedDict, total=False):
+    """Which layer decided, and what it decided on."""
+
+    state: VerdictState
+    passed: bool
+    basis: str
+    by: list[str]
+    risk: dict[str, Any] | None
+    confidence: str
+
+
+class RuleResult(TypedDict, total=False):
+    ruleName: str
+    passed: bool
+    score: float
+    message: str
+    skipped: bool
+    skipReason: str
+    evidence: list[dict[str, Any]]
+    value: dict[str, Any]
+
+
+class Evaluation(TypedDict, total=False):
+    """One evaluation, as ``evaluate_output`` and ``POST /api/v1/traces`` answer it."""
+
+    id: str
+    trace_id: str
+    eval_type: str
+    score: float
+    passed: bool
+    verdict: Verdict
+    rule_results: list[RuleResult]
+    critical_failures: list[str]
+    critical_skipped: list[str]
+    interpretations: list[dict[str, Any]]
+    coverage: dict[str, Any]
+    provenance: dict[str, Any]
+    note: str
+
+
+class TokenUsage(TypedDict, total=False):
+    input_tokens: int
+    output_tokens: int
+    total_tokens: int
+
+
+class ToolCall(TypedDict, total=False):
+    name: str
+    arguments: dict[str, Any]
+    result: Any
+    duration_ms: float
+    error: str
+
+
+class Trace(TypedDict, total=False):
+    trace_id: str
+    agent_name: str
+    framework: str
+    input: str
+    output: str
+    tool_calls: list[ToolCall]
+    latency_ms: float
+    token_usage: TokenUsage
+    cost_usd: float
+    metadata: dict[str, Any]
+    timestamp: str
+    tools: list[dict[str, Any]]
+    run_id: str
+    case_key: str
+    session_id: str
+    source: str
+
+
+class LoggedTrace(TypedDict, total=False):
+    """What ``POST /api/v1/traces`` answers: the id the server minted, and the evaluation when one was asked for."""
+
+    trace_id: str
+    status: str
+    evaluation: Evaluation
+
+
+class TracePage(TypedDict, total=False):
+    traces: list[Trace]
+    total: int
+    limit: int
+    offset: int
+
+
+class TraceDetail(TypedDict, total=False):
+    trace: Trace
+    spans: list[dict[str, Any]]
+    evals: list[Evaluation]
+
+
+class Health(TypedDict, total=False):
+    status: str
+    version: str
+    uptime_seconds: float
+    driver: str
+    checks: dict[str, Any]
+    judge: dict[str, Any]
+
+
+Capabilities = dict[str, Any]
+"""``GET /api/v1/capabilities``: the version, the rules with their proof, the judge state, the limits — see the API reference."""

iris_eval-0.1.0.dist-info/METADATA

@@ -0,0 +1,119 @@
+Metadata-Version: 2.5
+Name: iris-eval
+Version: 0.1.0
+Summary: The Python client for Iris, the open-source agent-evaluation MCP server: log a trace, get its verdict, gate a test on it.
+Project-URL: Homepage, https://iris-eval.com
+Project-URL: Documentation, https://github.com/iris-eval/mcp-server/blob/main/packages/python/README.md
+Project-URL: Source, https://github.com/iris-eval/mcp-server/tree/main/packages/python
+Project-URL: Changelog, https://github.com/iris-eval/mcp-server/blob/main/CHANGELOG.md
+Author: Iris
+License-Expression: MIT
+Keywords: agents,evaluation,iris,llm,mcp,pytest
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: httpx<1,>=0.27
+Provides-Extra: test
+Requires-Dist: pytest-asyncio>=0.24; extra == 'test'
+Requires-Dist: pytest>=8; extra == 'test'
+Description-Content-Type: text/markdown
+
+# iris-eval — the Python client
+
+[Iris](https://iris-eval.dev) is an open-source agent-evaluation MCP server: it stores your agent's traces and judges each one under 25 built-in rules — PII, injection, hallucination markers, tool loops, cost outliers, a regression watcher — with a verdict that says which layer decided and why. This package is the Python door to a running server: log a trace, get its verdict, gate a test on it.
+
+```bash
+pip install iris-eval
+npx -y @iris-eval/mcp-server --dashboard      # the server, in another terminal (Node 22.13+)
+```
+
+```python
+from iris_eval import IrisClient
+
+iris = IrisClient()   # finds the server: IRIS_URL, or the runtime.json a running dashboard wrote
+
+evaluation = iris.evaluate_output(
+    "The refund was approved and posts within five business days.",
+    input="Was the refund approved?",
+    agent_name="support-bot",
+)
+evaluation["verdict"]            # {"state": "pass", "basis": "clean", "by": []}
+
+logged = iris.log_trace("support-bot", input="…", output="…", run="nightly-42", case_key="refund-policy")
+page = iris.get_traces(agent_name="support-bot", limit=20)
+iris.health()["status"]          # "ok" | "degraded"
+iris.capabilities()["rules"]     # every rule, what it needs, its published accuracy
+```
+
+A thin client over the HTTP API: the rules, the composer and the storage live in the server, and this package speaks to them — the same verdict the MCP tools, the dashboard and the CI gate read. `AsyncIrisClient` has the same methods, awaited.
+
+## The methods
+
+| Method | Route | What it answers |
+|---|---|---|
+| `log_trace(agent_name, *, input, output, tool_calls, latency_ms, token_usage, cost_usd, metadata, tools, run, case_key, session_id, spans, timestamp, evaluate, eval_type)` | `POST /api/v1/traces` | `{"trace_id", "status"}` — with `evaluate=True`, `"evaluation"` too |
+| `evaluate_output(output, *, input, agent_name, eval_type, …)` | `POST /api/v1/traces` with `evaluate: true` | The evaluation: `verdict` (`state`, `basis`, `by`), `score`, `rule_results`, `critical_failures`, `coverage`, `provenance`. Over HTTP the evaluate door is the ingest door, so the output is stored as a trace of `agent_name` and shows on the dashboard |
+| `get_traces(*, agent_name, framework, session, since, until, min_score, max_score, limit, offset, sort_by, sort_order, **extra)` | `GET /api/v1/traces` | `{"traces", "total", "limit", "offset"}` — any other keyword is sent as a query parameter as it is; one the server does not read is a 400 naming it |
+| `get_trace(trace_id)` | `GET /api/v1/traces/:id` | `{"trace", "spans", "evals"}` |
+| `health()` | `GET /api/v1/health` | Open, unkeyed; `status`, `version`, `checks` |
+| `capabilities()` | `GET /api/v1/capabilities` | What this server can do |
+
+Every answer is the route's JSON as a typed dictionary (`iris_eval.types`): the keys the [API reference](https://github.com/iris-eval/mcp-server/blob/main/docs/api-reference.md) documents, and any key the server adds later carried through.
+
+**Errors.** A refusal raises `IrisError` with the server's own sentence, the status and the validation details: `IrisError: Invalid query parameters (GET /api/v1/traces → 400)`. A server that cannot be reached raises `IrisConnectionError` naming the URL and how to start one.
+
+**Auth.** `IrisClient(api_key="…")` sends `Authorization: Bearer` — a server bound beyond loopback requires it; `IRIS_API_KEY` is read by the pytest fixture.
+
+**Finding the server.** `IrisClient(base_url=None)`: `IRIS_URL` first (`http://host:port`), then the port the running dashboard recorded in `runtime.json` under `IRIS_HOME` (or `~/.iris`), verified with the health route before it is trusted. Nothing named: `IrisConnectionError` with the recipe.
+
+## pytest
+
+Installing the package registers a plugin. An `iris` fixture finds the server; `assert_iris` evaluates an output and asserts on the **verdict's state** — the composed verdict, never the score alone.
+
+```python
+from iris_eval.pytest_plugin import assert_iris
+
+def test_refund_answer(iris):
+    evaluation = assert_iris(
+        agent("Was the refund approved?"),
+        input="Was the refund approved?",
+        agent_name="support-bot",
+        client=iris,
+    )
+    assert evaluation["verdict"]["basis"] == "clean"
+
+def test_the_leak_is_caught(iris):
+    assert_iris("The SSN is 123-45-6789.", input="q", expect="fail", client=iris)
+```
+
+A failing assertion reads like the dashboard: `Iris verdict fail on detector_veto by no_pii (expected pass); score 0.4; evaluation eval_… on trace trace_…`. Without a server the tests are **skipped** with the sentence that says how to start one; set `IRIS_REQUIRE=1` in a CI job that must not pass green because no server ran. `--iris-url` and `--iris-api-key` override the environment.
+
+In CI, start the server in the job and point the tests at it:
+
+```yaml
+- run: npx -y @iris-eval/mcp-server --dashboard --dashboard-port 6920 --api-key "$IRIS_API_KEY" &
+- run: pip install iris-eval && pytest
+  env:
+    IRIS_URL: http://127.0.0.1:6920
+    IRIS_API_KEY: ${{ secrets.IRIS_API_KEY }}
+    IRIS_REQUIRE: "1"
+```
+
+Or gate a traces file without a server at all — the [CI gate](https://github.com/iris-eval/mcp-server/blob/main/docs/ci-gate.md) and its GitHub Action.
+
+## Tracing from Python
+
+This package does not instrument your code. Iris reads the OpenTelemetry traces your framework already emits — Pydantic AI, Google ADK, LangGraph, CrewAI, Semantic Kernel and the others — through `POST /v1/traces` on the dashboard port ([docs/otel-integration.md](https://github.com/iris-eval/mcp-server/blob/main/docs/otel-integration.md)); a decorator SDK that built a second trace model is a maintenance surface Iris chose not to carry.
+
+## Versions
+
+The client follows the HTTP API, which the server versions; `iris_eval.__version__` is this package's own. Python 3.10+, `httpx` the only dependency. Source: [packages/python](https://github.com/iris-eval/mcp-server/tree/main/packages/python).

iris_eval-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+iris_eval/__init__.py,sha256=hXEiAMDuh79nJtkK__q1vM_MBF5MUFDWjiwkI4CRRIU,1583
+iris_eval/client.py,sha256=0znRIashuu6vH0QTdr3y15qU5Eq7865qfvbSYBQc4P0,17426
+iris_eval/discovery.py,sha256=yhOp__OQjVpvFpcUud8txoclQV3McMYf3T3jwHWQKAk,2683
+iris_eval/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+iris_eval/pytest_plugin.py,sha256=6J-kVp_EvD323W14zTYygZRNaH3HwG8pNPfr2tvioQU,4518
+iris_eval/types.py,sha256=HoEcSAaSot5m01znx555t_pNb35Ry-NMnE5ESLh0Q7w,2775
+iris_eval-0.1.0.dist-info/METADATA,sha256=6atORPs3tzNCthRI1EAJ_z5QEXWTvgjpDgQvixaUMoQ,7321
+iris_eval-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+iris_eval-0.1.0.dist-info/entry_points.txt,sha256=Uamo2ZSotN1Ng2iw1nyxOJZBFptv7k-RTPXticHksLE,47
+iris_eval-0.1.0.dist-info/RECORD,,

iris_eval-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

iris_eval-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[pytest11]
+iris_eval = iris_eval.pytest_plugin