ecp-runtime 0.3.2__tar.gz → 0.4.0__tar.gz

{ecp_runtime-0.3.2 → ecp_runtime-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ecp-runtime
-Version: 0.3.2
+Version: 0.4.0
 Summary: Vendor-neutral runtime for portable AI agent evaluations with outputs, tool calls, and audit context.
 Project-URL: Homepage, https://github.com/evaluation-context-protocol/ecp
 Project-URL: Documentation, https://evaluationcontextprotocol.io/
@@ -25,7 +25,7 @@ ECP is a vendor-neutral protocol for testing agent outputs, tool calls, and eval
 ## Install
 
 ```bash
-pip install "ecp-runtime==0.3.2"
+pip install "ecp-runtime==0.4.0"
 ```
 
 ## Usage

{ecp_runtime-0.3.2 → ecp_runtime-0.4.0}/README.md

@@ -7,7 +7,7 @@ ECP is a vendor-neutral protocol for testing agent outputs, tool calls, and eval
 ## Install
 
 ```bash
-pip install "ecp-runtime==0.3.2"
+pip install "ecp-runtime==0.4.0"
 ```
 
 ## Usage

{ecp_runtime-0.3.2 → ecp_runtime-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "ecp-runtime"
-version = "0.3.2"
+version = "0.4.0"
 description = "Vendor-neutral runtime for portable AI agent evaluations with outputs, tool calls, and audit context."
 authors = [
   { name = "ECP Maintainers", email = "aniket.wattamwar17@gmail.com" },

ecp_runtime-0.4.0/src/ecp_runtime/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.4.0"
+

{ecp_runtime-0.3.2 → ecp_runtime-0.4.0}/src/ecp_runtime/cli.py

@@ -5,14 +5,21 @@ import os
 import shutil
 import sys
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Optional
 
 import typer
 from pydantic import ValidationError
 
+from .conformance import (
+    build_conformance_report,
+    conformance_check,
+    validate_initialize_result,
+    validate_reset_result,
+    validate_step_result,
+)
 from .manifest import ECPManifest
 from .reporter import HTMLReporter
-from .runner import ECPRunner
+from .runner import ECPRunner, resolve_rpc_timeout
 from .trend import RunTrendAnalyzer
 
 app = typer.Typer(
@@ -78,6 +85,11 @@ def run(
         "--fail-on-error/--no-fail-on-error",
         help="Exit non-zero if any checks fail (useful for CI)",
     ),
+    timeout: Optional[float] = typer.Option(
+        None,
+        "--timeout",
+        help="RPC timeout in seconds (overrides ECP_RPC_TIMEOUT)",
+    ),
 ):
     """
     Execute an evaluation run based on a manifest file.
@@ -92,7 +104,7 @@ def run(
         config = ECPManifest.from_yaml(str(manifest))
 
         # Run the Tests
-        runner = ECPRunner(config)
+        runner = ECPRunner(config, rpc_timeout=timeout)
         result_summary = runner.run_scenarios()
         total = int(result_summary.get("total", 0) or 0)
         passed = int(result_summary.get("passed", 0) or 0)
@@ -227,32 +239,96 @@ def conformance(
         "--input",
         help="Input text for the conformance step call",
     ),
+    json_out: Optional[Path] = typer.Option(
+        None,
+        "--json-out",
+        help="Path to save the machine-readable conformance report",
+        resolve_path=True,
+    ),
+    print_json: bool = typer.Option(
+        False,
+        "--json",
+        help="Print only the machine-readable conformance report",
+    ),
+    timeout: Optional[float] = typer.Option(
+        None,
+        "--timeout",
+        help="RPC timeout in seconds (overrides ECP_RPC_TIMEOUT)",
+    ),
 ):
     """
-    Run a small protocol conformance smoke test against an ECP agent.
+    Validate the core ECP protocol contract against an agent.
     """
-    runner = ECPRunner(type("Manifest", (), {"target": target, "scenarios": []})())
-    agent = runner._create_agent(target, rpc_timeout=float(os.environ.get("ECP_RPC_TIMEOUT", "30")))
-    agent.start()
+    rpc_timeout = resolve_rpc_timeout(timeout)
+    runner = ECPRunner(
+        type("Manifest", (), {"target": target, "scenarios": []})(),
+        rpc_timeout=rpc_timeout,
+    )
+    agent = runner._create_agent(target, rpc_timeout=rpc_timeout)
+    checks: List[Dict[str, Any]] = []
+    started = False
     try:
-        init_resp = agent.send_rpc("agent/initialize", {"config": {}})
-        _assert_rpc_result(init_resp, "agent/initialize")
-        step_resp = agent.send_rpc("agent/step", {"input": step_input})
-        _assert_rpc_result(step_resp, "agent/step")
-        result = step_resp.get("result", {})
-        if not isinstance(result, dict):
-            raise ValueError("agent/step result must be an object")
-        if "public_output" not in result and "evaluation_context" not in result and "tool_calls" not in result:
-            raise ValueError("agent/step result should expose public_output, evaluation_context, or tool_calls")
-        reset_resp = agent.send_rpc("agent/reset", {})
-        _assert_rpc_result(reset_resp, "agent/reset")
+        agent.start()
+        started = True
+        initialize = _run_conformance_call(
+            agent,
+            "initialize response",
+            "agent/initialize",
+            {"config": {}},
+            result_validator=validate_initialize_result,
+        )
+        checks.append(initialize)
+        if initialize["passed"]:
+            checks.append(
+                _run_conformance_call(
+                    agent,
+                    "step result contract",
+                    "agent/step",
+                    {"input": step_input},
+                    result_validator=validate_step_result,
+                )
+            )
+        else:
+            checks.append(_skipped_conformance_check("step result contract", "agent/step"))
     except Exception as exc:
-        typer.echo(f"Conformance failed: {exc}", err=True)
-        raise typer.Exit(code=1)
+        checks.append(
+            {
+                "name": "initialize response",
+                "method": "agent/initialize",
+                "passed": False,
+                "message": f"agent could not start: {exc}",
+            }
+        )
+        checks.append(_skipped_conformance_check("step result contract", "agent/step"))
     finally:
-        agent.stop()
-
-    typer.echo("Conformance smoke test passed")
+        if started:
+            checks.append(
+                _run_conformance_call(
+                    agent,
+                    "reset response",
+                    "agent/reset",
+                    {},
+                    result_validator=validate_reset_result,
+                )
+            )
+            agent.stop()
+        else:
+            checks.append(_skipped_conformance_check("reset response", "agent/reset"))
+
+    report = build_conformance_report(target, checks)
+    rendered = json.dumps(report, indent=2)
+    if json_out:
+        json_out.write_text(rendered + "\n", encoding="utf-8")
+    if print_json:
+        typer.echo(rendered)
+    else:
+        for check in checks:
+            marker = "PASS" if check["passed"] else "FAIL"
+            typer.echo(f"{marker} | {check['name']} | {check['message']}")
+        typer.echo(f"Conformance: {report['passed']}/{report['total']} checks passed")
+
+    if not report["conformant"]:
+        raise typer.Exit(code=1)
 
 
 @app.command()
@@ -316,15 +392,28 @@ def trend(
         raise typer.Exit(code=2)
 
 
-def _assert_rpc_result(response: Dict[str, Any], method: str) -> None:
-    if not isinstance(response, dict):
-        raise ValueError(f"{method} response must be a JSON-RPC object")
-    if response.get("jsonrpc") != "2.0":
-        raise ValueError(f"{method} response must include jsonrpc='2.0'")
-    if "error" in response:
-        raise ValueError(f"{method} returned error: {response['error']}")
-    if "result" not in response:
-        raise ValueError(f"{method} response must include result")
+def _run_conformance_call(
+    agent: Any,
+    name: str,
+    method: str,
+    params: Dict[str, Any],
+    *,
+    result_validator: Optional[Callable[[Any], Any]] = None,
+) -> Dict[str, Any]:
+    try:
+        response = agent.send_rpc(method, params)
+    except Exception as exc:
+        return {"name": name, "method": method, "passed": False, "message": str(exc)}
+    return conformance_check(name, method, response, result_validator=result_validator)
+
+
+def _skipped_conformance_check(name: str, method: str) -> Dict[str, Any]:
+    return {
+        "name": name,
+        "method": method,
+        "passed": False,
+        "message": "skipped because a prerequisite failed",
+    }
 
 
 def _starter_agent() -> str:

ecp_runtime-0.4.0/src/ecp_runtime/conformance.py

@@ -0,0 +1,108 @@
+"""Protocol contract validation shared by runtime execution and conformance checks."""
+
+from typing import Any, Callable, Dict, List, Optional
+
+VALID_STATUSES = {"done", "paused"}
+
+
+def validate_rpc_response(response: Any, method: str) -> Any:
+    """Validate the JSON-RPC response envelope and return its result."""
+    if not isinstance(response, dict):
+        raise ValueError(f"{method} response must be a JSON-RPC object")
+    if response.get("jsonrpc") != "2.0":
+        raise ValueError(f"{method} response must include jsonrpc='2.0'")
+    if "id" not in response:
+        raise ValueError(f"{method} response must include id")
+    if "result" in response and "error" in response:
+        raise ValueError(f"{method} response cannot include both result and error")
+    if "error" in response:
+        error = response["error"]
+        if not isinstance(error, dict):
+            raise ValueError(f"{method} error must be an object")
+        if not isinstance(error.get("code"), int) or not isinstance(error.get("message"), str):
+            raise ValueError(f"{method} error must include integer code and string message")
+        raise ValueError(f"{method} returned error: {error}")
+    if "result" not in response:
+        raise ValueError(f"{method} response must include result")
+    return response["result"]
+
+
+def validate_step_result(result: Any) -> Dict[str, Any]:
+    """Validate the normative agent/step result contract."""
+    if not isinstance(result, dict):
+        raise ValueError("agent/step result must be an object")
+
+    status = result.get("status")
+    if status not in VALID_STATUSES:
+        allowed = ", ".join(sorted(VALID_STATUSES))
+        raise ValueError(f"agent/step result status must be one of: {allowed}")
+
+    for field in ("public_output", "evaluation_context", "private_thought", "logs"):
+        value = result.get(field)
+        if value is not None and not isinstance(value, str):
+            raise ValueError(f"agent/step result {field} must be a string or null")
+
+    tool_calls = result.get("tool_calls")
+    if tool_calls is not None:
+        if not isinstance(tool_calls, list):
+            raise ValueError("agent/step result tool_calls must be an array or null")
+        for index, tool_call in enumerate(tool_calls):
+            _validate_tool_call(tool_call, index)
+
+    return result
+
+
+def validate_initialize_result(result: Any) -> Dict[str, Any]:
+    if not isinstance(result, dict):
+        raise ValueError("agent/initialize result must be an object")
+    if not isinstance(result.get("name"), str) or not result["name"]:
+        raise ValueError("agent/initialize result name must be a non-empty string")
+    if not isinstance(result.get("capabilities"), dict):
+        raise ValueError("agent/initialize result capabilities must be an object")
+    return result
+
+
+def validate_reset_result(result: Any) -> bool:
+    if result is not True:
+        raise ValueError("agent/reset result must be true")
+    return result
+
+
+def conformance_check(
+    name: str,
+    method: str,
+    response: Any,
+    *,
+    result_validator: Optional[Callable[[Any], Any]] = None,
+) -> Dict[str, Any]:
+    """Return a stable, serializable result for one protocol check."""
+    try:
+        result = validate_rpc_response(response, method)
+        if result_validator:
+            result_validator(result)
+    except ValueError as exc:
+        return {"name": name, "method": method, "passed": False, "message": str(exc)}
+    return {"name": name, "method": method, "passed": True, "message": "passed"}
+
+
+def build_conformance_report(target: str, checks: List[Dict[str, Any]]) -> Dict[str, Any]:
+    passed = sum(1 for check in checks if check["passed"])
+    total = len(checks)
+    return {
+        "target": target,
+        "conformant": passed == total,
+        "passed": passed,
+        "failed": total - passed,
+        "total": total,
+        "checks": checks,
+    }
+
+
+def _validate_tool_call(tool_call: Any, index: Optional[int] = None) -> None:
+    label = "tool call" if index is None else f"tool_calls[{index}]"
+    if not isinstance(tool_call, dict):
+        raise ValueError(f"agent/step result {label} must be an object")
+    if not isinstance(tool_call.get("name"), str) or not tool_call["name"]:
+        raise ValueError(f"agent/step result {label}.name must be a non-empty string")
+    if "arguments" in tool_call and not isinstance(tool_call["arguments"], dict):
+        raise ValueError(f"agent/step result {label}.arguments must be an object")

{ecp_runtime-0.3.2 → ecp_runtime-0.4.0}/src/ecp_runtime/manifest.py

@@ -1,11 +1,17 @@
 from typing import Any, Dict, List, Literal, Optional
 
 import yaml
-from pydantic import BaseModel, Field, model_validator
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class StrictModel(BaseModel):
+    """Base class that keeps runtime validation aligned with the JSON schema."""
+
+    model_config = ConfigDict(extra="forbid")
 
 
 # --- The Grader (Assertion) Schema ---
-class GraderConfig(BaseModel):
+class GraderConfig(StrictModel):
     type: Literal["text_match", "llm_judge", "tool_usage"]
     field: Literal["public_output", "evaluation_context", "private_thought"] = "public_output"
     # For text_match
@@ -41,20 +47,20 @@ class GraderConfig(BaseModel):
         return self
 
 # --- The Step (Scenario) Schema ---
-class StepConfig(BaseModel):
+class StepConfig(StrictModel):
     input: str
     constraints: Dict[str, Any] = Field(default_factory=dict)
     graders: List[GraderConfig] = Field(default_factory=list)
 
-class ScenarioConfig(BaseModel):
-    name: str
+class ScenarioConfig(StrictModel):
+    name: str = Field(min_length=1)
     steps: List[StepConfig]
 
 # --- The Root Manifest Schema ---
-class ECPManifest(BaseModel):
-    manifest_version: Literal["v1"] = "v1"
-    name: str
-    target: str
+class ECPManifest(StrictModel):
+    manifest_version: Literal["v1"]
+    name: str = Field(min_length=1)
+    target: str = Field(min_length=1)
     scenarios: List[ScenarioConfig]
 
     @classmethod

{ecp_runtime-0.3.2 → ecp_runtime-0.4.0}/src/ecp_runtime/runner.py

@@ -2,11 +2,11 @@
 Docstring for runtime.python.src.ecp_runtime.runner
 
 Simplified Version. V0.1
-AsyncIO Pending
 """
 
 import json
 import logging
+import math
 import os
 import queue
 import subprocess
@@ -17,6 +17,11 @@ from typing import Any, Dict, List, Optional
 from urllib import error, request
 from urllib.parse import urlparse
 
+from .conformance import (
+    validate_initialize_result,
+    validate_rpc_response,
+    validate_step_result,
+)
 from .graders import evaluate_step
 
 logger = logging.getLogger(__name__)
@@ -67,11 +72,12 @@ class AgentProcess:
         if not params:
             params = {}
 
+        request_id = int(time.time() * 1000)
         request = {
             "jsonrpc": "2.0",
             "method": method,
             "params": params,
-            "id": int(time.time() * 1000)
+            "id": request_id
         }
 
         # Write to Agent's STDIN
@@ -79,7 +85,9 @@ class AgentProcess:
         self.process.stdin.write(json_str + "\n")
         self.process.stdin.flush()
 
-        return self._read_json_response()
+        response = self._read_json_response()
+        _ensure_response_id(response, request_id)
+        return response
 
     def _read_json_response(self) -> Dict[str, Any]:
         start_time = time.time()
@@ -168,11 +176,12 @@ class HTTPAgentClient:
         if not params:
             params = {}
 
+        request_id = int(time.time() * 1000)
         payload = {
             "jsonrpc": "2.0",
             "method": method,
             "params": params,
-            "id": int(time.time() * 1000),
+            "id": request_id,
         }
         body = json.dumps(payload).encode("utf-8")
         req = request.Request(
@@ -198,12 +207,15 @@ class HTTPAgentClient:
             raise RuntimeError(f"HTTP RPC failed: {exc.reason}") from exc
 
         if "text/event-stream" in content_type:
-            return self._parse_sse_response(raw)
+            response = self._parse_sse_response(raw)
+            _ensure_response_id(response, request_id)
+            return response
         if not raw:
             raise RuntimeError("HTTP RPC response was empty")
         payload = json.loads(raw)
         if not isinstance(payload, dict):
             raise RuntimeError("HTTP RPC response must be a JSON object")
+        _ensure_response_id(payload, request_id)
         return payload
 
     def _parse_sse_response(self, raw: str) -> Dict[str, Any]:
@@ -223,8 +235,9 @@ class HTTPAgentClient:
 class ECPRunner:
     """The Orchestrator."""
 
-    def __init__(self, manifest):
+    def __init__(self, manifest, rpc_timeout: Optional[float] = None):
         self.manifest = manifest
+        self.rpc_timeout = resolve_rpc_timeout(rpc_timeout)
 
     def run_scenarios(self):
         total_passed = 0
@@ -234,20 +247,35 @@ class ECPRunner:
         for scenario in self.manifest.scenarios:
             logger.info("Scenario: %s", scenario.name)
 
-            rpc_timeout = float(os.environ.get("ECP_RPC_TIMEOUT", "30"))
-            agent = self._create_agent(self.manifest.target, rpc_timeout=rpc_timeout)
+            agent = self._create_agent(self.manifest.target, rpc_timeout=self.rpc_timeout)
             agent.start()
 
             try:
-                init_resp = agent.send_rpc("agent/initialize", {"config": {}})
-                self._ensure_rpc_success(init_resp, scenario.name, step_idx=None, method="agent/initialize")
+                init_resp = self._call_rpc(
+                    agent,
+                    "agent/initialize",
+                    {"config": {}},
+                    scenario.name,
+                    step_idx=None,
+                )
+                try:
+                    validate_initialize_result(init_resp["result"])
+                except ValueError as exc:
+                    raise RuntimeError(
+                        f"Invalid agent/initialize result at scenario='{scenario.name}': {exc}"
+                    ) from exc
                 scenario_steps: List[Dict[str, Any]] = []
 
                 for i, step in enumerate(scenario.steps):
                     # Execute
-                    rpc_resp = agent.send_rpc("agent/step", {"input": step.input})
-                    self._ensure_rpc_success(rpc_resp, scenario.name, step_idx=i + 1, method="agent/step")
-                    result_data = rpc_resp.get("result", {})
+                    rpc_resp = self._call_rpc(
+                        agent,
+                        "agent/step",
+                        {"input": step.input},
+                        scenario.name,
+                        step_idx=i + 1,
+                    )
+                    result_data = validate_step_result(rpc_resp["result"])
 
                     # Map to internal object
                     step_result = StepResult(
@@ -255,6 +283,7 @@ class ECPRunner:
                         public_output=result_data.get("public_output"),
                         evaluation_context=result_data.get("evaluation_context") or result_data.get("private_thought"),
                         private_thought=result_data.get("private_thought") or result_data.get("evaluation_context"),
+                        logs=result_data.get("logs"),
                         tool_calls=result_data.get("tool_calls") if isinstance(result_data.get("tool_calls"), list) else None
                     )
 
@@ -282,6 +311,7 @@ class ECPRunner:
                         "output": step_result.public_output,
                         "evaluation_context": step_result.evaluation_context,
                         "tool_calls": step_result.tool_calls or [],
+                        "logs": step_result.logs,
                         "checks": checks
                     })
 
@@ -312,20 +342,55 @@ class ECPRunner:
         step_idx: Optional[int],
         method: str,
     ) -> None:
-        if "error" not in rpc_resp:
-            return
+        where = f"scenario='{scenario_name}'"
+        if step_idx is not None:
+            where += f", step={step_idx}"
+        try:
+            validate_rpc_response(rpc_resp, method)
+        except ValueError as exc:
+            raise RuntimeError(f"RPC call failed ({method}) at {where}: {exc}") from exc
 
-        error = rpc_resp.get("error") or {}
-        code = error.get("code")
-        message = error.get("message", "Unknown JSON-RPC error")
+    def _call_rpc(
+        self,
+        agent: Any,
+        method: str,
+        params: Dict[str, Any],
+        scenario_name: str,
+        step_idx: Optional[int],
+    ) -> Dict[str, Any]:
         where = f"scenario='{scenario_name}'"
         if step_idx is not None:
             where += f", step={step_idx}"
-        raise RuntimeError(
-            f"RPC call failed ({method}) at {where}: code={code}, message={message}"
-        )
+        try:
+            response = agent.send_rpc(method, params)
+        except Exception as exc:
+            raise RuntimeError(f"RPC call failed ({method}) at {where}: {exc}") from exc
+        self._ensure_rpc_success(response, scenario_name, step_idx, method)
+        return response
+
+
+def resolve_rpc_timeout(value: Optional[float] = None) -> float:
+    """Resolve and validate an explicit timeout or the ECP_RPC_TIMEOUT fallback."""
+    raw_value: Any = value if value is not None else os.environ.get("ECP_RPC_TIMEOUT", "30")
+    try:
+        timeout = float(raw_value)
+    except (TypeError, ValueError) as exc:
+        raise ValueError(
+            f"RPC timeout must be a positive number; received {raw_value!r}"
+        ) from exc
+    if not math.isfinite(timeout) or timeout <= 0:
+        raise ValueError(f"RPC timeout must be a positive finite number; received {raw_value!r}")
+    return timeout
 
 
 def _is_http_url(target: str) -> bool:
     parsed = urlparse(target)
     return parsed.scheme in {"http", "https"} and bool(parsed.netloc)
+
+
+def _ensure_response_id(response: Dict[str, Any], request_id: int) -> None:
+    response_id = response.get("id")
+    if type(response_id) is not type(request_id) or response_id != request_id:
+        raise RuntimeError(
+            f"JSON-RPC response id mismatch: expected {request_id}, got {response_id}"
+        )

{ecp_runtime-0.3.2 → ecp_runtime-0.4.0}/src/ecp_runtime/trend.py

@@ -8,10 +8,9 @@ RunTrendAnalyzer - reads a sequence of saved JSON report files produced by
 from __future__ import annotations
 
 import json
-import statistics
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import List, Literal
+from typing import List, Literal, Optional
 
 
 @dataclass
@@ -81,7 +80,7 @@ class RunTrendAnalyzer:
         )
 
     @staticmethod
-    def _load_run_point(path: Path) -> RunPoint | None:
+    def _load_run_point(path: Path) -> Optional[RunPoint]:
         try:
             data = json.loads(path.read_text(encoding="utf-8"))
         except (OSError, json.JSONDecodeError):
@@ -101,9 +100,17 @@ class RunTrendAnalyzer:
 
     @staticmethod
     def _compute_slope(pass_rates: List[float]) -> float:
-        x = list(range(len(pass_rates)))
-        slope, _intercept = statistics.linear_regression(x, pass_rates)
-        return slope
+        count = len(pass_rates)
+        if count < 2:
+            return 0.0
+        mean_x = (count - 1) / 2
+        mean_y = sum(pass_rates) / count
+        numerator = sum(
+            (index - mean_x) * (rate - mean_y)
+            for index, rate in enumerate(pass_rates)
+        )
+        denominator = sum((index - mean_x) ** 2 for index in range(count))
+        return numerator / denominator if denominator else 0.0
 
     def _classify(self, slope: float) -> Literal["improving", "degrading", "stable"]:
         if slope > self._IMPROVING_THRESHOLD:

{ecp_runtime-0.3.2 → ecp_runtime-0.4.0}/tests/test_cli.py

@@ -9,6 +9,7 @@ RUNTIME_SRC = Path(__file__).resolve().parents[1] / "src"
 if str(RUNTIME_SRC) not in sys.path:
     sys.path.insert(0, str(RUNTIME_SRC))
 
+import ecp_runtime.cli as cli_module
 from ecp_runtime.cli import app
 from typer.testing import CliRunner
 
@@ -72,6 +73,26 @@ class CLISmokeTests(unittest.TestCase):
             result = self.runner.invoke(app, ["run", "--manifest", self.manifest_path, "--no-fail-on-error"])
         self.assertEqual(result.exit_code, 0, msg=result.output)
 
+    def test_run_passes_explicit_timeout_to_runner(self) -> None:
+        fake_config = object()
+        runtime_class = mock.Mock(
+            return_value=mock.Mock(
+                run_scenarios=mock.Mock(return_value={"passed": 0, "total": 0, "scenarios": []})
+            )
+        )
+        with mock.patch("ecp_runtime.cli._configure_logging"), mock.patch.object(
+            cli_module.ECPManifest,
+            "from_yaml",
+            return_value=fake_config,
+        ), mock.patch("ecp_runtime.cli.ECPRunner", runtime_class):
+            result = self.runner.invoke(
+                app,
+                ["run", "--manifest", self.manifest_path, "--timeout", "4.5"],
+            )
+
+        self.assertEqual(result.exit_code, 0, msg=result.output)
+        runtime_class.assert_called_once_with(fake_config, rpc_timeout=4.5)
+
     def test_validate_command(self) -> None:
         result = self.runner.invoke(app, ["validate", self.manifest_path])
 
@@ -95,6 +116,61 @@ class CLISmokeTests(unittest.TestCase):
             self.assertTrue(manifest.exists())
             self.assertIn("evaluation_context", manifest.read_text(encoding="utf-8"))
 
+    def test_conformance_json_report(self) -> None:
+        fake_agent = mock.Mock()
+        fake_agent.send_rpc.side_effect = [
+            {"jsonrpc": "2.0", "id": 1, "result": {"name": "test", "capabilities": {}}},
+            {"jsonrpc": "2.0", "id": 2, "result": {"status": "done", "public_output": "ok"}},
+            {"jsonrpc": "2.0", "id": 3, "result": True},
+        ]
+        fake_runtime = mock.Mock()
+        fake_runtime._create_agent.return_value = fake_agent
+
+        with mock.patch("ecp_runtime.cli.ECPRunner", return_value=fake_runtime):
+            result = self.runner.invoke(app, ["conformance", "--target", "python agent.py", "--json"])
+
+        self.assertEqual(result.exit_code, 0, msg=result.output)
+        payload = json.loads(result.stdout)
+        self.assertTrue(payload["conformant"])
+        self.assertEqual(payload["total"], 3)
+        fake_agent.stop.assert_called_once()
+
+    def test_conformance_passes_explicit_timeout_to_transport(self) -> None:
+        fake_agent = mock.Mock()
+        fake_agent.send_rpc.side_effect = [
+            {"jsonrpc": "2.0", "id": 1, "result": {"name": "test", "capabilities": {}}},
+            {"jsonrpc": "2.0", "id": 2, "result": {"status": "done"}},
+            {"jsonrpc": "2.0", "id": 3, "result": True},
+        ]
+        fake_runtime = mock.Mock()
+        fake_runtime._create_agent.return_value = fake_agent
+        runtime_class = mock.Mock(return_value=fake_runtime)
+
+        with mock.patch("ecp_runtime.cli.ECPRunner", runtime_class):
+            result = self.runner.invoke(
+                app,
+                ["conformance", "--target", "python agent.py", "--timeout", "6", "--json"],
+            )
+
+        self.assertEqual(result.exit_code, 0, msg=result.output)
+        fake_runtime._create_agent.assert_called_once_with("python agent.py", rpc_timeout=6.0)
+
+    def test_conformance_rejects_invalid_step_result(self) -> None:
+        fake_agent = mock.Mock()
+        fake_agent.send_rpc.side_effect = [
+            {"jsonrpc": "2.0", "id": 1, "result": {}},
+            {"jsonrpc": "2.0", "id": 2, "result": {"status": "invalid"}},
+            {"jsonrpc": "2.0", "id": 3, "result": True},
+        ]
+        fake_runtime = mock.Mock()
+        fake_runtime._create_agent.return_value = fake_agent
+
+        with mock.patch("ecp_runtime.cli.ECPRunner", return_value=fake_runtime):
+            result = self.runner.invoke(app, ["conformance", "--target", "python agent.py"])
+
+        self.assertEqual(result.exit_code, 1, msg=result.output)
+        self.assertIn("FAIL | step result contract", result.output)
+
 
 if __name__ == "__main__":
     unittest.main()

ecp_runtime-0.4.0/tests/test_conformance.py

@@ -0,0 +1,91 @@
+import sys
+import unittest
+from pathlib import Path
+
+RUNTIME_SRC = Path(__file__).resolve().parents[1] / "src"
+if str(RUNTIME_SRC) not in sys.path:
+    sys.path.insert(0, str(RUNTIME_SRC))
+
+from ecp_runtime.conformance import (
+    build_conformance_report,
+    conformance_check,
+    validate_initialize_result,
+    validate_reset_result,
+    validate_rpc_response,
+    validate_step_result,
+)
+
+
+class ConformanceTests(unittest.TestCase):
+    def test_valid_step_result(self) -> None:
+        result = {
+            "status": "done",
+            "public_output": "ok",
+            "evaluation_context": "verified",
+            "tool_calls": [{"name": "lookup", "arguments": {"id": 1}}],
+            "logs": "complete",
+        }
+
+        self.assertIs(validate_step_result(result), result)
+
+    def test_initialize_contract(self) -> None:
+        result = {"name": "agent", "capabilities": {}}
+        self.assertIs(validate_initialize_result(result), result)
+
+        with self.assertRaisesRegex(ValueError, "capabilities"):
+            validate_initialize_result({"name": "agent"})
+
+    def test_reset_contract(self) -> None:
+        self.assertTrue(validate_reset_result(True))
+        with self.assertRaisesRegex(ValueError, "must be true"):
+            validate_reset_result(None)
+
+    def test_invalid_status_is_rejected(self) -> None:
+        with self.assertRaisesRegex(ValueError, "status"):
+            validate_step_result({"status": "complete"})
+
+    def test_invalid_tool_call_is_rejected(self) -> None:
+        with self.assertRaisesRegex(ValueError, "name"):
+            validate_step_result({"status": "done", "tool_calls": [{"arguments": {}}]})
+
+    def test_rpc_envelope_requires_id(self) -> None:
+        with self.assertRaisesRegex(ValueError, "include id"):
+            validate_rpc_response({"jsonrpc": "2.0", "result": {}}, "agent/initialize")
+
+    def test_rpc_envelope_rejects_result_and_error(self) -> None:
+        with self.assertRaisesRegex(ValueError, "both result and error"):
+            validate_rpc_response(
+                {
+                    "jsonrpc": "2.0",
+                    "id": 1,
+                    "result": {},
+                    "error": {"code": -32000, "message": "failure"},
+                },
+                "agent/initialize",
+            )
+
+    def test_report_has_stable_counts(self) -> None:
+        checks = [
+            conformance_check(
+                "initialize",
+                "agent/initialize",
+                {"jsonrpc": "2.0", "id": 1, "result": {}},
+            ),
+            conformance_check(
+                "step",
+                "agent/step",
+                {"jsonrpc": "2.0", "id": 2, "result": {"status": "invalid"}},
+                result_validator=validate_step_result,
+            ),
+        ]
+
+        report = build_conformance_report("python agent.py", checks)
+
+        self.assertFalse(report["conformant"])
+        self.assertEqual(report["passed"], 1)
+        self.assertEqual(report["failed"], 1)
+        self.assertEqual(report["total"], 2)
+
+
+if __name__ == "__main__":
+    unittest.main()

{ecp_runtime-0.3.2 → ecp_runtime-0.4.0}/tests/test_example_integrations.py

@@ -50,6 +50,62 @@ class ExampleIntegrationTests(unittest.TestCase):
         self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr)
         self.assertIn('"failed": 0', result.stdout)
 
+    def test_async_python_demo_manifest_passes(self) -> None:
+        manifest = REPO_ROOT / "examples" / "async_python_demo" / "manifest.yaml"
+        result = self._run_manifest(manifest)
+        self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr)
+        self.assertIn('"failed": 0', result.stdout)
+
+    def test_async_python_demo_conforms_over_http(self) -> None:
+        port = self._free_port()
+        env = dict(os.environ)
+        existing = env.get("PYTHONPATH", "")
+        extra_paths = [str(RUNTIME_SRC), str(REPO_ROOT / "sdk" / "python" / "src")]
+        if existing:
+            extra_paths.append(existing)
+        env["PYTHONPATH"] = os.pathsep.join(extra_paths)
+        env["ECP_TRANSPORT"] = "http"
+        env["ECP_HTTP_PORT"] = str(port)
+
+        server = subprocess.Popen(
+            [sys.executable, str(REPO_ROOT / "examples" / "async_python_demo" / "agent.py")],
+            cwd=REPO_ROOT,
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=env,
+        )
+
+        try:
+            self._wait_for_port("127.0.0.1", port)
+            result = subprocess.run(
+                [
+                    sys.executable,
+                    "-m",
+                    "ecp_runtime.cli",
+                    "conformance",
+                    "--target",
+                    f"http://127.0.0.1:{port}/ecp",
+                    "--timeout",
+                    "5",
+                    "--json",
+                ],
+                cwd=REPO_ROOT,
+                text=True,
+                capture_output=True,
+                env=env,
+                check=False,
+            )
+            self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr)
+            self.assertIn('"conformant": true', result.stdout)
+        finally:
+            server.terminate()
+            try:
+                server.communicate(timeout=2)
+            except subprocess.TimeoutExpired:
+                server.kill()
+                server.communicate()
+
     def test_streamable_http_demo_manifest_passes(self) -> None:
         port = self._free_port()
         server_env = dict(os.environ)

{ecp_runtime-0.3.2 → ecp_runtime-0.4.0}/tests/test_manifest.py

@@ -29,6 +29,9 @@ class ManifestValidationTests(unittest.TestCase):
         with self.assertRaises(ValidationError):
             GraderConfig(type="llm_judge")
 
+        with self.assertRaises(ValidationError):
+            GraderConfig(type="llm_judge", prompt="   ")
+
     def test_invalid_grader_type_rejected(self) -> None:
         with self.assertRaises(ValidationError):
             GraderConfig(type="unknown")  # type: ignore[arg-type]
@@ -51,6 +54,25 @@ class ManifestValidationTests(unittest.TestCase):
         with self.assertRaises(ValueError):
             ECPManifest.from_yaml(tmp_path)
 
+    def test_unknown_manifest_fields_are_rejected(self) -> None:
+        with self.assertRaises(ValidationError):
+            ECPManifest(
+                manifest_version="v1",
+                name="test",
+                target="python agent.py",
+                scenarios=[],
+                unexpected=True,
+            )
+
+    def test_unknown_grader_fields_are_rejected(self) -> None:
+        with self.assertRaises(ValidationError):
+            GraderConfig(
+                type="text_match",
+                condition="contains",
+                value="ok",
+                unexpected=True,
+            )
+
 
 if __name__ == "__main__":
     unittest.main()

{ecp_runtime-0.3.2 → ecp_runtime-0.4.0}/tests/test_runner.py

@@ -1,3 +1,4 @@
+import json
 import sys
 import unittest
 from pathlib import Path
@@ -9,7 +10,12 @@ if str(RUNTIME_SRC) not in sys.path:
     sys.path.insert(0, str(RUNTIME_SRC))
 
 from ecp_runtime.manifest import StepConfig
-from ecp_runtime.runner import ECPRunner, HTTPAgentClient
+from ecp_runtime.runner import (
+    ECPRunner,
+    HTTPAgentClient,
+    _ensure_response_id,
+    resolve_rpc_timeout,
+)
 
 
 class RunnerTests(unittest.TestCase):
@@ -73,6 +79,24 @@ class RunnerTests(unittest.TestCase):
         self.assertIn("step=1", msg)
         self.assertIn("boom", msg)
 
+    def test_runner_rejects_invalid_initialize_result(self) -> None:
+        class FakeAgentProcess:
+            def __init__(self, command, rpc_timeout=30.0):
+                pass
+
+            def start(self):
+                return None
+
+            def stop(self):
+                return None
+
+            def send_rpc(self, method, params=None):
+                return {"jsonrpc": "2.0", "id": 1, "result": {"name": "missing-capabilities"}}
+
+        with mock.patch("ecp_runtime.runner.AgentProcess", FakeAgentProcess):
+            with self.assertRaisesRegex(RuntimeError, "Invalid agent/initialize result"):
+                ECPRunner(self._manifest()).run_scenarios()
+
     def test_runner_uses_http_client_for_url_target(self) -> None:
         runner = ECPRunner(SimpleNamespace(target="http://127.0.0.1:8765/ecp", scenarios=[]))
 
@@ -82,6 +106,66 @@ class RunnerTests(unittest.TestCase):
         self.assertEqual(agent.endpoint, "http://127.0.0.1:8765/ecp")
         self.assertEqual(agent.rpc_timeout, 12.0)
 
+    def test_runner_passes_explicit_timeout_to_agent(self) -> None:
+        observed = {}
+
+        class FakeAgentProcess:
+            def __init__(self, command, rpc_timeout=30.0):
+                observed["timeout"] = rpc_timeout
+
+            def start(self):
+                return None
+
+            def stop(self):
+                return None
+
+            def send_rpc(self, method, params=None):
+                if method == "agent/initialize":
+                    return {"jsonrpc": "2.0", "id": 1, "result": {"name": "x", "capabilities": {}}}
+                return {"jsonrpc": "2.0", "id": 2, "result": {"status": "done"}}
+
+        with mock.patch("ecp_runtime.runner.AgentProcess", FakeAgentProcess):
+            ECPRunner(self._manifest(), rpc_timeout=4.25).run_scenarios()
+
+        self.assertEqual(observed["timeout"], 4.25)
+
+    def test_transport_timeout_includes_method_and_step_context(self) -> None:
+        class FakeAgentProcess:
+            def __init__(self, command, rpc_timeout=30.0):
+                pass
+
+            def start(self):
+                return None
+
+            def stop(self):
+                return None
+
+            def send_rpc(self, method, params=None):
+                if method == "agent/initialize":
+                    return {"jsonrpc": "2.0", "id": 1, "result": {"name": "x", "capabilities": {}}}
+                raise RuntimeError("Agent response timed out after 0.1s")
+
+        with mock.patch("ecp_runtime.runner.AgentProcess", FakeAgentProcess):
+            with self.assertRaises(RuntimeError) as ctx:
+                ECPRunner(self._manifest(), rpc_timeout=0.1).run_scenarios()
+
+        message = str(ctx.exception)
+        self.assertIn("agent/step", message)
+        self.assertIn("Scenario A", message)
+        self.assertIn("step=1", message)
+        self.assertIn("timed out", message)
+
+    def test_timeout_resolution_prefers_explicit_value_and_validates_environment(self) -> None:
+        with mock.patch.dict("os.environ", {"ECP_RPC_TIMEOUT": "8.5"}):
+            self.assertEqual(resolve_rpc_timeout(), 8.5)
+            self.assertEqual(resolve_rpc_timeout(2), 2.0)
+
+        for invalid in ("invalid", "0", "-1", "inf", "nan"):
+            with self.subTest(invalid=invalid):
+                with mock.patch.dict("os.environ", {"ECP_RPC_TIMEOUT": invalid}):
+                    with self.assertRaisesRegex(ValueError, "positive"):
+                        resolve_rpc_timeout()
+
     def test_http_agent_client_posts_json_rpc(self) -> None:
         class FakeResponse:
             headers = {"Content-Type": "application/json; charset=utf-8"}
@@ -102,6 +186,10 @@ class RunnerTests(unittest.TestCase):
             captured["timeout"] = timeout
             captured["body"] = req.data
             captured["accept"] = req.headers.get("Accept")
+            request_id = json.loads(req.data.decode("utf-8"))["id"]
+            FakeResponse.read = lambda self: json.dumps(
+                {"jsonrpc": "2.0", "id": request_id, "result": {"ok": True}}
+            ).encode("utf-8")
             return FakeResponse()
 
         with mock.patch("ecp_runtime.runner.request.urlopen", fake_urlopen):
@@ -117,6 +205,13 @@ class RunnerTests(unittest.TestCase):
         self.assertIn('"method": "agent/step"', body)
         self.assertIn('"input": "hi"', body)
 
+    def test_response_id_must_match_request(self) -> None:
+        with self.assertRaisesRegex(RuntimeError, "id mismatch"):
+            _ensure_response_id({"jsonrpc": "2.0", "id": 2, "result": {}}, 1)
+
+        with self.assertRaisesRegex(RuntimeError, "id mismatch"):
+            _ensure_response_id({"jsonrpc": "2.0", "id": True, "result": {}}, 1)
+
 
 if __name__ == "__main__":
     unittest.main()

{ecp_runtime-0.3.2 → ecp_runtime-0.4.0}/tests/test_trend.py

@@ -65,6 +65,9 @@ class RunTrendAnalyzerTests(unittest.TestCase):
         self.assertEqual(report.direction, "stable")
         self.assertEqual(report.pass_rate_slope, 0.0)
 
+    def test_slope_matches_expected_least_squares_result(self) -> None:
+        self.assertAlmostEqual(RunTrendAnalyzer._compute_slope([0.1, 0.5, 0.9]), 0.4)
+
 
 class RunTrendAnalyzerFileTests(unittest.TestCase):
     def test_load_run_point_from_file(self) -> None:

ecp_runtime-0.3.2/src/ecp_runtime/__init__.py

@@ -1,2 +0,0 @@
-__version__ = "0.3.2"
-