ecp-runtime 0.3.3__tar.gz → 0.5.0__tar.gz

{ecp_runtime-0.3.3 → ecp_runtime-0.5.0}/.gitignore

@@ -208,4 +208,8 @@ cython_debug/
 marimo/_static/
 marimo/_lsp/
 __marimo__/
+*.zip
+/ECPBrain
+/worktrees
+/logs
 /worktrees

{ecp_runtime-0.3.3 → ecp_runtime-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ecp-runtime
-Version: 0.3.3
+Version: 0.5.0
 Summary: Vendor-neutral runtime for portable AI agent evaluations with outputs, tool calls, and audit context.
 Project-URL: Homepage, https://github.com/evaluation-context-protocol/ecp
 Project-URL: Documentation, https://evaluationcontextprotocol.io/
@@ -25,7 +25,7 @@ ECP is a vendor-neutral protocol for testing agent outputs, tool calls, and eval
 ## Install
 
 ```bash
-pip install "ecp-runtime==0.3.3"
+pip install "ecp-runtime==0.5.0"
 ```
 
 ## Usage

{ecp_runtime-0.3.3 → ecp_runtime-0.5.0}/README.md

@@ -7,7 +7,7 @@ ECP is a vendor-neutral protocol for testing agent outputs, tool calls, and eval
 ## Install
 
 ```bash
-pip install "ecp-runtime==0.3.3"
+pip install "ecp-runtime==0.5.0"
 ```
 
 ## Usage

{ecp_runtime-0.3.3 → ecp_runtime-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "ecp-runtime"
-version = "0.3.3"
+version = "0.5.0"
 description = "Vendor-neutral runtime for portable AI agent evaluations with outputs, tool calls, and audit context."
 authors = [
   { name = "ECP Maintainers", email = "aniket.wattamwar17@gmail.com" },
@@ -29,4 +29,7 @@ Issues = "https://github.com/evaluation-context-protocol/ecp/issues"
 [project.scripts]
 ecp = "ecp_runtime.cli:app" 
 
+[project.entry-points.pytest11]
+ecp_runtime = "ecp_runtime.pytest_plugin"
+
 

ecp_runtime-0.5.0/src/ecp_runtime/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.5.0"
+

{ecp_runtime-0.3.3 → ecp_runtime-0.5.0}/src/ecp_runtime/cli.py

@@ -1,18 +1,26 @@
-import glob
+import glob
 import json
 import logging
 import os
 import shutil
 import sys
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Optional
 
 import typer
 from pydantic import ValidationError
 
+# Import local modules (Using relative imports)
+from .conformance import (
+    build_conformance_report,
+    conformance_check,
+    validate_initialize_result,
+    validate_reset_result,
+    validate_step_result,
+)
 from .manifest import ECPManifest
 from .reporter import HTMLReporter
-from .runner import ECPRunner
+from .runner import ECPRunner, resolve_rpc_timeout
 from .trend import RunTrendAnalyzer
 
 app = typer.Typer(
@@ -73,11 +81,21 @@ def run(
         "--json",
         help="Print the JSON report to stdout",
     ),
+    export: Optional[str] = typer.Option(
+        None,
+        "--export",
+        help="Export evaluation results to an external platform (e.g., langsmith)",
+    ),
     fail_on_error: bool = typer.Option(
         True,
         "--fail-on-error/--no-fail-on-error",
         help="Exit non-zero if any checks fail (useful for CI)",
     ),
+    timeout: Optional[float] = typer.Option(
+        None,
+        "--timeout",
+        help="RPC timeout in seconds (overrides ECP_RPC_TIMEOUT)",
+    ),
 ):
     """
     Execute an evaluation run based on a manifest file.
@@ -92,7 +110,7 @@ def run(
         config = ECPManifest.from_yaml(str(manifest))
 
         # Run the Tests
-        runner = ECPRunner(config)
+        runner = ECPRunner(config, rpc_timeout=timeout)
         result_summary = runner.run_scenarios()
         total = int(result_summary.get("total", 0) or 0)
         passed = int(result_summary.get("passed", 0) or 0)
@@ -122,6 +140,30 @@ def run(
         if print_json:
             typer.echo(json.dumps(report_payload, indent=2))
 
+        if export:
+            if export.lower() == "langsmith":
+                logger.info("Exporting results to LangSmith...")
+                try:
+                    from langsmith import Client  # type: ignore
+                    client = Client()
+                    project_name = config.name
+                    for scenario in result_summary.get("scenarios", []):
+                        for step in scenario.get("steps", []):
+                            client.create_run(
+                                name=scenario.get("name"),
+                                run_type="chain",
+                                inputs={"input": step.get("input")},
+                                outputs={"output": step.get("output")},
+                                project_name=project_name
+                            )
+                    logger.info("Successfully exported to LangSmith.")
+                except ImportError:
+                    logger.warning("langsmith package not installed. Skipping export.")
+                except Exception as e:
+                    logger.error("Failed to export to LangSmith: %s", e)
+            else:
+                logger.warning("Unsupported export target: %s", export)
+
         if fail_on_error and failed > 0:
             raise typer.Exit(code=2)
 
@@ -227,32 +269,96 @@ def conformance(
         "--input",
         help="Input text for the conformance step call",
     ),
+    json_out: Optional[Path] = typer.Option(
+        None,
+        "--json-out",
+        help="Path to save the machine-readable conformance report",
+        resolve_path=True,
+    ),
+    print_json: bool = typer.Option(
+        False,
+        "--json",
+        help="Print only the machine-readable conformance report",
+    ),
+    timeout: Optional[float] = typer.Option(
+        None,
+        "--timeout",
+        help="RPC timeout in seconds (overrides ECP_RPC_TIMEOUT)",
+    ),
 ):
     """
-    Run a small protocol conformance smoke test against an ECP agent.
+    Validate the core ECP protocol contract against an agent.
     """
-    runner = ECPRunner(type("Manifest", (), {"target": target, "scenarios": []})())
-    agent = runner._create_agent(target, rpc_timeout=float(os.environ.get("ECP_RPC_TIMEOUT", "30")))
-    agent.start()
+    rpc_timeout = resolve_rpc_timeout(timeout)
+    runner = ECPRunner(
+        type("Manifest", (), {"target": target, "scenarios": []})(),
+        rpc_timeout=rpc_timeout,
+    )
+    agent = runner._create_agent(target, rpc_timeout=rpc_timeout)
+    checks: List[Dict[str, Any]] = []
+    started = False
     try:
-        init_resp = agent.send_rpc("agent/initialize", {"config": {}})
-        _assert_rpc_result(init_resp, "agent/initialize")
-        step_resp = agent.send_rpc("agent/step", {"input": step_input})
-        _assert_rpc_result(step_resp, "agent/step")
-        result = step_resp.get("result", {})
-        if not isinstance(result, dict):
-            raise ValueError("agent/step result must be an object")
-        if "public_output" not in result and "evaluation_context" not in result and "tool_calls" not in result:
-            raise ValueError("agent/step result should expose public_output, evaluation_context, or tool_calls")
-        reset_resp = agent.send_rpc("agent/reset", {})
-        _assert_rpc_result(reset_resp, "agent/reset")
+        agent.start()
+        started = True
+        initialize = _run_conformance_call(
+            agent,
+            "initialize response",
+            "agent/initialize",
+            {"config": {}},
+            result_validator=validate_initialize_result,
+        )
+        checks.append(initialize)
+        if initialize["passed"]:
+            checks.append(
+                _run_conformance_call(
+                    agent,
+                    "step result contract",
+                    "agent/step",
+                    {"input": step_input},
+                    result_validator=validate_step_result,
+                )
+            )
+        else:
+            checks.append(_skipped_conformance_check("step result contract", "agent/step"))
     except Exception as exc:
-        typer.echo(f"Conformance failed: {exc}", err=True)
-        raise typer.Exit(code=1)
+        checks.append(
+            {
+                "name": "initialize response",
+                "method": "agent/initialize",
+                "passed": False,
+                "message": f"agent could not start: {exc}",
+            }
+        )
+        checks.append(_skipped_conformance_check("step result contract", "agent/step"))
     finally:
-        agent.stop()
-
-    typer.echo("Conformance smoke test passed")
+        if started:
+            checks.append(
+                _run_conformance_call(
+                    agent,
+                    "reset response",
+                    "agent/reset",
+                    {},
+                    result_validator=validate_reset_result,
+                )
+            )
+            agent.stop()
+        else:
+            checks.append(_skipped_conformance_check("reset response", "agent/reset"))
+
+    report = build_conformance_report(target, checks)
+    rendered = json.dumps(report, indent=2)
+    if json_out:
+        json_out.write_text(rendered + "\n", encoding="utf-8")
+    if print_json:
+        typer.echo(rendered)
+    else:
+        for check in checks:
+            marker = "PASS" if check["passed"] else "FAIL"
+            typer.echo(f"{marker} | {check['name']} | {check['message']}")
+        typer.echo(f"Conformance: {report['passed']}/{report['total']} checks passed")
+
+    if not report["conformant"]:
+        raise typer.Exit(code=1)
 
 
 @app.command()
@@ -316,15 +422,28 @@ def trend(
         raise typer.Exit(code=2)
 
 
-def _assert_rpc_result(response: Dict[str, Any], method: str) -> None:
-    if not isinstance(response, dict):
-        raise ValueError(f"{method} response must be a JSON-RPC object")
-    if response.get("jsonrpc") != "2.0":
-        raise ValueError(f"{method} response must include jsonrpc='2.0'")
-    if "error" in response:
-        raise ValueError(f"{method} returned error: {response['error']}")
-    if "result" not in response:
-        raise ValueError(f"{method} response must include result")
+def _run_conformance_call(
+    agent: Any,
+    name: str,
+    method: str,
+    params: Dict[str, Any],
+    *,
+    result_validator: Optional[Callable[[Any], Any]] = None,
+) -> Dict[str, Any]:
+    try:
+        response = agent.send_rpc(method, params)
+    except Exception as exc:
+        return {"name": name, "method": method, "passed": False, "message": str(exc)}
+    return conformance_check(name, method, response, result_validator=result_validator)
+
+
+def _skipped_conformance_check(name: str, method: str) -> Dict[str, Any]:
+    return {
+        "name": name,
+        "method": method,
+        "passed": False,
+        "message": "skipped because a prerequisite failed",
+    }
 
 
 def _starter_agent() -> str:

ecp_runtime-0.5.0/src/ecp_runtime/conformance.py

@@ -0,0 +1,108 @@
+"""Protocol contract validation shared by runtime execution and conformance checks."""
+
+from typing import Any, Callable, Dict, List, Optional
+
+VALID_STATUSES = {"done", "paused"}
+
+
+def validate_rpc_response(response: Any, method: str) -> Any:
+    """Validate the JSON-RPC response envelope and return its result."""
+    if not isinstance(response, dict):
+        raise ValueError(f"{method} response must be a JSON-RPC object")
+    if response.get("jsonrpc") != "2.0":
+        raise ValueError(f"{method} response must include jsonrpc='2.0'")
+    if "id" not in response:
+        raise ValueError(f"{method} response must include id")
+    if "result" in response and "error" in response:
+        raise ValueError(f"{method} response cannot include both result and error")
+    if "error" in response:
+        error = response["error"]
+        if not isinstance(error, dict):
+            raise ValueError(f"{method} error must be an object")
+        if not isinstance(error.get("code"), int) or not isinstance(error.get("message"), str):
+            raise ValueError(f"{method} error must include integer code and string message")
+        raise ValueError(f"{method} returned error: {error}")
+    if "result" not in response:
+        raise ValueError(f"{method} response must include result")
+    return response["result"]
+
+
+def validate_step_result(result: Any) -> Dict[str, Any]:
+    """Validate the normative agent/step result contract."""
+    if not isinstance(result, dict):
+        raise ValueError("agent/step result must be an object")
+
+    status = result.get("status")
+    if status not in VALID_STATUSES:
+        allowed = ", ".join(sorted(VALID_STATUSES))
+        raise ValueError(f"agent/step result status must be one of: {allowed}")
+
+    for field in ("public_output", "evaluation_context", "private_thought", "logs"):
+        value = result.get(field)
+        if value is not None and not isinstance(value, str):
+            raise ValueError(f"agent/step result {field} must be a string or null")
+
+    tool_calls = result.get("tool_calls")
+    if tool_calls is not None:
+        if not isinstance(tool_calls, list):
+            raise ValueError("agent/step result tool_calls must be an array or null")
+        for index, tool_call in enumerate(tool_calls):
+            _validate_tool_call(tool_call, index)
+
+    return result
+
+
+def validate_initialize_result(result: Any) -> Dict[str, Any]:
+    if not isinstance(result, dict):
+        raise ValueError("agent/initialize result must be an object")
+    if not isinstance(result.get("name"), str) or not result["name"]:
+        raise ValueError("agent/initialize result name must be a non-empty string")
+    if not isinstance(result.get("capabilities"), dict):
+        raise ValueError("agent/initialize result capabilities must be an object")
+    return result
+
+
+def validate_reset_result(result: Any) -> bool:
+    if result is not True:
+        raise ValueError("agent/reset result must be true")
+    return result
+
+
+def conformance_check(
+    name: str,
+    method: str,
+    response: Any,
+    *,
+    result_validator: Optional[Callable[[Any], Any]] = None,
+) -> Dict[str, Any]:
+    """Return a stable, serializable result for one protocol check."""
+    try:
+        result = validate_rpc_response(response, method)
+        if result_validator:
+            result_validator(result)
+    except ValueError as exc:
+        return {"name": name, "method": method, "passed": False, "message": str(exc)}
+    return {"name": name, "method": method, "passed": True, "message": "passed"}
+
+
+def build_conformance_report(target: str, checks: List[Dict[str, Any]]) -> Dict[str, Any]:
+    passed = sum(1 for check in checks if check["passed"])
+    total = len(checks)
+    return {
+        "target": target,
+        "conformant": passed == total,
+        "passed": passed,
+        "failed": total - passed,
+        "total": total,
+        "checks": checks,
+    }
+
+
+def _validate_tool_call(tool_call: Any, index: Optional[int] = None) -> None:
+    label = "tool call" if index is None else f"tool_calls[{index}]"
+    if not isinstance(tool_call, dict):
+        raise ValueError(f"agent/step result {label} must be an object")
+    if not isinstance(tool_call.get("name"), str) or not tool_call["name"]:
+        raise ValueError(f"agent/step result {label}.name must be a non-empty string")
+    if "arguments" in tool_call and not isinstance(tool_call["arguments"], dict):
+        raise ValueError(f"agent/step result {label}.arguments must be an object")

{ecp_runtime-0.3.3 → ecp_runtime-0.5.0}/src/ecp_runtime/graders.py

@@ -2,13 +2,19 @@ import os
 import re
 from typing import Any, Dict, List, Tuple
 
-from .manifest import GraderConfig, StepConfig
+try:
+    from .manifest import GraderConfig, StepConfig
+except ImportError:
+    import os
+    import sys
+    sys.path.append(os.path.dirname(__file__))
+    from manifest import GraderConfig, StepConfig  # type: ignore
 
 # Try importing OpenAI, but don't crash if it's missing (unless used)
 try:
-    from openai import OpenAI
+    from openai import OpenAI  # type: ignore
 except ImportError:
-    OpenAI = None
+    OpenAI = None  # type: ignore
 
 
 def _llm_judge_model() -> str:

ecp_runtime-0.5.0/src/ecp_runtime/manifest.py

@@ -0,0 +1,129 @@
+import csv
+import json
+import os
+from typing import Any, Dict, List, Literal, Optional
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class StrictModel(BaseModel):
+    """Base class that keeps runtime validation aligned with the JSON schema."""
+
+    model_config = ConfigDict(extra="forbid")
+
+
+# --- The Grader (Assertion) Schema ---
+class GraderConfig(StrictModel):
+    type: Literal["text_match", "llm_judge", "tool_usage"]
+    field: Literal["public_output", "evaluation_context", "private_thought"] = "public_output"
+    # For text_match
+    condition: Optional[Literal["contains", "equals", "does_not_contain", "regex"]] = None
+    value: Optional[str] = None
+    pattern: Optional[str] = None
+    # For llm_judge
+    prompt: Optional[str] = None
+    assertion: Optional[str] = None
+    # For tool_usage
+    tool_name: Optional[str] = None
+    arguments: Dict[str, Any] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def _validate_by_type(self) -> "GraderConfig":
+        if self.type == "text_match":
+            if not self.condition:
+                raise ValueError("text_match grader requires 'condition'")
+            if self.condition == "regex":
+                if not self.pattern:
+                    raise ValueError("text_match with condition=regex requires 'pattern'")
+            elif self.value is None:
+                raise ValueError(f"text_match with condition={self.condition} requires 'value'")
+
+        elif self.type == "llm_judge":
+            if not self.prompt or not self.prompt.strip():
+                raise ValueError("llm_judge grader requires non-empty 'prompt'")
+
+        elif self.type == "tool_usage":
+            if not isinstance(self.arguments, dict):
+                raise ValueError("tool_usage grader 'arguments' must be an object/dictionary")
+
+        return self
+
+# --- The Step (Scenario) Schema ---
+class StepConfig(StrictModel):
+    input: str
+    constraints: Dict[str, Any] = Field(default_factory=dict)
+    graders: List[GraderConfig] = Field(default_factory=list)
+
+class DatasetConfig(BaseModel):
+    type: Literal["csv", "jsonl"]
+    source: str
+    input_column: str = "input"
+    output_column: str = "output"
+
+class ScenarioConfig(BaseModel):
+    name: str
+    steps: List[StepConfig] = Field(default_factory=list)
+    dataset: Optional[DatasetConfig] = None
+    
+    @model_validator(mode="after")
+    def _validate_steps_or_dataset(self) -> "ScenarioConfig":
+        if not self.steps and not self.dataset:
+            raise ValueError("Scenario requires either 'steps' or 'dataset'")
+        return self
+
+# --- The Root Manifest Schema ---
+class ECPManifest(StrictModel):
+    manifest_version: Literal["v1"]
+    name: str = Field(min_length=1)
+    target: str = Field(min_length=1)
+    scenarios: List[ScenarioConfig]
+
+    @classmethod
+    def from_yaml(cls, path: str) -> "ECPManifest":
+        with open(path, "r", encoding="utf-8") as f:
+            data = yaml.safe_load(f)
+        if not isinstance(data, dict):
+            raise ValueError("Manifest YAML root must be an object")
+        manifest = cls(**data)
+        base_dir = os.path.dirname(os.path.abspath(path))
+        manifest._resolve_datasets(base_dir)
+        return manifest
+
+    def _resolve_datasets(self, base_dir: str):
+        for scenario in self.scenarios:
+            if scenario.dataset and not scenario.steps:
+                source_path = os.path.join(base_dir, scenario.dataset.source)
+                if not os.path.exists(source_path):
+                    raise ValueError(f"Dataset file not found: {source_path}")
+                
+                if scenario.dataset.type == "csv":
+                    with open(source_path, "r", encoding="utf-8") as f:
+                        reader = csv.DictReader(f)
+                        for row in reader:
+                            input_text = row.get(scenario.dataset.input_column, "")
+                            output_text = row.get(scenario.dataset.output_column, "")
+                            scenario.steps.append(StepConfig(
+                                input=input_text,
+                                graders=[GraderConfig(
+                                    type="text_match",
+                                    condition="equals",
+                                    value=output_text
+                                )] if output_text else []
+                            ))
+                elif scenario.dataset.type == "jsonl":
+                    with open(source_path, "r", encoding="utf-8") as f:
+                        for line in f:
+                            if not line.strip():
+                                continue
+                            row = json.loads(line)
+                            input_text = row.get(scenario.dataset.input_column, "")
+                            output_text = row.get(scenario.dataset.output_column, "")
+                            scenario.steps.append(StepConfig(
+                                input=input_text,
+                                graders=[GraderConfig(
+                                    type="text_match",
+                                    condition="equals",
+                                    value=output_text
+                                )] if output_text else []
+                            ))

ecp_runtime-0.5.0/src/ecp_runtime/pytest_plugin.py

@@ -0,0 +1,76 @@
+import os
+from typing import Any, Dict
+
+import pytest
+
+from .runner import _create_agent
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--ecp-target",
+        action="store",
+        default=None,
+        help="The target command or HTTP URL for the ECP agent"
+    )
+
+class ECPAgentFixture:
+    def __init__(self, target: str, rpc_timeout: float = 30.0):
+        self.target = target
+        self.rpc_timeout = rpc_timeout
+        self._agent = None
+    
+    def start(self):
+        self._agent = _create_agent(self.target, self.rpc_timeout)
+        self._agent.start()
+        resp = self._agent.send_rpc("agent/initialize", {"config": {}})
+        self._ensure_rpc_success(resp, "agent/initialize")
+        return self
+
+    def stop(self):
+        if self._agent:
+            self._agent.stop()
+            self._agent = None
+
+    def step(self, input_text: str) -> Dict[str, Any]:
+        resp = self._agent.send_rpc("agent/step", {"input": input_text})
+        self._ensure_rpc_success(resp, "agent/step")
+        return resp.get("result", {})
+
+    def reset(self) -> Dict[str, Any]:
+        resp = self._agent.send_rpc("agent/reset", {})
+        self._ensure_rpc_success(resp, "agent/reset")
+        return resp.get("result", {})
+
+    def _ensure_rpc_success(self, rpc_resp: Dict[str, Any], method: str) -> None:
+        if "error" not in rpc_resp:
+            return
+        error = rpc_resp.get("error") or {}
+        code = error.get("code")
+        message = error.get("message", "Unknown JSON-RPC error")
+        raise RuntimeError(
+            f"RPC call failed ({method}): code={code}, message={message}"
+        )
+
+@pytest.fixture
+def ecp_agent(request):
+    target = request.config.getoption("--ecp-target")
+    marker = request.node.get_closest_marker("ecp")
+    if marker and "target" in marker.kwargs:
+        target = marker.kwargs["target"]
+        
+    if not target:
+        pytest.skip("No ECP target specified. Use --ecp-target or @pytest.mark.ecp(target=...)")
+        
+    rpc_timeout = float(os.environ.get("ECP_RPC_TIMEOUT", "30"))
+    agent = ECPAgentFixture(target, rpc_timeout=rpc_timeout)
+    agent.start()
+    
+    yield agent
+    
+    agent.stop()
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "ecp: marks tests that require an ECP agent"
+    )