ecp-runtime 0.3.0__tar.gz → 0.3.2__tar.gz

{ecp_runtime-0.3.0 → ecp_runtime-0.3.2}/.gitignore

@@ -208,3 +208,4 @@ cython_debug/
 marimo/_static/
 marimo/_lsp/
 __marimo__/
+/worktrees

{ecp_runtime-0.3.0 → ecp_runtime-0.3.2}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: ecp-runtime
-Version: 0.3.0
-Summary: The reference runtime for the Evaluation Context Protocol (ECP).
+Version: 0.3.2
+Summary: Vendor-neutral runtime for portable AI agent evaluations with outputs, tool calls, and audit context.
 Project-URL: Homepage, https://github.com/evaluation-context-protocol/ecp
 Project-URL: Documentation, https://evaluationcontextprotocol.io/
 Project-URL: Repository, https://github.com/evaluation-context-protocol/ecp
 Project-URL: Issues, https://github.com/evaluation-context-protocol/ecp/issues
-Author-email: ECP Maintainers <hello@ecp.org>
+Author-email: ECP Maintainers <aniket.wattamwar17@gmail.com>
 Requires-Python: >=3.9
 Requires-Dist: jinja2>=3.1.0
 Requires-Dist: openai>=1.0.0
@@ -18,33 +18,39 @@ Description-Content-Type: text/markdown
 
 # ECP Runtime
 
-This is the reference implementation of the Evaluation Context Protocol (ECP) Runtime.
-It includes the CLI tool `ecp` for running agent evaluations.
+Reference runtime and CLI for the Evaluation Context Protocol (ECP).
+
+ECP is a vendor-neutral protocol for testing agent outputs, tool calls, and evaluator-visible audit context across frameworks, models, eval platforms, and CI systems.
 
 ## Install
 
 ```bash
-pip install "ecp-runtime==0.2.9"
+pip install "ecp-runtime==0.3.2"
 ```
 
-The latest stable PyPI release is now `0.2.9` and matches the current GitHub release line.
-
 ## Usage
 
-Run an evaluation manifest:
+```bash
+ecp init
+ecp validate ecp_eval/manifest.yaml
+ecp run --manifest ecp_eval/manifest.yaml --json
+```
+
+Run the flagship example:
 
 ```bash
-ecp run --manifest .\examples\langchain_demo\manifest.yaml
+ecp run --manifest examples/customer_support_demo/manifest.yaml --report report.html
 ```
 
-You can also run via module entrypoint:
+Useful commands:
 
 ```bash
-python -m ecp_runtime.cli run --manifest .\examples\langchain_demo\manifest.yaml
+ecp validate examples/customer_support_demo/manifest.yaml
+ecp conformance --target "python examples/customer_support_demo/agent.py"
+ecp doctor
 ```
 
-Manifest `target` values may be either a command for the default stdio transport
-or an ECP Streamable HTTP endpoint:
+Manifest `target` values may be either a command for the default stdio transport or an ECP Streamable HTTP endpoint:
 
 ```yaml
 target: "http://127.0.0.1:8765/ecp"
@@ -62,3 +68,4 @@ $env:ECP_LLM_JUDGE_MODEL="gpt-4o-mini"
 - Documentation: https://evaluationcontextprotocol.io/
 - Repository: https://github.com/evaluation-context-protocol/ecp
 - Issues: https://github.com/evaluation-context-protocol/ecp/issues
+

ecp_runtime-0.3.2/README.md

@@ -0,0 +1,53 @@
+# ECP Runtime
+
+Reference runtime and CLI for the Evaluation Context Protocol (ECP).
+
+ECP is a vendor-neutral protocol for testing agent outputs, tool calls, and evaluator-visible audit context across frameworks, models, eval platforms, and CI systems.
+
+## Install
+
+```bash
+pip install "ecp-runtime==0.3.2"
+```
+
+## Usage
+
+```bash
+ecp init
+ecp validate ecp_eval/manifest.yaml
+ecp run --manifest ecp_eval/manifest.yaml --json
+```
+
+Run the flagship example:
+
+```bash
+ecp run --manifest examples/customer_support_demo/manifest.yaml --report report.html
+```
+
+Useful commands:
+
+```bash
+ecp validate examples/customer_support_demo/manifest.yaml
+ecp conformance --target "python examples/customer_support_demo/agent.py"
+ecp doctor
+```
+
+Manifest `target` values may be either a command for the default stdio transport or an ECP Streamable HTTP endpoint:
+
+```yaml
+target: "http://127.0.0.1:8765/ecp"
+```
+
+If your manifest includes `llm_judge`, set an API key and optional judge model:
+
+```bash
+$env:OPENAI_API_KEY="your_key_here"
+$env:ECP_LLM_JUDGE_MODEL="gpt-4o-mini"
+```
+
+## Links
+
+- Documentation: https://evaluationcontextprotocol.io/
+- Repository: https://github.com/evaluation-context-protocol/ecp
+- Issues: https://github.com/evaluation-context-protocol/ecp/issues
+

{ecp_runtime-0.3.0 → ecp_runtime-0.3.2}/pyproject.toml

@@ -4,10 +4,10 @@ build-backend = "hatchling.build"
 
 [project]
 name = "ecp-runtime"
-version = "0.3.0"
-description = "The reference runtime for the Evaluation Context Protocol (ECP)."
+version = "0.3.2"
+description = "Vendor-neutral runtime for portable AI agent evaluations with outputs, tool calls, and audit context."
 authors = [
-  { name = "ECP Maintainers", email = "hello@ecp.org" },
+  { name = "ECP Maintainers", email = "aniket.wattamwar17@gmail.com" },
 ]
 readme = "README.md"
 requires-python = ">=3.9"
@@ -29,3 +29,4 @@ Issues = "https://github.com/evaluation-context-protocol/ecp/issues"
 [project.scripts]
 ecp = "ecp_runtime.cli:app" 
 
+

ecp_runtime-0.3.2/src/ecp_runtime/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.3.2"
+

ecp_runtime-0.3.2/src/ecp_runtime/cli.py

@@ -0,0 +1,384 @@
+import glob
+import json
+import logging
+import os
+import shutil
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import typer
+from pydantic import ValidationError
+
+from .manifest import ECPManifest
+from .reporter import HTMLReporter
+from .runner import ECPRunner
+from .trend import RunTrendAnalyzer
+
+app = typer.Typer(
+    name="ecp",
+    help="Evaluation Context Protocol Runtime CLI",
+    add_completion=False
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _configure_logging(verbose: bool) -> None:
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(
+        level=level,
+        format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+        datefmt="%H:%M:%S",
+    )
+
+
+@app.callback()
+def main():
+    """
+    Official Runtime for the Evaluation Context Protocol (ECP).
+    """
+    pass
+
+
+@app.command()
+def run(
+    manifest: Path = typer.Option(
+        ...,
+        "--manifest", "-m",
+        exists=True,
+        file_okay=True,
+        dir_okay=False,
+        writable=False,
+        readable=True,
+        resolve_path=True,
+        help="Path to the test manifest YAML file"
+    ),
+
+    verbose: bool = False,
+    report: Optional[Path] = typer.Option(
+        None,
+        "--report",
+        help="Path to save an HTML report (e.g., output.html)",
+        resolve_path=True,
+    ),
+    json_out: Optional[Path] = typer.Option(
+        None,
+        "--json-out",
+        help="Path to save a JSON report (e.g., output.json)",
+        resolve_path=True,
+    ),
+    print_json: bool = typer.Option(
+        False,
+        "--json",
+        help="Print the JSON report to stdout",
+    ),
+    fail_on_error: bool = typer.Option(
+        True,
+        "--fail-on-error/--no-fail-on-error",
+        help="Exit non-zero if any checks fail (useful for CI)",
+    ),
+):
+    """
+    Execute an evaluation run based on a manifest file.
+    """
+    _configure_logging(verbose)
+
+    logger.info("ECP Runtime Initializing...")
+    logger.info("Loading manifest: %s", manifest)
+
+    try:
+        # Load the YAML
+        config = ECPManifest.from_yaml(str(manifest))
+
+        # Run the Tests
+        runner = ECPRunner(config)
+        result_summary = runner.run_scenarios()
+        total = int(result_summary.get("total", 0) or 0)
+        passed = int(result_summary.get("passed", 0) or 0)
+        failed = max(total - passed, 0)
+
+        report_payload: Dict[str, Any] = {
+            "manifest": str(manifest),
+            "passed": passed,
+            "total": total,
+            "failed": failed,
+            "scenarios": result_summary.get("scenarios", []),
+        }
+
+        if report:
+            logger.info("Generating HTML report: %s", report)
+            reporter = HTMLReporter()
+            # Feed scenarios directly to reporter
+            for scenario in result_summary.get("scenarios", []):
+                reporter.add_scenario(scenario.get("name"), scenario.get("steps", []))
+            reporter.save(str(report))
+            logger.info("Report saved to %s", report)
+
+        if json_out:
+            json_out.write_text(json.dumps(report_payload, indent=2), encoding="utf-8")
+            logger.info("JSON report saved to %s", json_out)
+
+        if print_json:
+            typer.echo(json.dumps(report_payload, indent=2))
+
+        if fail_on_error and failed > 0:
+            raise typer.Exit(code=2)
+
+    except typer.Exit:
+        raise
+    except Exception as e:
+        logger.error("CRITICAL ERROR: %s", e)
+        if verbose:
+            raise e
+        sys.exit(1)
+
+
+@app.command()
+def validate(
+    manifest: Path = typer.Argument(
+        ...,
+        exists=True,
+        file_okay=True,
+        dir_okay=False,
+        readable=True,
+        resolve_path=True,
+        help="Path to the ECP manifest YAML file",
+    ),
+):
+    """
+    Validate a manifest without running the agent.
+    """
+    try:
+        config = ECPManifest.from_yaml(str(manifest))
+    except ValidationError as exc:
+        typer.echo(f"Manifest invalid: {manifest}", err=True)
+        typer.echo(str(exc), err=True)
+        raise typer.Exit(code=1)
+    except Exception as exc:
+        typer.echo(f"Manifest invalid: {manifest}", err=True)
+        typer.echo(str(exc), err=True)
+        raise typer.Exit(code=1)
+
+    step_count = sum(len(scenario.steps) for scenario in config.scenarios)
+    grader_count = sum(len(step.graders) for scenario in config.scenarios for step in scenario.steps)
+    typer.echo(f"Manifest valid: {manifest}")
+    typer.echo(f"Name: {config.name}")
+    typer.echo(f"Scenarios: {len(config.scenarios)}")
+    typer.echo(f"Steps: {step_count}")
+    typer.echo(f"Graders: {grader_count}")
+
+
+@app.command()
+def doctor():
+    """
+    Check the local ECP runtime environment.
+    """
+    typer.echo("ECP doctor")
+    typer.echo(f"Python: {sys.version.split()[0]}")
+    typer.echo(f"Executable: {sys.executable}")
+    typer.echo(f"Working directory: {Path.cwd()}")
+    typer.echo(f"OpenAI key: {'set' if os.environ.get('OPENAI_API_KEY') else 'not set'}")
+    typer.echo(f"Git: {'available' if shutil.which('git') else 'not found'}")
+
+
+@app.command()
+def init(
+    directory: Path = typer.Argument(
+        Path("ecp_eval"),
+        help="Directory where a starter ECP agent and manifest should be created",
+    ),
+    force: bool = typer.Option(
+        False,
+        "--force",
+        help="Overwrite existing starter files",
+    ),
+):
+    """
+    Create a minimal ECP evaluation project.
+    """
+    directory.mkdir(parents=True, exist_ok=True)
+    agent_path = directory / "agent.py"
+    manifest_path = directory / "manifest.yaml"
+
+    if not force and (agent_path.exists() or manifest_path.exists()):
+        typer.echo(f"Refusing to overwrite existing files in {directory}. Use --force to replace starters.", err=True)
+        raise typer.Exit(code=1)
+
+    agent_path.write_text(_starter_agent(), encoding="utf-8")
+    manifest_path.write_text(_starter_manifest(agent_path.as_posix()), encoding="utf-8")
+
+    typer.echo(f"Created {agent_path}")
+    typer.echo(f"Created {manifest_path}")
+    typer.echo(f"Try: ecp validate {manifest_path}")
+    typer.echo(f"Then: ecp run --manifest {manifest_path}")
+
+
+@app.command()
+def conformance(
+    target: str = typer.Option(
+        ...,
+        "--target",
+        "-t",
+        help="Agent command or ECP HTTP endpoint to check",
+    ),
+    step_input: str = typer.Option(
+        "hello",
+        "--input",
+        help="Input text for the conformance step call",
+    ),
+):
+    """
+    Run a small protocol conformance smoke test against an ECP agent.
+    """
+    runner = ECPRunner(type("Manifest", (), {"target": target, "scenarios": []})())
+    agent = runner._create_agent(target, rpc_timeout=float(os.environ.get("ECP_RPC_TIMEOUT", "30")))
+    agent.start()
+    try:
+        init_resp = agent.send_rpc("agent/initialize", {"config": {}})
+        _assert_rpc_result(init_resp, "agent/initialize")
+        step_resp = agent.send_rpc("agent/step", {"input": step_input})
+        _assert_rpc_result(step_resp, "agent/step")
+        result = step_resp.get("result", {})
+        if not isinstance(result, dict):
+            raise ValueError("agent/step result must be an object")
+        if "public_output" not in result and "evaluation_context" not in result and "tool_calls" not in result:
+            raise ValueError("agent/step result should expose public_output, evaluation_context, or tool_calls")
+        reset_resp = agent.send_rpc("agent/reset", {})
+        _assert_rpc_result(reset_resp, "agent/reset")
+    except Exception as exc:
+        typer.echo(f"Conformance failed: {exc}", err=True)
+        raise typer.Exit(code=1)
+    finally:
+        agent.stop()
+
+    typer.echo("Conformance smoke test passed")
+
+
+@app.command()
+def trend(
+    pattern: str = typer.Argument(
+        ...,
+        help="Glob pattern matching saved JSON report files (e.g. 'results/run-*.json')",
+    ),
+    window: int = typer.Option(
+        20,
+        "--window",
+        "-w",
+        help="Maximum number of recent runs to include in the analysis",
+        min=1,
+    ),
+    exit_on_regression: bool = typer.Option(
+        False,
+        "--exit-on-regression",
+        help="Exit with code 2 when a degrading trend is detected (for CI gates)",
+    ),
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        "-v",
+        help="Enable verbose output",
+    ),
+):
+    """
+    Analyse pass-rate trends across a sequence of saved JSON report files.
+    """
+    _configure_logging(verbose)
+
+    matched: List[Path] = sorted(Path(path) for path in glob.glob(pattern, recursive=True))
+
+    if not matched:
+        typer.echo(f"No files matched the pattern: {pattern}", err=True)
+        raise typer.Exit(code=1)
+
+    typer.echo(f"Found {len(matched)} report(s). Analysing last {window}...")
+
+    analyzer = RunTrendAnalyzer(matched, window=window)
+    report = analyzer.analyze()
+
+    typer.echo("")
+    typer.echo(f"{'Run':<5}  {'Manifest':<45}  {'Passed':>6}  {'Total':>6}  {'Rate':>6}")
+    typer.echo("-" * 74)
+    for index, run in enumerate(report.runs, start=1):
+        manifest_label = Path(run.manifest).name if len(run.manifest) > 45 else run.manifest
+        rate_pct = f"{run.pass_rate * 100:.1f}%"
+        typer.echo(f"{index:<5}  {manifest_label:<45}  {run.passed:>6}  {run.total:>6}  {rate_pct:>6}")
+
+    typer.echo("")
+    typer.echo(f"Trend Direction : {report.direction.upper()}")
+    typer.echo(f"Slope (per run) : {report.pass_rate_slope:+.6f}")
+    typer.echo(f"Window          : {len(report.runs)} run(s) analysed (max {report.window})")
+    typer.echo(f"Regression flag : {'YES' if report.any_regression else 'No'}")
+    typer.echo("")
+
+    if report.any_regression and exit_on_regression:
+        typer.echo("Regression detected. Exiting with code 2 (--exit-on-regression).", err=True)
+        raise typer.Exit(code=2)
+
+
+def _assert_rpc_result(response: Dict[str, Any], method: str) -> None:
+    if not isinstance(response, dict):
+        raise ValueError(f"{method} response must be a JSON-RPC object")
+    if response.get("jsonrpc") != "2.0":
+        raise ValueError(f"{method} response must include jsonrpc='2.0'")
+    if "error" in response:
+        raise ValueError(f"{method} returned error: {response['error']}")
+    if "result" not in response:
+        raise ValueError(f"{method} response must include result")
+
+
+def _starter_agent() -> str:
+    return '''from ecp import Result, agent, on_reset, on_step, serve
+
+
+@agent(name="StarterAgent")
+class StarterAgent:
+    def __init__(self):
+        self.seen = []
+
+    @on_step
+    def step(self, user_input: str):
+        self.seen.append(user_input)
+        return Result(
+            public_output=f"Echo: {user_input}",
+            evaluation_context="The starter agent echoed the user input.",
+            tool_calls=[{"name": "echo", "arguments": {"text": user_input}}],
+        )
+
+    @on_reset
+    def reset(self):
+        self.seen.clear()
+
+
+if __name__ == "__main__":
+    serve(StarterAgent())
+'''
+
+
+def _starter_manifest(agent_filename: str) -> str:
+    return f'''manifest_version: "v1"
+name: "Starter ECP Evaluation"
+target: 'python {agent_filename}'
+
+scenarios:
+  - name: "Echo contract"
+    steps:
+      - input: "hello ecp"
+        graders:
+          - type: text_match
+            field: public_output
+            condition: contains
+            value: "hello ecp"
+          - type: text_match
+            field: evaluation_context
+            condition: contains
+            value: "echoed"
+          - type: tool_usage
+            tool_name: "echo"
+            arguments:
+              text: "hello ecp"
+'''
+
+
+if __name__ == "__main__":
+    app()

{ecp_runtime-0.3.0 → ecp_runtime-0.3.2}/src/ecp_runtime/graders.py

@@ -112,8 +112,10 @@ def evaluate_step(step_config: StepConfig, result_obj: Any) -> List[Dict[str, An
         target_text = ""
         if grader.field == "public_output":
             target_text = result_obj.public_output
+        elif grader.field == "evaluation_context":
+            target_text = getattr(result_obj, "evaluation_context", None)
         elif grader.field == "private_thought":
-            target_text = result_obj.private_thought
+            target_text = getattr(result_obj, "private_thought", None) or getattr(result_obj, "evaluation_context", None)
             
         passed = False
         reasoning = ""
@@ -132,7 +134,7 @@ def evaluate_step(step_config: StepConfig, result_obj: Any) -> List[Dict[str, An
         
         check_results.append({
             "type": grader.type,
-            "field": grader.field,
+            "field": "tool_calls" if grader.type == "tool_usage" else grader.field,
             "passed": passed,
             "score": score,
             "reasoning": reasoning

{ecp_runtime-0.3.0 → ecp_runtime-0.3.2}/src/ecp_runtime/manifest.py

@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field, model_validator
 # --- The Grader (Assertion) Schema ---
 class GraderConfig(BaseModel):
     type: Literal["text_match", "llm_judge", "tool_usage"]
-    field: Literal["public_output", "private_thought"] = "public_output"
+    field: Literal["public_output", "evaluation_context", "private_thought"] = "public_output"
     # For text_match
     condition: Optional[Literal["contains", "equals", "does_not_contain", "regex"]] = None
     value: Optional[str] = None

{ecp_runtime-0.3.0 → ecp_runtime-0.3.2}/src/ecp_runtime/reporter.py

@@ -30,6 +30,13 @@ HTML_TEMPLATE = """
         <div style="border-top: 1px solid #eee; margin-top: 10px; padding-top: 10px;">
             <p><strong>Input:</strong> {{ step.input }}</p>
             <p><strong>Output:</strong> {{ step.output }}</p>
+            {% if step.evaluation_context %}
+            <p><strong>Evaluation context:</strong> {{ step.evaluation_context }}</p>
+            {% endif %}
+            {% if step.tool_calls %}
+            <h4>Tool Calls:</h4>
+            <pre>{{ step.tool_calls | tojson(indent=2) }}</pre>
+            {% endif %}
 
             <h4>Graders:</h4>
             <ul>

{ecp_runtime-0.3.0 → ecp_runtime-0.3.2}/src/ecp_runtime/runner.py

@@ -26,6 +26,7 @@ logger = logging.getLogger(__name__)
 class StepResult:
     status: str
     public_output: Optional[str] = None
+    evaluation_context: Optional[str] = None
     private_thought: Optional[str] = None
     logs: Optional[str] = None
     tool_calls: Optional[List[Dict[str, Any]]] = None
@@ -252,14 +253,15 @@ class ECPRunner:
                     step_result = StepResult(
                         status=result_data.get("status", "done"),
                         public_output=result_data.get("public_output"),
-                        private_thought=result_data.get("private_thought"),
+                        evaluation_context=result_data.get("evaluation_context") or result_data.get("private_thought"),
+                        private_thought=result_data.get("private_thought") or result_data.get("evaluation_context"),
                         tool_calls=result_data.get("tool_calls") if isinstance(result_data.get("tool_calls"), list) else None
                     )
 
                     logger.info("Step %d: Input='%s'", i + 1, step.input)
                     logger.info("Output: %s", step_result.public_output)
-                    if step_result.private_thought:
-                        logger.debug("Thought: %s", step_result.private_thought)
+                    if step_result.evaluation_context:
+                        logger.debug("Evaluation context: %s", step_result.evaluation_context)
 
                     checks = evaluate_step(step, step_result)
 
@@ -278,6 +280,8 @@ class ECPRunner:
                     scenario_steps.append({
                         "input": step.input,
                         "output": step_result.public_output,
+                        "evaluation_context": step_result.evaluation_context,
+                        "tool_calls": step_result.tool_calls or [],
                         "checks": checks
                     })
 

{ecp_runtime-0.3.0 → ecp_runtime-0.3.2}/tests/test_cli.py

@@ -72,6 +72,29 @@ class CLISmokeTests(unittest.TestCase):
             result = self.runner.invoke(app, ["run", "--manifest", self.manifest_path, "--no-fail-on-error"])
         self.assertEqual(result.exit_code, 0, msg=result.output)
 
+    def test_validate_command(self) -> None:
+        result = self.runner.invoke(app, ["validate", self.manifest_path])
+
+        self.assertEqual(result.exit_code, 0, msg=result.output)
+        self.assertIn("Manifest valid", result.output)
+
+    def test_doctor_command(self) -> None:
+        result = self.runner.invoke(app, ["doctor"])
+
+        self.assertEqual(result.exit_code, 0, msg=result.output)
+        self.assertIn("ECP doctor", result.output)
+
+    def test_init_command_creates_starter_files(self) -> None:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            target = Path(tmpdir) / "evals"
+            result = self.runner.invoke(app, ["init", str(target)])
+
+            self.assertEqual(result.exit_code, 0, msg=result.output)
+            self.assertTrue((target / "agent.py").exists())
+            manifest = target / "manifest.yaml"
+            self.assertTrue(manifest.exists())
+            self.assertIn("evaluation_context", manifest.read_text(encoding="utf-8"))
+
 
 if __name__ == "__main__":
     unittest.main()

{ecp_runtime-0.3.0 → ecp_runtime-0.3.2}/tests/test_graders.py

@@ -81,6 +81,26 @@ class EvaluateStepSmokeTests(unittest.TestCase):
         self.assertEqual(len(checks), 1)
         self.assertTrue(checks[0]["passed"])
 
+    def test_step_can_grade_evaluation_context(self) -> None:
+        grader = GraderConfig(
+            type="text_match",
+            field="evaluation_context",
+            condition="contains",
+            value="policy checked",
+        )
+        step = StepConfig(input="refund", graders=[grader])
+        result_obj = SimpleNamespace(
+            public_output="ok",
+            evaluation_context="policy checked before response",
+            private_thought=None,
+            tool_calls=[],
+        )
+
+        from ecp_runtime.graders import evaluate_step
+
+        checks = evaluate_step(step, result_obj)
+        self.assertTrue(checks[0]["passed"])
+
 
 if __name__ == "__main__":
     unittest.main()

{ecp_runtime-0.3.0 → ecp_runtime-0.3.2}/tests/test_manifest.py

@@ -33,6 +33,16 @@ class ManifestValidationTests(unittest.TestCase):
         with self.assertRaises(ValidationError):
             GraderConfig(type="unknown")  # type: ignore[arg-type]
 
+    def test_evaluation_context_field_allowed(self) -> None:
+        grader = GraderConfig(
+            type="text_match",
+            field="evaluation_context",
+            condition="contains",
+            value="checked",
+        )
+
+        self.assertEqual(grader.field, "evaluation_context")
+
     def test_manifest_root_must_be_mapping(self) -> None:
         with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False, encoding="utf-8") as tmp:
             tmp.write("- not: a mapping\n")

{ecp_runtime-0.3.0 → ecp_runtime-0.3.2}/tests/test_runner.py

@@ -33,7 +33,11 @@ class RunnerTests(unittest.TestCase):
             def send_rpc(self, method, params=None):
                 if method == "agent/initialize":
                     return {"jsonrpc": "2.0", "id": 1, "result": {"name": "x", "capabilities": {}}}
-                return {"jsonrpc": "2.0", "id": 2, "result": {"status": "done", "public_output": "ok"}}
+                return {
+                    "jsonrpc": "2.0",
+                    "id": 2,
+                    "result": {"status": "done", "public_output": "ok", "evaluation_context": "checked"},
+                }
 
         with mock.patch("ecp_runtime.runner.AgentProcess", FakeAgentProcess):
             output = ECPRunner(self._manifest()).run_scenarios()
@@ -41,6 +45,7 @@ class RunnerTests(unittest.TestCase):
         self.assertEqual(output["total"], 0)
         self.assertEqual(len(output["scenarios"]), 1)
         self.assertEqual(output["scenarios"][0]["name"], "Scenario A")
+        self.assertEqual(output["scenarios"][0]["steps"][0]["evaluation_context"], "checked")
 
     def test_runner_raises_on_rpc_error_with_context(self) -> None:
         class FakeAgentProcess:

ecp_runtime-0.3.0/README.md

@@ -1,46 +0,0 @@
-# ECP Runtime
-
-This is the reference implementation of the Evaluation Context Protocol (ECP) Runtime.
-It includes the CLI tool `ecp` for running agent evaluations.
-
-## Install
-
-```bash
-pip install "ecp-runtime==0.2.9"
-```
-
-The latest stable PyPI release is now `0.2.9` and matches the current GitHub release line.
-
-## Usage
-
-Run an evaluation manifest:
-
-```bash
-ecp run --manifest .\examples\langchain_demo\manifest.yaml
-```
-
-You can also run via module entrypoint:
-
-```bash
-python -m ecp_runtime.cli run --manifest .\examples\langchain_demo\manifest.yaml
-```
-
-Manifest `target` values may be either a command for the default stdio transport
-or an ECP Streamable HTTP endpoint:
-
-```yaml
-target: "http://127.0.0.1:8765/ecp"
-```
-
-If your manifest includes `llm_judge`, set an API key and optional judge model:
-
-```bash
-$env:OPENAI_API_KEY="your_key_here"
-$env:ECP_LLM_JUDGE_MODEL="gpt-4o-mini"
-```
-
-## Links
-
-- Documentation: https://evaluationcontextprotocol.io/
-- Repository: https://github.com/evaluation-context-protocol/ecp
-- Issues: https://github.com/evaluation-context-protocol/ecp/issues

ecp_runtime-0.3.0/src/ecp_runtime/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.2.9"

ecp_runtime-0.3.0/src/ecp_runtime/cli.py

@@ -1,205 +0,0 @@
-import glob
-import json
-import logging
-import os
-import sys
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-import typer
-
-from .reporter import HTMLReporter
-from .trend import RunTrendAnalyzer
-
-# Import local modules (Using relative imports)
-try:
-    from .manifest import ECPManifest
-    from .runner import ECPRunner
-except ImportError:
-    # Fallback for direct execution debugging
-    sys.path.append(os.path.dirname(__file__))
-    from manifest import ECPManifest
-    from runner import ECPRunner
-
-app = typer.Typer(
-    name="ecp",
-    help="Evaluation Context Protocol Runtime CLI",
-    add_completion=False
-)
-
-logger = logging.getLogger(__name__)
-
-
-def _configure_logging(verbose: bool) -> None:
-    level = logging.DEBUG if verbose else logging.INFO
-    logging.basicConfig(
-        level=level,
-        format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
-        datefmt="%H:%M:%S",
-    )
-
-
-@app.callback()
-def main():
-    """
-    Official Runtime for the Evaluation Context Protocol (ECP).
-    """
-    pass
-
-
-@app.command()
-def run(
-    manifest: Path = typer.Option(
-        ...,
-        "--manifest", "-m",
-        exists=True,
-        file_okay=True,
-        dir_okay=False,
-        writable=False,
-        readable=True,
-        resolve_path=True,
-        help="Path to the test manifest YAML file"
-    ),
-
-    verbose: bool = False,
-    report: Optional[Path] = typer.Option(
-        None,
-        "--report",
-        help="Path to save an HTML report (e.g., output.html)",
-        resolve_path=True,
-    ),
-    json_out: Optional[Path] = typer.Option(
-        None,
-        "--json-out",
-        help="Path to save a JSON report (e.g., output.json)",
-        resolve_path=True,
-    ),
-    print_json: bool = typer.Option(
-        False,
-        "--json",
-        help="Print the JSON report to stdout",
-    ),
-    fail_on_error: bool = typer.Option(
-        True,
-        "--fail-on-error/--no-fail-on-error",
-        help="Exit non-zero if any checks fail (useful for CI)",
-    ),
-):
-    """
-    Execute an evaluation run based on a manifest file.
-    """
-    _configure_logging(verbose)
-
-    logger.info("ECP Runtime Initializing...")
-    logger.info("Loading manifest: %s", manifest)
-
-    try:
-        # Load the YAML
-        config = ECPManifest.from_yaml(str(manifest))
-
-        # Run the Tests
-        runner = ECPRunner(config)
-        result_summary = runner.run_scenarios()
-        total = int(result_summary.get("total", 0) or 0)
-        passed = int(result_summary.get("passed", 0) or 0)
-        failed = max(total - passed, 0)
-
-        report_payload: Dict[str, Any] = {
-            "manifest": str(manifest),
-            "passed": passed,
-            "total": total,
-            "failed": failed,
-            "scenarios": result_summary.get("scenarios", []),
-        }
-
-        if report:
-            logger.info("Generating HTML report: %s", report)
-            reporter = HTMLReporter()
-            # Feed scenarios directly to reporter
-            for scenario in result_summary.get("scenarios", []):
-                reporter.add_scenario(scenario.get("name"), scenario.get("steps", []))
-            reporter.save(str(report))
-            logger.info("Report saved to %s", report)
-
-        if json_out:
-            json_out.write_text(json.dumps(report_payload, indent=2), encoding="utf-8")
-            logger.info("JSON report saved to %s", json_out)
-
-        if print_json:
-            typer.echo(json.dumps(report_payload, indent=2))
-
-        if fail_on_error and failed > 0:
-            raise typer.Exit(code=2)
-
-    except typer.Exit:
-        raise
-    except Exception as e:
-        logger.error("CRITICAL ERROR: %s", e)
-        if verbose:
-            raise e
-        sys.exit(1)
-
-
-@app.command()
-def trend(
-    pattern: str = typer.Argument(
-        ...,
-        help="Glob pattern matching saved JSON report files (e.g. 'results/run-*.json')",
-    ),
-    window: int = typer.Option(
-        20,
-        "--window",
-        "-w",
-        help="Maximum number of recent runs to include in the analysis",
-        min=1,
-    ),
-    exit_on_regression: bool = typer.Option(
-        False,
-        "--exit-on-regression",
-        help="Exit with code 2 when a degrading trend is detected (for CI gates)",
-    ),
-    verbose: bool = typer.Option(
-        False,
-        "--verbose",
-        "-v",
-        help="Enable verbose output",
-    ),
-):
-    """
-    Analyse pass-rate trends across a sequence of saved JSON report files.
-    """
-    _configure_logging(verbose)
-
-    matched: List[Path] = sorted(Path(path) for path in glob.glob(pattern, recursive=True))
-
-    if not matched:
-        typer.echo(f"No files matched the pattern: {pattern}", err=True)
-        raise typer.Exit(code=1)
-
-    typer.echo(f"Found {len(matched)} report(s). Analysing last {window}...")
-
-    analyzer = RunTrendAnalyzer(matched, window=window)
-    report = analyzer.analyze()
-
-    typer.echo("")
-    typer.echo(f"{'Run':<5}  {'Manifest':<45}  {'Passed':>6}  {'Total':>6}  {'Rate':>6}")
-    typer.echo("-" * 74)
-    for index, run in enumerate(report.runs, start=1):
-        manifest_label = Path(run.manifest).name if len(run.manifest) > 45 else run.manifest
-        rate_pct = f"{run.pass_rate * 100:.1f}%"
-        typer.echo(f"{index:<5}  {manifest_label:<45}  {run.passed:>6}  {run.total:>6}  {rate_pct:>6}")
-
-    typer.echo("")
-    typer.echo(f"Trend Direction : {report.direction.upper()}")
-    typer.echo(f"Slope (per run) : {report.pass_rate_slope:+.6f}")
-    typer.echo(f"Window          : {len(report.runs)} run(s) analysed (max {report.window})")
-    typer.echo(f"Regression flag : {'YES' if report.any_regression else 'No'}")
-    typer.echo("")
-
-    if report.any_regression and exit_on_regression:
-        typer.echo("Regression detected. Exiting with code 2 (--exit-on-regression).", err=True)
-        raise typer.Exit(code=2)
-
-
-if __name__ == "__main__":
-    app()