ecp-runtime 0.2.1__tar.gz

ecp_runtime-0.2.1/.gitignore

@@ -0,0 +1,207 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/

ecp_runtime-0.2.1/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.4
+Name: ecp-runtime
+Version: 0.2.1
+Summary: The reference runtime for the Evaluation Context Protocol (ECP).
+Author-email: ECP Maintainers <hello@ecp.org>
+Requires-Python: >=3.9
+Requires-Dist: jinja2>=3.1.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0
+Requires-Dist: typer>=0.9.0
+Description-Content-Type: text/markdown
+
+# ECP Runtime
+
+This is the reference implementation of the Evaluation Context Protocol (ECP) Runtime.
+It includes the CLI tool `ecp` for running agent evaluations.

ecp_runtime-0.2.1/README.md

@@ -0,0 +1,4 @@
+# ECP Runtime
+
+This is the reference implementation of the Evaluation Context Protocol (ECP) Runtime.
+It includes the CLI tool `ecp` for running agent evaluations.

ecp_runtime-0.2.1/pyproject.toml

@@ -0,0 +1,24 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "ecp-runtime"
+version = "0.2.1"
+description = "The reference runtime for the Evaluation Context Protocol (ECP)."
+authors = [
+  { name = "ECP Maintainers", email = "hello@ecp.org" },
+]
+readme = "README.md"
+requires-python = ">=3.9"
+dependencies = [
+    "typer>=0.9.0",         # For the CLI
+    "pyyaml>=6.0",          # For reading Manifests
+    "pydantic>=2.0",        # For validating Schemas
+    "rich>=13.0",           # For pretty terminal output
+    "openai>=1.0.0",        # For the LLM Judge
+  "Jinja2>=3.1.0",        # For HTML report rendering
+]
+
+[project.scripts]
+ecp = "ecp_runtime.cli:app" 

ecp_runtime-0.2.1/src/ecp_runtime/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

ecp_runtime-0.2.1/src/ecp_runtime/cli.py

@@ -0,0 +1,141 @@
+import logging
+import typer
+import sys
+import os
+import json
+from pathlib import Path
+from typing import Dict, Any, Optional
+
+from .reporter import HTMLReporter
+
+# Import local modules (Using relative imports)
+try:
+    from .manifest import ECPManifest
+    from .runner import ECPRunner
+except ImportError:
+    # Fallback for direct execution debugging
+    sys.path.append(os.path.dirname(__file__))
+    from manifest import ECPManifest
+    from runner import ECPRunner
+
+app = typer.Typer(
+    name="ecp",
+    help="Evaluation Context Protocol Runtime CLI",
+    add_completion=False
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _configure_logging(verbose: bool) -> None:
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(
+        level=level,
+        format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+        datefmt="%H:%M:%S",
+    )
+
+
+@app.callback()
+def main():
+    """
+    Official Runtime for the Evaluation Context Protocol (ECP).
+    """
+    pass
+
+
+@app.command()
+def run(
+    manifest: Path = typer.Option(
+        ...,
+        "--manifest", "-m",
+        exists=True,
+        file_okay=True,
+        dir_okay=False,
+        writable=False,
+        readable=True,
+        resolve_path=True,
+        help="Path to the test manifest YAML file"
+    ),
+
+    verbose: bool = False,
+    report: Optional[Path] = typer.Option(
+        None,
+        "--report",
+        help="Path to save an HTML report (e.g., output.html)",
+        resolve_path=True,
+    ),
+    json_out: Optional[Path] = typer.Option(
+        None,
+        "--json-out",
+        help="Path to save a JSON report (e.g., output.json)",
+        resolve_path=True,
+    ),
+    print_json: bool = typer.Option(
+        False,
+        "--json",
+        help="Print the JSON report to stdout",
+    ),
+    fail_on_error: bool = typer.Option(
+        True,
+        "--fail-on-error/--no-fail-on-error",
+        help="Exit non-zero if any checks fail (useful for CI)",
+    ),
+):
+    """
+    Execute an evaluation run based on a manifest file.
+    """
+    _configure_logging(verbose)
+
+    logger.info("ECP Runtime Initializing...")
+    logger.info("Loading manifest: %s", manifest)
+
+    try:
+        # Load the YAML
+        config = ECPManifest.from_yaml(str(manifest))
+
+        # Run the Tests
+        runner = ECPRunner(config)
+        result_summary = runner.run_scenarios()
+        total = int(result_summary.get("total", 0) or 0)
+        passed = int(result_summary.get("passed", 0) or 0)
+        failed = max(total - passed, 0)
+
+        report_payload: Dict[str, Any] = {
+            "manifest": str(manifest),
+            "passed": passed,
+            "total": total,
+            "failed": failed,
+            "scenarios": result_summary.get("scenarios", []),
+        }
+
+        if report:
+            logger.info("Generating HTML report: %s", report)
+            reporter = HTMLReporter()
+            # Feed scenarios directly to reporter
+            for scenario in result_summary.get("scenarios", []):
+                reporter.add_scenario(scenario.get("name"), scenario.get("steps", []))
+            reporter.save(str(report))
+            logger.info("Report saved to %s", report)
+
+        if json_out:
+            json_out.write_text(json.dumps(report_payload, indent=2), encoding="utf-8")
+            logger.info("JSON report saved to %s", json_out)
+
+        if print_json:
+            typer.echo(json.dumps(report_payload, indent=2))
+
+        if fail_on_error and failed > 0:
+            raise typer.Exit(code=2)
+
+    except typer.Exit:
+        raise
+    except Exception as e:
+        logger.error("CRITICAL ERROR: %s", e)
+        if verbose:
+            raise e
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    app()

ecp_runtime-0.2.1/src/ecp_runtime/graders.py

@@ -0,0 +1,168 @@
+import re
+import os
+from typing import Any, Dict, List, Tuple
+from .manifest import GraderConfig, StepConfig
+
+# Try importing OpenAI, but don't crash if it's missing (unless used)
+try:
+    from openai import OpenAI
+except ImportError:
+    OpenAI = None
+
+def check_text_match(grader: GraderConfig, text: str) -> Tuple[bool, str]:
+    """Handles simple string assertions."""
+    if not text:
+        return False, "Text was empty"
+
+    val = grader.value
+    if grader.condition in {"contains", "equals", "does_not_contain"} and val is None:
+        return False, "No 'value' provided for text_match condition"
+    if grader.condition == "regex" and not grader.pattern:
+        return False, "No 'pattern' provided for regex condition"
+    
+    if grader.condition == "contains":
+        return val in text, f"Expected to contain '{val}'"
+    elif grader.condition == "equals":
+        return val == text, f"Expected to equal '{val}'"
+    elif grader.condition == "does_not_contain":
+        return val not in text, f"Expected NOT to contain '{val}'"
+    elif grader.condition == "regex":
+        return bool(re.search(grader.pattern, text)), f"Regex '{grader.pattern}' failed"
+    return False, "Unknown condition"
+
+def check_llm_judge(grader: GraderConfig, text: str) -> Tuple[bool, str, float]:
+    """
+    Uses an LLM to evaluate the text.
+    Returns: (passed, reasoning, score)
+    """
+    if not grader.prompt:
+        return False, "No prompt provided for llm_judge", 0.0
+    if OpenAI is None:
+        return False, "OpenAI library not installed. Run 'pip install openai'", 0.0
+    
+    api_key = os.environ.get("OPENAI_API_KEY")
+    if not api_key:
+        return False, "OPENAI_API_KEY not set in environment", 0.0
+
+    client = OpenAI(api_key=api_key)
+    
+    # 1. Construct the Prompt for the Judge
+    system_prompt = "You are an impartial AI Judge. You evaluate outputs based on specific criteria."
+    user_prompt = f"""
+    TASK: Evaluate the following text against the provided criteria.
+    
+    [TEXT TO EVALUATE]
+    {text}
+    
+    [CRITERIA]
+    {grader.prompt}
+    
+    [ASSERTION]
+    Does the text satisfy the criteria? 
+    If YES, end your response with "RESULT: PASS".
+    If NO, end your response with "RESULT: FAIL".
+    Provide a short reasoning before the result.
+    """
+
+    # 2. Call the Judge (using a cheap, smart model)
+    try:
+        response = client.chat.completions.create(
+            model="gpt-4o-mini", # Fast, cheap, smart enough for grading
+            messages=[
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt}
+            ],
+            temperature=0.0
+        )
+        content = response.choices[0].message.content
+        
+        # 3. Parse the Verdict
+        passed = "RESULT: PASS" in content
+        reasoning = content.replace("RESULT: PASS", "").replace("RESULT: FAIL", "").strip()
+        
+        return passed, reasoning, 1.0 if passed else 0.0
+
+    except Exception as e:
+        return False, f"LLM Judge Error: {str(e)}", 0.0
+
+def evaluate_step(step_config: StepConfig, result_obj: Any) -> List[Dict[str, Any]]:
+    """
+    Runs all graders for a single step against the agent's result.
+    """
+    check_results = []
+    
+    for grader in step_config.graders:
+        # Select target field
+        target_text = ""
+        if grader.field == "public_output":
+            target_text = result_obj.public_output
+        elif grader.field == "private_thought":
+            target_text = result_obj.private_thought
+            
+        passed = False
+        reasoning = ""
+        score = 0.0
+        
+        # Router
+        if grader.type == "text_match":
+            passed, reasoning = check_text_match(grader, target_text or "")
+            score = 1.0 if passed else 0.0
+            
+        elif grader.type == "llm_judge":
+            passed, reasoning, score = check_llm_judge(grader, target_text or "")
+        elif grader.type == "tool_usage":
+            passed, reasoning = check_tool_usage(grader, getattr(result_obj, "tool_calls", None) or [])
+            score = 1.0 if passed else 0.0
+        
+        check_results.append({
+            "type": grader.type,
+            "field": grader.field,
+            "passed": passed,
+            "score": score,
+            "reasoning": reasoning
+        })
+        
+    return check_results
+
+def check_tool_usage(grader: GraderConfig, tool_calls: List[Dict[str, Any]]) -> Tuple[bool, str]:
+    """Checks the JSON-RPC result.tool_calls list for a matching tool invocation.
+
+    Expectations come from the manifest:
+    - grader.tool_name: required exact match against call name (if provided)
+    - grader.arguments: all provided key/value pairs must be present in call arguments
+    """
+    if not isinstance(tool_calls, list) or not tool_calls:
+        return False, "No tool_calls present. Agent did not report any tool usage."
+
+    expected_name = grader.tool_name
+    expected_args = grader.arguments or {}
+
+    def args_match(actual: Dict[str, Any], expected: Dict[str, Any]) -> bool:
+        for k, v in expected.items():
+            if k not in actual:
+                return False
+            if actual[k] != v:
+                return False
+        return True
+
+    for call in tool_calls:
+        name = call.get("name") or call.get("tool") or call.get("id")
+        args = call.get("arguments") or call.get("args") or {}
+
+        # Name must match if specified
+        if expected_name and name != expected_name:
+            continue
+
+        # Arguments must include expected pairs
+        if args_match(args if isinstance(args, dict) else {}, expected_args):
+            return True, f"Found tool call '{name}' with expected arguments"
+
+    available = [call.get("name") or call.get("tool") or call.get("id") for call in tool_calls]
+    reason = "No matching tool call"
+    if expected_name:
+        reason += f" for name='{expected_name}'"
+    if expected_args:
+        reason += f" and arguments={expected_args}"
+    if available:
+        reason += f". Calls seen: {available}"
+    return False, reason

ecp_runtime-0.2.1/src/ecp_runtime/manifest.py

@@ -0,0 +1,41 @@
+from typing import List, Optional, Dict, Any, Union
+from pydantic import BaseModel, Field
+import yaml
+
+# --- The Grader (Assertion) Schema ---
+class GraderConfig(BaseModel):
+    type: str 
+    field: str = "public_output"
+    # For text_match
+    condition: Optional[str] = None 
+    value: Optional[str] = None
+    pattern: Optional[str] = None
+    # For llm_judge
+    prompt: Optional[str] = None
+    assertion: Optional[str] = None
+    # For tool_usage
+    tool_name: Optional[str] = None
+    arguments: Dict[str, Any] = Field(default_factory=dict)
+
+# --- The Step (Scenario) Schema ---
+class StepConfig(BaseModel):
+    input: str
+    constraints: Dict[str, Any] = Field(default_factory=dict)
+    graders: List[GraderConfig] = Field(default_factory=list)
+
+class ScenarioConfig(BaseModel):
+    name: str
+    steps: List[StepConfig]
+
+# --- The Root Manifest Schema ---
+class ECPManifest(BaseModel):
+    manifest_version: str = "v1"
+    name: str
+    target: str 
+    scenarios: List[ScenarioConfig]
+
+    @classmethod
+    def from_yaml(cls, path: str) -> "ECPManifest":
+        with open(path, "r") as f:
+            data = yaml.safe_load(f)
+        return cls(**data)

ecp_runtime-0.2.1/src/ecp_runtime/reporter.py

@@ -0,0 +1,70 @@
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import List, Dict, Any
+from jinja2 import Template
+
+HTML_TEMPLATE = """
+<!DOCTYPE html>
+<html>
+<head>
+    <title>ECP Evaluation Report</title>
+    <style>
+        body { font-family: sans-serif; max-width: 800px; margin: 0 auto; padding: 20px; background: #f4f4f9; }
+        .card { background: white; padding: 20px; margin-bottom: 20px; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.1); }
+        .pass { color: green; font-weight: bold; }
+        .fail { color: red; font-weight: bold; }
+        h1 { border-bottom: 2px solid #ddd; padding-bottom: 10px; }
+        .meta { color: #666; font-size: 0.9em; }
+        pre { background: #eee; padding: 10px; border-radius: 4px; overflow-x: auto; }
+    </style>
+</head>
+<body>
+    <h1>ECP Evaluation Report</h1>
+    <div class="meta">Generated: {{ timestamp }}</div>
+    
+    {% for scenario in results %}
+    <div class="card">
+        <h2>Scenario: {{ scenario.name }}</h2>
+        {% for step in scenario.steps %}
+        <div style="border-top: 1px solid #eee; margin-top: 10px; padding-top: 10px;">
+            <p><strong>Input:</strong> {{ step.input }}</p>
+            <p><strong>Output:</strong> {{ step.output }}</p>
+            
+            <h4>Graders:</h4>
+            <ul>
+            {% for check in step.checks %}
+                <li>
+                    <span class="{{ 'pass' if check.passed else 'fail' }}">
+                        {{ '✅ PASS' if check.passed else '❌ FAIL' }}
+                    </span>
+                    {{ check.type }} 
+                    {% if check.reasoning %}
+                    <br><small>Reason: {{ check.reasoning }}</small>
+                    {% endif %}
+                </li>
+            {% endfor %}
+            </ul>
+        </div>
+        {% endfor %}
+    </div>
+    {% endfor %}
+</body>
+</html>
+"""
+
+class HTMLReporter:
+    def __init__(self):
+        self.results = []
+
+    def add_scenario(self, name: str, steps: List[Dict[str, Any]]):
+        self.results.append({"name": name, "steps": steps})
+
+    def save(self, filepath: str):
+        template = Template(HTML_TEMPLATE)
+        html_content = template.render(
+            results=self.results,
+            timestamp=datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        )
+        Path(filepath).write_text(html_content, encoding="utf-8")
+        return filepath

ecp_runtime-0.2.1/src/ecp_runtime/runner.py

@@ -0,0 +1,214 @@
+"""
+Docstring for runtime.python.src.ecp_runtime.runner
+
+Simplified Version. V0.1
+AsyncIO Pending
+"""
+
+import subprocess
+import json
+import os
+import time
+import logging
+import threading
+import queue
+from typing import Dict, Any, Optional, List
+from dataclasses import dataclass
+from .graders import evaluate_step
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class StepResult:
+    status: str
+    public_output: Optional[str] = None
+    private_thought: Optional[str] = None
+    logs: Optional[str] = None
+    tool_calls: Optional[List[Dict[str, Any]]] = None
+
+
+class AgentProcess:
+    """Manages the lifecycle of the Agent Child Process."""
+
+    def __init__(self, command: str, rpc_timeout: float = 30.0):
+        self.command = command
+        self.rpc_timeout = rpc_timeout
+        self.process = None
+
+    def start(self):
+        # Launch the agent and connect pipes to stdio
+        self.process = subprocess.Popen(
+            self.command,
+            shell=True,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=1 # Line buffered
+        )
+
+    def stop(self):
+        if self.process:
+            self.process.terminate()
+
+    def send_rpc(self, method: str, params: Dict[str, Any] = None) -> Dict[str, Any]:
+        """Sends a JSON-RPC request and waits for the response."""
+        if not params:
+            params = {}
+
+        request = {
+            "jsonrpc": "2.0",
+            "method": method,
+            "params": params,
+            "id": int(time.time() * 1000)
+        }
+
+        # Write to Agent's STDIN
+        json_str = json.dumps(request)
+        self.process.stdin.write(json_str + "\n")
+        self.process.stdin.flush()
+
+        return self._read_json_response()
+
+    def _read_json_response(self) -> Dict[str, Any]:
+        start_time = time.time()
+        last_non_json = None
+
+        while True:
+            elapsed = time.time() - start_time
+            remaining = max(self.rpc_timeout - elapsed, 0)
+            if remaining <= 0:
+                stderr = self._safe_read_stderr()
+                raise RuntimeError(
+                    f"Agent response timed out after {self.rpc_timeout:.1f}s. "
+                    f"Last non-JSON line: {last_non_json}. Stderr: {stderr}"
+                )
+
+            response_line = self._readline_with_timeout(remaining)
+            if response_line is None:
+                stderr = self._safe_read_stderr()
+                raise RuntimeError(
+                    f"Agent response timed out after {self.rpc_timeout:.1f}s. "
+                    f"Last non-JSON line: {last_non_json}. Stderr: {stderr}"
+                )
+
+            if response_line == "":
+                stderr = self._safe_read_stderr()
+                raise RuntimeError(f"Agent crashed or closed connection. Stderr: {stderr}")
+
+            line = response_line.strip()
+            if not line:
+                continue
+
+            try:
+                return json.loads(line)
+            except json.JSONDecodeError:
+                last_non_json = line
+                logger.warning("Agent emitted non-JSON stdout: %s", line)
+                continue
+
+    def _readline_with_timeout(self, timeout: float) -> Optional[str]:
+        if not self.process or not self.process.stdout:
+            return None
+
+        q: queue.Queue = queue.Queue(maxsize=1)
+
+        def _reader():
+            try:
+                q.put(self.process.stdout.readline())
+            except Exception:
+                q.put("")
+
+        t = threading.Thread(target=_reader, daemon=True)
+        t.start()
+        try:
+            return q.get(timeout=timeout)
+        except queue.Empty:
+            return None
+
+    def _safe_read_stderr(self) -> str:
+        if not self.process or not self.process.stderr:
+            return ""
+        if self.process.poll() is None:
+            return ""
+        try:
+            return self.process.stderr.read()
+        except Exception:
+            return ""
+
+
+class ECPRunner:
+    """The Orchestrator."""
+
+    def __init__(self, manifest):
+        self.manifest = manifest
+
+    def run_scenarios(self):
+        total_passed = 0
+        total_checks = 0
+        report_data: List[Dict[str, Any]] = []
+
+        for scenario in self.manifest.scenarios:
+            logger.info("Scenario: %s", scenario.name)
+
+            rpc_timeout = float(os.environ.get("ECP_RPC_TIMEOUT", "30"))
+            agent = AgentProcess(self.manifest.target, rpc_timeout=rpc_timeout)
+            agent.start()
+
+            try:
+                agent.send_rpc("agent/initialize", {"config": {}})
+                scenario_steps: List[Dict[str, Any]] = []
+
+                for i, step in enumerate(scenario.steps):
+                    # Execute
+                    rpc_resp = agent.send_rpc("agent/step", {"input": step.input})
+                    result_data = rpc_resp.get("result", {})
+
+                    # Map to internal object
+                    step_result = StepResult(
+                        status=result_data.get("status", "done"),
+                        public_output=result_data.get("public_output"),
+                        private_thought=result_data.get("private_thought"),
+                        tool_calls=result_data.get("tool_calls") if isinstance(result_data.get("tool_calls"), list) else None
+                    )
+
+                    logger.info("Step %d: Input='%s'", i + 1, step.input)
+                    logger.info("Output: %s", step_result.public_output)
+                    if step_result.private_thought:
+                        logger.debug("Thought: %s", step_result.private_thought)
+
+                    checks = evaluate_step(step, step_result)
+
+                    for check in checks:
+                        total_checks += 1
+                        status = "PASS" if check["passed"] else "FAIL"
+                        logger.info("%s | %s on %s", status, check["type"], check["field"])
+
+                        if check["type"] == "llm_judge" or not check["passed"]:
+                            logger.info("Reason: %s", check["reasoning"])
+
+                        if check["passed"]:
+                            total_passed += 1
+
+                    # Collect for HTML report
+                    scenario_steps.append({
+                        "input": step.input,
+                        "output": step_result.public_output,
+                        "checks": checks
+                    })
+
+            finally:
+                agent.stop()
+
+            # Append scenario block
+            report_data.append({"name": scenario.name, "steps": scenario_steps})
+
+        logger.info("Run Complete. Passed: %d/%d", total_passed, total_checks)
+
+        # Return structured report data
+        return {
+            "passed": total_passed,
+            "total": total_checks,
+            "scenarios": report_data
+        }