ecp-runtime 0.2.9__tar.gz → 0.3.0__tar.gz

{ecp_runtime-0.2.9 → ecp_runtime-0.3.0}/.gitignore

@@ -31,7 +31,7 @@ MANIFEST
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec
-
+/specs/
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt
@@ -50,7 +50,10 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 cover/
-
+node_modules/
+.github/agents
+.github/prompts/
+.specify/
 # Translations
 *.mo
 *.pot

{ecp_runtime-0.2.9 → ecp_runtime-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ecp-runtime
-Version: 0.2.9
+Version: 0.3.0
 Summary: The reference runtime for the Evaluation Context Protocol (ECP).
 Project-URL: Homepage, https://github.com/evaluation-context-protocol/ecp
 Project-URL: Documentation, https://evaluationcontextprotocol.io/
@@ -43,6 +43,13 @@ You can also run via module entrypoint:
 python -m ecp_runtime.cli run --manifest .\examples\langchain_demo\manifest.yaml
 ```
 
+Manifest `target` values may be either a command for the default stdio transport
+or an ECP Streamable HTTP endpoint:
+
+```yaml
+target: "http://127.0.0.1:8765/ecp"
+```
+
 If your manifest includes `llm_judge`, set an API key and optional judge model:
 
 ```bash

{ecp_runtime-0.2.9 → ecp_runtime-0.3.0}/README.md

@@ -25,6 +25,13 @@ You can also run via module entrypoint:
 python -m ecp_runtime.cli run --manifest .\examples\langchain_demo\manifest.yaml
 ```
 
+Manifest `target` values may be either a command for the default stdio transport
+or an ECP Streamable HTTP endpoint:
+
+```yaml
+target: "http://127.0.0.1:8765/ecp"
+```
+
 If your manifest includes `llm_judge`, set an API key and optional judge model:
 
 ```bash

{ecp_runtime-0.2.9 → ecp_runtime-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "ecp-runtime"
-version = "0.2.9"
+version = "0.3.0"
 description = "The reference runtime for the Evaluation Context Protocol (ECP)."
 authors = [
   { name = "ECP Maintainers", email = "hello@ecp.org" },

{ecp_runtime-0.2.9 → ecp_runtime-0.3.0}/src/ecp_runtime/runner.py

@@ -14,6 +14,8 @@ import threading
 import time
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional
+from urllib import error, request
+from urllib.parse import urlparse
 
 from .graders import evaluate_step
 
@@ -148,6 +150,75 @@ class AgentProcess:
             return ""
 
 
+class HTTPAgentClient:
+    """JSON-RPC client for ECP Streamable HTTP endpoints."""
+
+    def __init__(self, endpoint: str, rpc_timeout: float = 30.0):
+        self.endpoint = endpoint
+        self.rpc_timeout = rpc_timeout
+
+    def start(self):
+        return None
+
+    def stop(self):
+        return None
+
+    def send_rpc(self, method: str, params: Dict[str, Any] = None) -> Dict[str, Any]:
+        if not params:
+            params = {}
+
+        payload = {
+            "jsonrpc": "2.0",
+            "method": method,
+            "params": params,
+            "id": int(time.time() * 1000),
+        }
+        body = json.dumps(payload).encode("utf-8")
+        req = request.Request(
+            self.endpoint,
+            data=body,
+            method="POST",
+            headers={
+                "Accept": "application/json, text/event-stream",
+                "Content-Type": "application/json",
+            },
+        )
+
+        try:
+            with request.urlopen(req, timeout=self.rpc_timeout) as resp:
+                content_type = resp.headers.get("Content-Type", "")
+                raw = resp.read().decode("utf-8")
+        except error.HTTPError as exc:
+            raw = exc.read().decode("utf-8", errors="replace")
+            raise RuntimeError(
+                f"HTTP RPC failed: status={exc.code}, body={raw or exc.reason}"
+            ) from exc
+        except error.URLError as exc:
+            raise RuntimeError(f"HTTP RPC failed: {exc.reason}") from exc
+
+        if "text/event-stream" in content_type:
+            return self._parse_sse_response(raw)
+        if not raw:
+            raise RuntimeError("HTTP RPC response was empty")
+        payload = json.loads(raw)
+        if not isinstance(payload, dict):
+            raise RuntimeError("HTTP RPC response must be a JSON object")
+        return payload
+
+    def _parse_sse_response(self, raw: str) -> Dict[str, Any]:
+        for event in raw.split("\n\n"):
+            data_lines = []
+            for line in event.splitlines():
+                if line.startswith("data:"):
+                    data_lines.append(line[5:].lstrip())
+            if not data_lines:
+                continue
+            payload = json.loads("\n".join(data_lines))
+            if isinstance(payload, dict) and ("result" in payload or "error" in payload):
+                return payload
+        raise RuntimeError("SSE stream ended without a JSON-RPC response")
+
+
 class ECPRunner:
     """The Orchestrator."""
 
@@ -163,7 +234,7 @@ class ECPRunner:
             logger.info("Scenario: %s", scenario.name)
 
             rpc_timeout = float(os.environ.get("ECP_RPC_TIMEOUT", "30"))
-            agent = AgentProcess(self.manifest.target, rpc_timeout=rpc_timeout)
+            agent = self._create_agent(self.manifest.target, rpc_timeout=rpc_timeout)
             agent.start()
 
             try:
@@ -225,6 +296,11 @@ class ECPRunner:
             "scenarios": report_data
         }
 
+    def _create_agent(self, target: str, rpc_timeout: float):
+        if _is_http_url(target):
+            return HTTPAgentClient(target, rpc_timeout=rpc_timeout)
+        return AgentProcess(target, rpc_timeout=rpc_timeout)
+
     def _ensure_rpc_success(
         self,
         rpc_resp: Dict[str, Any],
@@ -244,3 +320,8 @@ class ECPRunner:
         raise RuntimeError(
             f"RPC call failed ({method}) at {where}: code={code}, message={message}"
         )
+
+
+def _is_http_url(target: str) -> bool:
+    parsed = urlparse(target)
+    return parsed.scheme in {"http", "https"} and bool(parsed.netloc)

ecp_runtime-0.3.0/tests/test_example_integrations.py

@@ -0,0 +1,135 @@
+import os
+import socket
+import subprocess
+import sys
+import tempfile
+import textwrap
+import time
+import unittest
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+RUNTIME_SRC = Path(__file__).resolve().parents[1] / "src"
+
+
+class ExampleIntegrationTests(unittest.TestCase):
+    def _run_manifest(self, manifest_path: Path) -> subprocess.CompletedProcess[str]:
+        env = dict(os.environ)
+        existing = env.get("PYTHONPATH", "")
+        extra_paths = [str(RUNTIME_SRC)]
+        if existing:
+            extra_paths.append(existing)
+        env["PYTHONPATH"] = os.pathsep.join(extra_paths)
+
+        return subprocess.run(
+            [
+                sys.executable,
+                "-m",
+                "ecp_runtime.cli",
+                "run",
+                "--manifest",
+                str(manifest_path),
+                "--json",
+            ],
+            cwd=REPO_ROOT,
+            text=True,
+            capture_output=True,
+            env=env,
+            check=False,
+        )
+
+    def test_plain_python_demo_manifest_passes(self) -> None:
+        manifest = REPO_ROOT / "examples" / "plain_python_demo" / "manifest.yaml"
+        result = self._run_manifest(manifest)
+        self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr)
+        self.assertIn('"failed": 0', result.stdout)
+
+    def test_two_agent_demo_manifest_passes(self) -> None:
+        manifest = REPO_ROOT / "examples" / "two_agent_demo" / "manifest.yaml"
+        result = self._run_manifest(manifest)
+        self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr)
+        self.assertIn('"failed": 0', result.stdout)
+
+    def test_streamable_http_demo_manifest_passes(self) -> None:
+        port = self._free_port()
+        server_env = dict(os.environ)
+        sdk_src = REPO_ROOT / "sdk" / "python" / "src"
+        existing = server_env.get("PYTHONPATH", "")
+        extra_paths = [str(sdk_src)]
+        if existing:
+            extra_paths.append(existing)
+        server_env["PYTHONPATH"] = os.pathsep.join(extra_paths)
+        server_env["ECP_HTTP_PORT"] = str(port)
+
+        server = subprocess.Popen(
+            [sys.executable, str(REPO_ROOT / "examples" / "streamable_http_demo" / "agent.py")],
+            cwd=REPO_ROOT,
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=server_env,
+        )
+
+        try:
+            self._wait_for_port("127.0.0.1", port)
+            with tempfile.TemporaryDirectory() as tmpdir:
+                manifest_path = Path(tmpdir) / "streamable-http-manifest.yaml"
+                manifest_path.write_text(
+                    textwrap.dedent(
+                        f"""
+                        manifest_version: "v1"
+                        name: "Streamable HTTP Transport Validation"
+                        target: "http://127.0.0.1:{port}/ecp"
+
+                        scenarios:
+                          - name: "HTTP Echo"
+                            steps:
+                              - input: "echo: transport is online"
+                                graders:
+                                  - type: text_match
+                                    field: public_output
+                                    condition: contains
+                                    value: "transport is online"
+                                  - type: tool_usage
+                                    tool_name: "http_echo"
+                                    arguments:
+                                      message: "transport is online"
+                        """
+                    ).strip(),
+                    encoding="utf-8",
+                )
+                result = self._run_manifest(manifest_path)
+
+            self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr)
+            self.assertIn('"failed": 0', result.stdout)
+        finally:
+            server.terminate()
+            try:
+                server.communicate(timeout=2)
+            except subprocess.TimeoutExpired:
+                server.kill()
+                server.communicate()
+
+    def _free_port(self) -> int:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+            sock.bind(("127.0.0.1", 0))
+            return int(sock.getsockname()[1])
+
+    def _wait_for_port(self, host: str, port: int) -> None:
+        deadline = time.time() + 5
+        while time.time() < deadline:
+            if self._port_is_open(host, port):
+                return
+            time.sleep(0.05)
+        raise AssertionError(f"Timed out waiting for {host}:{port}")
+
+    def _port_is_open(self, host: str, port: int) -> bool:
+        try:
+            with socket.create_connection((host, port), timeout=0.2):
+                return True
+        except OSError:
+            return False
+
+
+if __name__ == "__main__":
+    unittest.main()

{ecp_runtime-0.2.9 → ecp_runtime-0.3.0}/tests/test_runner.py

@@ -9,7 +9,7 @@ if str(RUNTIME_SRC) not in sys.path:
     sys.path.insert(0, str(RUNTIME_SRC))
 
 from ecp_runtime.manifest import StepConfig
-from ecp_runtime.runner import ECPRunner
+from ecp_runtime.runner import ECPRunner, HTTPAgentClient
 
 
 class RunnerTests(unittest.TestCase):
@@ -68,6 +68,50 @@ class RunnerTests(unittest.TestCase):
         self.assertIn("step=1", msg)
         self.assertIn("boom", msg)
 
+    def test_runner_uses_http_client_for_url_target(self) -> None:
+        runner = ECPRunner(SimpleNamespace(target="http://127.0.0.1:8765/ecp", scenarios=[]))
+
+        agent = runner._create_agent(runner.manifest.target, rpc_timeout=12.0)
+
+        self.assertIsInstance(agent, HTTPAgentClient)
+        self.assertEqual(agent.endpoint, "http://127.0.0.1:8765/ecp")
+        self.assertEqual(agent.rpc_timeout, 12.0)
+
+    def test_http_agent_client_posts_json_rpc(self) -> None:
+        class FakeResponse:
+            headers = {"Content-Type": "application/json; charset=utf-8"}
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *_args):
+                return None
+
+            def read(self):
+                return b'{"jsonrpc":"2.0","id":1,"result":{"ok":true}}'
+
+        captured = {}
+
+        def fake_urlopen(req, timeout):
+            captured["url"] = req.full_url
+            captured["timeout"] = timeout
+            captured["body"] = req.data
+            captured["accept"] = req.headers.get("Accept")
+            return FakeResponse()
+
+        with mock.patch("ecp_runtime.runner.request.urlopen", fake_urlopen):
+            response = HTTPAgentClient("http://agent.test/ecp", rpc_timeout=3).send_rpc(
+                "agent/step", {"input": "hi"}
+            )
+
+        self.assertEqual(response["result"]["ok"], True)
+        self.assertEqual(captured["url"], "http://agent.test/ecp")
+        self.assertEqual(captured["timeout"], 3)
+        self.assertIn("application/json", captured["accept"])
+        body = captured["body"].decode("utf-8")
+        self.assertIn('"method": "agent/step"', body)
+        self.assertIn('"input": "hi"', body)
+
 
 if __name__ == "__main__":
     unittest.main()

ecp_runtime-0.2.9/tests/test_example_integrations.py

@@ -1,51 +0,0 @@
-import os
-import subprocess
-import sys
-import unittest
-from pathlib import Path
-
-REPO_ROOT = Path(__file__).resolve().parents[3]
-RUNTIME_SRC = Path(__file__).resolve().parents[1] / "src"
-
-
-class ExampleIntegrationTests(unittest.TestCase):
-    def _run_manifest(self, manifest_path: Path) -> subprocess.CompletedProcess[str]:
-        env = dict(os.environ)
-        existing = env.get("PYTHONPATH", "")
-        extra_paths = [str(RUNTIME_SRC)]
-        if existing:
-            extra_paths.append(existing)
-        env["PYTHONPATH"] = os.pathsep.join(extra_paths)
-
-        return subprocess.run(
-            [
-                sys.executable,
-                "-m",
-                "ecp_runtime.cli",
-                "run",
-                "--manifest",
-                str(manifest_path),
-                "--json",
-            ],
-            cwd=REPO_ROOT,
-            text=True,
-            capture_output=True,
-            env=env,
-            check=False,
-        )
-
-    def test_plain_python_demo_manifest_passes(self) -> None:
-        manifest = REPO_ROOT / "examples" / "plain_python_demo" / "manifest.yaml"
-        result = self._run_manifest(manifest)
-        self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr)
-        self.assertIn('"failed": 0', result.stdout)
-
-    def test_two_agent_demo_manifest_passes(self) -> None:
-        manifest = REPO_ROOT / "examples" / "two_agent_demo" / "manifest.yaml"
-        result = self._run_manifest(manifest)
-        self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr)
-        self.assertIn('"failed": 0', result.stdout)
-
-
-if __name__ == "__main__":
-    unittest.main()