eval-protocol 0.3.2__tar.gz → 0.3.2.dev1__tar.gz

{eval_protocol-0.3.2/eval_protocol.egg-info → eval_protocol-0.3.2.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.2
+Version: 0.3.2.dev1
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT
@@ -103,6 +103,8 @@ Provides-Extra: openenv
 Requires-Dist: openenv-core; extra == "openenv"
 Provides-Extra: dspy
 Requires-Dist: dspy>=3.0.0; extra == "dspy"
+Provides-Extra: klavis
+Requires-Dist: klavis>=2.18.0; extra == "klavis"
 Provides-Extra: langgraph
 Requires-Dist: langgraph>=0.6.7; extra == "langgraph"
 Requires-Dist: langchain-core>=0.3.75; extra == "langgraph"

{eval_protocol-0.3.2 → eval_protocol-0.3.2.dev1}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-12-23T14:49:24-0800",
+ "date": "2026-01-03T20:47:59-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "bd1be9570f0193a6d8277e62c9dfa3a0d9950652",
- "version": "0.3.2"
+ "full-revisionid": "aee419378ff01cfd8ff38b185ff5527a6550a83d",
+ "version": "0.3.2.dev.1"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.3.2 → eval_protocol-0.3.2.dev1}/eval_protocol/adapters/fireworks_tracing.py

@@ -264,6 +264,7 @@ class FireworksTracingAdapter(BaseAdapter):
         self.project_id = project_id
         self.base_url = base_url.rstrip("/")
         self.timeout = timeout
+        self._session = requests.Session()
 
     def search_logs(self, tags: List[str], limit: int = 100, hours_back: int = 24) -> List[Dict[str, Any]]:
         """Fetch logs from Fireworks tracing gateway /logs endpoint.
@@ -287,14 +288,14 @@ class FireworksTracingAdapter(BaseAdapter):
         last_error: Optional[str] = None
         for url in urls_to_try:
             try:
-                response = requests.get(url, params=params, timeout=self.timeout, headers=headers)
-                if response.status_code == 404:
-                    # Try next variant
-                    last_error = f"404 for {url}"
-                    continue
-                response.raise_for_status()
-                data = response.json() or {}
-                break
+                with self._session.get(url, params=params, timeout=self.timeout, headers=headers) as response:
+                    if response.status_code == 404:
+                        # Try next variant (must close response to release connection)
+                        last_error = f"404 for {url}"
+                        continue
+                    response.raise_for_status()
+                    data = response.json() or {}
+                    break
             except requests.exceptions.RequestException as e:
                 last_error = str(e)
                 continue
@@ -412,22 +413,20 @@ class FireworksTracingAdapter(BaseAdapter):
 
         result = None
         try:
-            response = requests.get(url, params=params, timeout=self.timeout, headers=headers)
-            response.raise_for_status()
-            result = response.json()
-        except requests.exceptions.HTTPError as e:
-            error_msg = str(e)
-
-            # Try to extract detail message from response
-            if e.response is not None:
-                try:
-                    error_detail = e.response.json().get("detail", {})
-                    error_msg = error_detail or e.response.text
-                except Exception:  # In case e.response.json() fails
-                    error_msg = f"Proxy error: {e.response.text}"
-
-            logger.error("Failed to fetch traces from proxy (HTTP %s): %s", e.response.status_code, error_msg)
-            return eval_rows
+            with self._session.get(url, params=params, timeout=self.timeout, headers=headers) as response:
+                if response.status_code >= 400:
+                    error_msg: str = response.text
+                    try:
+                        payload = response.json()
+                        if isinstance(payload, dict) and "detail" in payload:
+                            detail = payload.get("detail")
+                            if detail:
+                                error_msg = str(detail)
+                    except Exception:
+                        pass
+                    logger.error("Failed to fetch traces from proxy (HTTP %s): %s", response.status_code, error_msg)
+                    return eval_rows
+                result = response.json()
         except requests.exceptions.RequestException as e:
             # Non-HTTP errors (network issues, timeouts, etc.)
             logger.error("Failed to fetch traces from proxy: %s", str(e))
@@ -451,3 +450,10 @@ class FireworksTracingAdapter(BaseAdapter):
 
         logger.info("Successfully converted %d traces to evaluation rows", len(eval_rows))
         return eval_rows
+
+    def close(self) -> None:
+        """Close underlying HTTP resources."""
+        try:
+            self._session.close()
+        except Exception:
+            pass

{eval_protocol-0.3.2 → eval_protocol-0.3.2.dev1}/eval_protocol/pytest/__init__.py

@@ -11,6 +11,15 @@ from .rollout_processor import RolloutProcessor
 from .rollout_result_post_processor import RolloutResultPostProcessor, NoOpRolloutResultPostProcessor
 from .types import RolloutProcessorConfig
 
+# Conditional import for optional Klavis dependency
+try:
+    from .default_klavis_sandbox_rollout_processor import KlavisSandboxRolloutProcessor
+
+    KLAVIS_AVAILABLE = True
+except ImportError:
+    KLAVIS_AVAILABLE = False
+    KlavisSandboxRolloutProcessor = None
+
 # Conditional import for optional dependencies
 try:
     from .default_pydantic_ai_rollout_processor import PydanticAgentRolloutProcessor
@@ -47,6 +56,10 @@ __all__ = [
     "NoOpRolloutResultPostProcessor",
 ]
 
+# Only add to __all__ if available
+if KLAVIS_AVAILABLE:
+    __all__.append("KlavisSandboxRolloutProcessor")
+
 # Only add to __all__ if available
 if PYDANTIC_AI_AVAILABLE:
     __all__.append("PydanticAgentRolloutProcessor")

eval_protocol-0.3.2.dev1/eval_protocol/pytest/default_klavis_sandbox_rollout_processor.py

@@ -0,0 +1,174 @@
+import asyncio
+import json
+import logging
+import os
+import tempfile
+import time
+from typing import Any, Callable, Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+from eval_protocol.models import EvaluationRow
+from eval_protocol.pytest.rollout_processor import RolloutProcessor
+from eval_protocol.pytest.types import RolloutProcessorConfig
+
+from eval_protocol.pytest.default_agent_rollout_processor import Agent
+from klavis import Klavis
+from klavis.types import CreateSandboxResponse, SandboxMcpServer
+from openai.types import CompletionUsage
+
+logger = logging.getLogger(__name__)
+
+
+class KlavisSandboxRolloutProcessor(RolloutProcessor):
+    def __init__(
+        self,
+        server_name: str,
+        initialize_data_factory: Optional[Callable[[EvaluationRow], Dict[str, Any]]] = None,
+    ):
+        super().__init__()
+        self.server_name = server_name
+        self.initialize_data_factory = initialize_data_factory
+        self.klavis_client = Klavis(api_key=os.environ.get("KLAVIS_API_KEY"))
+        
+    def _init_sandbox(self) -> CreateSandboxResponse:
+        try:
+            server_name_enum = SandboxMcpServer(self.server_name)
+            return self.klavis_client.sandbox.create_sandbox(server_name=server_name_enum)
+        except Exception as e:
+            logger.error(f"Error creating sandbox: {str(e)}", exc_info=True)
+            raise
+    
+    @staticmethod
+    def create_mcp_config(server_url: str, server_key: str = "main", auth_token: str | None = None) -> str:
+        """Create a temporary MCP config file and return its path."""
+        config = {
+            "mcpServers": {
+                server_key: {
+                    "url": server_url,
+                    "transport": "streamable_http",
+                    **({"authorization": f"Bearer {auth_token}"} if auth_token else {})
+                }
+            }
+        }
+        
+        # Create a temp file that persists for the session
+        fd, path = tempfile.mkstemp(suffix=".json", prefix="mcp_config_")
+        with os.fdopen(fd, 'w') as f:
+            json.dump(config, f)
+        return path
+
+    def __call__(
+        self, rows: List[EvaluationRow], config: RolloutProcessorConfig
+    ) -> List[asyncio.Task[EvaluationRow]]:
+        """Process evaluation rows with Klavis sandbox lifecycle management"""
+        semaphore = config.semaphore
+
+        async def process_row(row: EvaluationRow) -> EvaluationRow:
+            """Process a single row with complete sandbox lifecycle"""
+            
+            start_time = time.perf_counter()
+            agent: Agent | None = None
+            temp_config_path: str | None = None
+            sandbox: CreateSandboxResponse | None = None
+
+            try:
+                # Step 0: Create a sandbox for this row
+                sandbox = self._init_sandbox()
+                logger.info(f"Sandbox created: {sandbox}")
+
+                # Step 1: Initialize data in the sandbox
+                init_data: Dict[str, Any] | None = None
+                if self.initialize_data_factory:
+                    init_data = self.initialize_data_factory(row)
+                else:
+                    # Allow datasets to provide initialization payload directly
+                    init_data = (
+                        (row.input_metadata.session_data or {}).get("initialize_data")
+                        if row.input_metadata is not None
+                        else None
+                    )
+                
+                if init_data:
+                    logger.info(f"Initializing {self.server_name} sandbox {sandbox.sandbox_id}")
+                    initialize_method = getattr(
+                        self.klavis_client.sandbox, f"initialize_{sandbox.server_name.value}_sandbox"
+                    )
+                    init_response = initialize_method(sandbox_id=sandbox.sandbox_id, **init_data)
+                    logger.info(f"Initialization response: {init_response}")
+                    
+                # Step 2: Create temporary MCP config with sandbox URL
+                temp_config_path = self.create_mcp_config(
+                    server_url=sandbox.server_url, server_key=sandbox.server_name.value
+                )
+                logger.info(f"MCP config created: {temp_config_path}")
+
+                # Step 3: Run agent with sandbox MCP server
+                logger.info(f"Running agent for row {row.execution_metadata.rollout_id} with {self.server_name} sandbox")
+                agent = Agent(
+                    model=row.input_metadata.completion_params["model"],
+                    row=row,
+                    config_path=temp_config_path,
+                    logger=config.logger,
+                )
+                await agent.setup()
+                await agent.call_agent()
+
+                # Update usage metadata
+                row.execution_metadata.usage = CompletionUsage(
+                    prompt_tokens=agent.usage.get("prompt_tokens", 0),
+                    completion_tokens=agent.usage.get("completion_tokens", 0),
+                    total_tokens=agent.usage.get("total_tokens", 0),
+                )
+                row = agent.evaluation_row
+                logger.info(f"Agent execution completed for row {row.execution_metadata.rollout_id}")
+
+                # Step 4: Export sandbox data
+                dump_method = getattr(self.klavis_client.sandbox, f"dump_{sandbox.server_name.value}_sandbox")
+                dump_response = dump_method(sandbox_id=sandbox.sandbox_id)
+                sandbox_data = dump_response.data
+                logger.info(f"Sandbox data: {sandbox_data}")
+
+                # Store sandbox data in row metadata for evaluation
+                if not row.execution_metadata.extra:
+                    row.execution_metadata.extra = {}
+                row.execution_metadata.extra["sandbox_data"] = sandbox_data
+                row.execution_metadata.extra["sandbox_id"] = sandbox.sandbox_id
+                row.execution_metadata.extra["server_name"] = self.server_name
+
+            except Exception as e:
+                logger.error(f"Error processing row {row.execution_metadata.rollout_id}: {str(e)}", exc_info=True)
+                if not row.execution_metadata.extra:
+                    row.execution_metadata.extra = {}
+                row.execution_metadata.extra["error"] = str(e)
+                raise
+
+            finally:
+                # Cleanup agent MCP client and temp config
+                if agent and agent.mcp_client:
+                    await agent.mcp_client.cleanup()
+                if temp_config_path and os.path.exists(temp_config_path):
+                    os.unlink(temp_config_path)
+                
+                # Release sandbox
+                if sandbox and sandbox.sandbox_id:
+                    try:
+                        self.klavis_client.sandbox.delete_sandbox(
+                            server_name=sandbox.server_name, sandbox_id=sandbox.sandbox_id
+                        )
+                        logger.info(f"Sandbox {sandbox.sandbox_id} released successfully")
+                    except Exception as e:
+                        logger.error(f"Error releasing sandbox {sandbox.sandbox_id}: {str(e)}", exc_info=True)
+
+                row.execution_metadata.rollout_duration_seconds = time.perf_counter() - start_time
+
+            return row
+
+        async def _sem_wrapper(r: EvaluationRow) -> EvaluationRow:
+            async with semaphore:
+                result = await process_row(r)
+                return result
+
+        # Create and return tasks
+        tasks = [asyncio.create_task(_sem_wrapper(row)) for row in rows]
+        return tasks

{eval_protocol-0.3.2 → eval_protocol-0.3.2.dev1}/eval_protocol/pytest/remote_rollout_processor.py

@@ -54,6 +54,7 @@ class RemoteRolloutProcessor(RolloutProcessor):
         self._timeout_seconds = timeout_seconds
         self._output_data_loader = output_data_loader or default_fireworks_output_data_loader
         self._tracing_adapter = FireworksTracingAdapter(base_url=self._model_base_url)
+        self._session = requests.Session()
 
     def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) -> List[asyncio.Task[EvaluationRow]]:
         tasks: List[asyncio.Task[EvaluationRow]] = []
@@ -94,8 +95,8 @@ class RemoteRolloutProcessor(RolloutProcessor):
             def _post_init() -> None:
                 url = f"{remote_base_url}/init"
                 try:
-                    r = requests.post(url, json=init_payload.model_dump(), timeout=300)
-                    r.raise_for_status()
+                    with self._session.post(url, json=init_payload.model_dump(), timeout=300) as r:
+                        r.raise_for_status()
                 except requests.exceptions.Timeout:
                     raise TimeoutError(
                         f"The /init endpoint tried {url} with {init_payload.model_dump()} but timed out after 300 seconds."
@@ -108,9 +109,9 @@ class RemoteRolloutProcessor(RolloutProcessor):
 
             def _get_status() -> Dict[str, Any]:
                 url = f"{remote_base_url}/status"
-                r = requests.get(url, params={"rollout_id": row.execution_metadata.rollout_id}, timeout=15)
-                r.raise_for_status()
-                return r.json()
+                with self._session.get(url, params={"rollout_id": row.execution_metadata.rollout_id}, timeout=15) as r:
+                    r.raise_for_status()
+                    return r.json()
 
             continue_polling_status = True
             while time.time() < deadline:
@@ -204,4 +205,12 @@ class RemoteRolloutProcessor(RolloutProcessor):
         return tasks
 
     def cleanup(self) -> None:
+        try:
+            self._tracing_adapter.close()
+        except Exception:
+            pass
+        try:
+            self._session.close()
+        except Exception:
+            pass
         return None

{eval_protocol-0.3.2 → eval_protocol-0.3.2.dev1/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.2
+Version: 0.3.2.dev1
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT
@@ -103,6 +103,8 @@ Provides-Extra: openenv
 Requires-Dist: openenv-core; extra == "openenv"
 Provides-Extra: dspy
 Requires-Dist: dspy>=3.0.0; extra == "dspy"
+Provides-Extra: klavis
+Requires-Dist: klavis>=2.18.0; extra == "klavis"
 Provides-Extra: langgraph
 Requires-Dist: langgraph>=0.6.7; extra == "langgraph"
 Requires-Dist: langchain-core>=0.3.75; extra == "langgraph"

{eval_protocol-0.3.2 → eval_protocol-0.3.2.dev1}/eval_protocol.egg-info/SOURCES.txt

@@ -193,6 +193,7 @@ eval_protocol/pytest/__init__.py
 eval_protocol/pytest/buffer.py
 eval_protocol/pytest/default_agent_rollout_processor.py
 eval_protocol/pytest/default_dataset_adapter.py
+eval_protocol/pytest/default_klavis_sandbox_rollout_processor.py
 eval_protocol/pytest/default_langchain_rollout_processor.py
 eval_protocol/pytest/default_mcp_gym_rollout_processor.py
 eval_protocol/pytest/default_no_op_rollout_processor.py

{eval_protocol-0.3.2 → eval_protocol-0.3.2.dev1}/eval_protocol.egg-info/requires.txt

@@ -77,6 +77,9 @@ dspy>=3.0.0
 datasets>=3.0.0
 transformers>=4.0.0
 
+[klavis]
+klavis>=2.18.0
+
 [langchain]
 langchain-core>=0.3.0
 

{eval_protocol-0.3.2 → eval_protocol-0.3.2.dev1}/pyproject.toml

@@ -134,6 +134,9 @@ openenv = [
 dspy = [
     "dspy>=3.0.0",
 ]
+klavis = [
+    "klavis>=2.18.0",
+]
 
 # Optional deps for LangGraph example/tests
 langgraph = [