eval-protocol 0.3.24__tar.gz → 0.3.26__tar.gz

{eval_protocol-0.3.24/eval_protocol.egg-info → eval_protocol-0.3.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.24
+Version: 0.3.26
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.3.24 → eval_protocol-0.3.26}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2026-03-10T03:58:26-0700",
+ "date": "2026-03-22T23:53:03-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "0f3c47172cc248d6a9ebbec809ee870f59b03698",
- "version": "0.3.24"
+ "full-revisionid": "b0cbc2584647484c26662944ba59bd137f42a33f",
+ "version": "0.3.26"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.3.24 → eval_protocol-0.3.26}/eval_protocol/pytest/evaluation_test.py

@@ -211,7 +211,7 @@ def evaluation_test(
     completion_params = parse_ep_completion_params_overwrite(completion_params)
     original_completion_params = completion_params
     passed_threshold = parse_ep_passed_threshold(passed_threshold)
-    data_loaders = parse_ep_dataloaders(data_loaders)
+    data_loaders = parse_ep_dataloaders(data_loaders, dataset_adapter=dataset_adapter)
     custom_invocation_id = os.environ.get("EP_INVOCATION_ID", None)
 
     # ignore other data input params when dataloader is provided
@@ -449,6 +449,8 @@ def evaluation_test(
                         finally:
                             if output_buffer:
                                 await output_buffer.close()
+                            await rollout_processor.acleanup()
+                            rollout_processor.cleanup()
 
                         for res in priority_results:
                             run_idx = (res.execution_metadata.extra or {}).get("run_index", 0)
@@ -697,15 +699,19 @@ def evaluation_test(
                         # Lazy import (cached after first import above)
                         from eval_protocol.pytest.default_mcp_gym_rollout_processor import MCPGymRolloutProcessor
 
-                        if isinstance(rollout_processor, MCPGymRolloutProcessor):
-                            # For MCPGymRolloutProcessor, create and execute tasks one at a time to avoid port conflicts
-                            for run_idx in range(num_runs):
-                                task = asyncio.create_task(execute_run(run_idx, config))
-                                await task
-                        else:
-                            # For other processors, create all tasks at once and run in parallel
-                            # Concurrency is now controlled by the shared semaphore in each rollout processor
-                            await run_tasks_with_run_progress(execute_run, num_runs, config)
+                        try:
+                            if isinstance(rollout_processor, MCPGymRolloutProcessor):
+                                # For MCPGymRolloutProcessor, create and execute tasks one at a time to avoid port conflicts
+                                for run_idx in range(num_runs):
+                                    task = asyncio.create_task(execute_run(run_idx, config))
+                                    await task
+                            else:
+                                # For other processors, create all tasks at once and run in parallel
+                                # Concurrency is now controlled by the shared semaphore in each rollout processor
+                                await run_tasks_with_run_progress(execute_run, num_runs, config)
+                        finally:
+                            await rollout_processor.acleanup()
+                            rollout_processor.cleanup()
 
                         experiment_duration_seconds = time.perf_counter() - experiment_start_time
 

{eval_protocol-0.3.24 → eval_protocol-0.3.26}/eval_protocol/pytest/evaluation_test_utils.py

@@ -21,6 +21,7 @@ from eval_protocol.models import (
     EvaluationThresholdDict,
     Status,
 )
+from eval_protocol.common_utils import load_jsonl
 from eval_protocol.data_loader import DynamicDataLoader
 from eval_protocol.data_loader.models import EvaluationDataLoader
 from eval_protocol.pytest.rollout_processor import RolloutProcessor
@@ -288,10 +289,21 @@ def _rows_from_jsonl(path: str) -> list[EvaluationRow]:
 
 def parse_ep_dataloaders(
     dataloaders: Sequence[EvaluationDataLoader] | EvaluationDataLoader | None,
+    *,
+    dataset_adapter: Callable[[list[dict[str, Any]]], list[EvaluationRow]] | None = None,
 ) -> Sequence[EvaluationDataLoader] | EvaluationDataLoader | None:
+    """When ``EP_JSONL_PATH`` is set, load JSONL as raw dicts and run ``dataset_adapter`` if provided.
+
+    Without ``dataset_adapter``, rows are built with ``EvaluationRow(**dict)`` (legacy behavior),
+    which skips custom label fields that adapters normally attach.
+    """
     try:
         load_from_jsonl_path = os.getenv("EP_JSONL_PATH")
         if load_from_jsonl_path:
+            if dataset_adapter is not None:
+                return DynamicDataLoader(
+                    generators=[lambda path=load_from_jsonl_path, da=dataset_adapter: da(load_jsonl(path))]
+                )
             return DynamicDataLoader(generators=[lambda path=load_from_jsonl_path: _rows_from_jsonl(path)])
     except Exception:
         pass
@@ -476,8 +488,12 @@ async def rollout_processor_with_retry(
                 yield result
 
     finally:
-        await rollout_processor.acleanup()
-        rollout_processor.cleanup()
+        # Cleanup is intentionally NOT called here. rollout_processor_with_retry
+        # is invoked per-run, but the processor (and its session) is shared
+        # across parallel runs.  Closing per-run would kill in-flight requests
+        # in other runs.  Cleanup is called once after all runs complete in
+        # evaluation_test.py.
+        pass
 
 
 def sanitize_filename(text: str) -> str:

{eval_protocol-0.3.24 → eval_protocol-0.3.26}/eval_protocol/pytest/exception_config.py

@@ -23,6 +23,7 @@ def get_default_retryable_exceptions() -> Set[Type[Exception]]:
         return _default_retryable_exceptions
 
     # Lazy imports (these are expensive)
+    import aiohttp
     import httpx
     import litellm
     import requests
@@ -32,6 +33,9 @@ def get_default_retryable_exceptions() -> Set[Type[Exception]]:
         ConnectionError,  # type: ignore[assignment]
         TimeoutError,  # type: ignore[assignment]
         OSError,  # type: ignore[assignment]  # Covers network-related OS errors
+        # aiohttp library exceptions
+        aiohttp.ClientConnectionError,
+        aiohttp.ServerDisconnectedError,
         # Requests library exceptions
         requests.exceptions.ConnectionError,
         requests.exceptions.Timeout,

{eval_protocol-0.3.24 → eval_protocol-0.3.26}/eval_protocol/pytest/remote_rollout_processor.py

@@ -104,6 +104,9 @@ class RemoteRolloutProcessor(RolloutProcessor):
             try:
                 session = self._get_or_create_session()
                 async with session.post(init_url, json=init_payload.model_dump(), timeout=timeout_init) as resp:
+                    if resp.status >= 500:
+                        body = await resp.text()
+                        raise ConnectionError(f"Remote /init returned server error (HTTP {resp.status}): {body}")
                     if resp.status >= 400:
                         body = await resp.text()
                         raise RuntimeError(f"Remote /init failed (HTTP {resp.status}): {body}")
@@ -215,8 +218,6 @@ class RemoteRolloutProcessor(RolloutProcessor):
                 loop = asyncio.get_running_loop()
                 loop.create_task(self._session.close())
             except RuntimeError:
-                # No running event loop - can't safely close the session.
-                # The session will be garbage collected eventually, but warn about it.
                 logger.warning(
                     "RemoteRolloutProcessor.cleanup() called outside of async context. "
                     "Session may not be properly closed. Use `await processor.acleanup()` when possible."

{eval_protocol-0.3.24 → eval_protocol-0.3.26/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.24
+Version: 0.3.26
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT