eval-protocol 0.2.28__tar.gz → 0.2.30__tar.gz

{eval_protocol-0.2.28/eval_protocol.egg-info → eval_protocol-0.2.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.28
+Version: 0.2.30
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.28 → eval_protocol-0.2.30}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-09-26T11:40:07-0700",
+ "date": "2025-09-26T14:10:14-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "f9c6f1be71edb167cdbb8fc1d57e71ec3df4fe18",
- "version": "0.2.28"
+ "full-revisionid": "626a125899fb42ed135a1f223b3e827f37e44ae0",
+ "version": "0.2.30"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.28 → eval_protocol-0.2.30}/eval_protocol/pytest/evaluation_test.py

@@ -189,6 +189,7 @@ def evaluation_test(
     completion_params = parse_ep_completion_params(completion_params)
     original_completion_params = completion_params
     passed_threshold = parse_ep_passed_threshold(passed_threshold)
+    custom_invocation_id = os.environ.get("EP_INVOCATION_ID", None)
 
     def decorator(
         test_func: TestFunction,
@@ -228,7 +229,10 @@ def evaluation_test(
         # Create wrapper function with exact signature that pytest expects
         def create_wrapper_with_signature() -> Callable[[], None]:
             # Create the function body that will be used
-            invocation_id = generate_id()
+            if custom_invocation_id:
+                invocation_id = custom_invocation_id
+            else:
+                invocation_id = generate_id()
 
             async def wrapper_body(**kwargs: Unpack[ParameterizedTestKwargs]) -> None:
                 # Store URL for viewing results (after all postprocessing is complete)

{eval_protocol-0.2.28 → eval_protocol-0.2.30}/eval_protocol/pytest/handle_persist_flow.py

@@ -7,6 +7,7 @@ import pathlib
 import re
 from typing import Any
 
+from eval_protocol.directory_utils import find_eval_protocol_dir
 from eval_protocol.models import EvaluationRow
 from eval_protocol.pytest.store_experiment_link import store_experiment_link
 import requests
@@ -25,7 +26,8 @@ def handle_persist_flow(all_results: list[list[EvaluationRow]], test_func_name:
                     if row.execution_metadata and row.execution_metadata.experiment_id:
                         experiments[row.execution_metadata.experiment_id].append(row)
 
-                exp_dir = pathlib.Path("experiment_results")
+                eval_protocol_dir = find_eval_protocol_dir()
+                exp_dir = pathlib.Path(eval_protocol_dir) / "experiment_results"
                 exp_dir.mkdir(parents=True, exist_ok=True)
 
                 # Create one JSONL file per experiment_id

{eval_protocol-0.2.28 → eval_protocol-0.2.30}/eval_protocol/pytest/remote_rollout_processor.py

@@ -6,6 +6,7 @@ import requests
 
 from eval_protocol.models import EvaluationRow, Status
 from eval_protocol.data_loader.dynamic_data_loader import DynamicDataLoader
+from eval_protocol.types.remote_rollout_processor import InitRequest, RolloutMetadata
 from .rollout_processor import RolloutProcessor
 from .types import RolloutProcessorConfig
 
@@ -14,25 +15,7 @@ class RemoteRolloutProcessor(RolloutProcessor):
     """
     Rollout processor that triggers a remote HTTP server to perform the rollout.
 
-    Expected remote API:
-    - POST {remote_base_url}/init
-      Body: {
-        "rollout_id": str,
-        "model": str,
-        "messages": list[dict],
-        "tools": list[dict] | null,
-        "metadata": {
-          "invocation_id": str,
-          "experiment_id": str,
-          "rollout_id": str,
-          "run_id": str | null,
-          "row_id": str | null
-        },
-      }
-      Returns: {"ok": true}
-
-    - GET {remote_base_url}/status?rollout_id=...
-      Returns: {"terminated": bool, "info": {...}?}
+    See https://evalprotocol.io/tutorial/remote-rollout-processor for documentation.
     """
 
     def __init__(
@@ -71,14 +54,25 @@ class RemoteRolloutProcessor(RolloutProcessor):
         async def _process_row(row: EvaluationRow) -> EvaluationRow:
             start_time = time.perf_counter()
 
+            if row.execution_metadata.invocation_id is None:
+                raise ValueError("Invocation ID is required in RemoteRolloutProcessor")
+            if row.execution_metadata.experiment_id is None:
+                raise ValueError("Experiment ID is required in RemoteRolloutProcessor")
+            if row.execution_metadata.rollout_id is None:
+                raise ValueError("Rollout ID is required in RemoteRolloutProcessor")
+            if row.execution_metadata.run_id is None:
+                raise ValueError("Run ID is required in RemoteRolloutProcessor")
+            if row.input_metadata.row_id is None:
+                raise ValueError("Row ID is required in RemoteRolloutProcessor")
+
             # Build request metadata and payload
-            meta: Dict[str, Any] = {
-                "invocation_id": row.execution_metadata.invocation_id,
-                "experiment_id": row.execution_metadata.experiment_id,
-                "rollout_id": row.execution_metadata.rollout_id,
-                "run_id": row.execution_metadata.run_id,
-                "row_id": row.input_metadata.row_id,
-            }
+            meta: RolloutMetadata = RolloutMetadata(
+                invocation_id=row.execution_metadata.invocation_id,
+                experiment_id=row.execution_metadata.experiment_id,
+                rollout_id=row.execution_metadata.rollout_id,
+                run_id=row.execution_metadata.run_id,
+                row_id=row.input_metadata.row_id,
+            )
 
             model: Optional[str] = None
             if row.input_metadata and row.input_metadata.completion_params:
@@ -110,18 +104,21 @@ class RemoteRolloutProcessor(RolloutProcessor):
                     }
                 clean_messages.append({k: v for k, v in md.items() if k in allowed_message_fields and v is not None})
 
-            init_payload: Dict[str, Any] = {
-                "rollout_id": row.execution_metadata.rollout_id,
-                "model": model,
-                "messages": clean_messages,
-                "tools": row.tools,
-                "metadata": meta,
-            }
+            if row.execution_metadata.rollout_id is None:
+                raise ValueError("Rollout ID is required in RemoteRolloutProcessor")
+
+            init_payload: InitRequest = InitRequest(
+                model=model,
+                messages=clean_messages,
+                tools=row.tools,
+                metadata=meta,
+                model_base_url=config.kwargs.get("model_base_url", None),
+            )
 
             # Fire-and-poll
             def _post_init() -> None:
                 url = f"{remote_base_url}/init"
-                r = requests.post(url, json=init_payload, timeout=30)
+                r = requests.post(url, json=init_payload.model_dump(), timeout=30)
                 r.raise_for_status()
 
             await asyncio.to_thread(_post_init)

{eval_protocol-0.2.28 → eval_protocol-0.2.30}/eval_protocol/types/remote_rollout_processor.py

@@ -20,10 +20,17 @@ class RolloutMetadata(BaseModel):
 class InitRequest(BaseModel):
     """Request model for POST /init endpoint."""
 
-    rollout_id: str
     model: str
     messages: List[Message] = Field(min_length=1)
     tools: Optional[List[Dict[str, Any]]] = None
+
+    model_base_url: Optional[str] = None
+    """
+    A Base URL that the remote server can use to make LLM calls. This is useful
+    to configure on the eval-protocol side for flexibility in
+    development/traning.
+    """
+
     metadata: RolloutMetadata
 
 

{eval_protocol-0.2.28 → eval_protocol-0.2.30/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.28
+Version: 0.2.30
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT