eval-protocol 0.3.5.dev3__tar.gz → 0.3.7__tar.gz

{eval_protocol-0.3.5.dev3/eval_protocol.egg-info → eval_protocol-0.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.5.dev3
+Version: 0.3.7
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.3.5.dev3 → eval_protocol-0.3.7}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2026-01-06T11:34:06-0800",
+ "date": "2026-01-07T01:14:04-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "5a26ef701c03cf6d62d677188dde481a25725a9a",
- "version": "0.3.5.dev.3"
+ "full-revisionid": "77842b531fff8ab0b70d51205a270f326410ba6e",
+ "version": "0.3.7"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.3.5.dev3 → eval_protocol-0.3.7}/eval_protocol/proxy/proxy_core/app.py

@@ -4,7 +4,7 @@ A FastAPI service that sits in front of LiteLLM and extracts metadata from URL p
 """
 
 from fastapi import FastAPI, Depends, Request, Query
-from typing import Optional, List
+from typing import Optional, List, Callable
 import os
 import redis
 import logging
@@ -105,6 +105,7 @@ def create_app(
     auth_provider: AuthProvider = NoAuthProvider(),
     preprocess_chat_request: Optional[ChatRequestHook] = None,
     preprocess_traces_request: Optional[TracesRequestHook] = None,
+    extra_routes: Optional[Callable[[FastAPI], None]] = None,
 ) -> FastAPI:
     @asynccontextmanager
     async def lifespan(app: FastAPI):
@@ -288,6 +289,9 @@ def create_app(
             params=params,
         )
 
+    if extra_routes is not None:
+        extra_routes(app)
+
     # Health
     @app.get("/health")
     async def health():

{eval_protocol-0.3.5.dev3 → eval_protocol-0.3.7}/eval_protocol/proxy/proxy_core/redis_utils.py

@@ -8,8 +8,12 @@ import redis
 
 logger = logging.getLogger(__name__)
 
+DEFAULT_ROLLOUT_TTL_SECONDS = 60 * 60 * 24
 
-def register_insertion_id(redis_client: redis.Redis, rollout_id: str, insertion_id: str) -> bool:
+
+def register_insertion_id(
+    redis_client: redis.Redis, rollout_id: str, insertion_id: str, ttl_seconds: int = DEFAULT_ROLLOUT_TTL_SECONDS
+) -> bool:
     """Register an insertion_id for a rollout_id in Redis.
 
     Tracks all expected completion insertion_ids for this rollout.
@@ -22,7 +26,10 @@ def register_insertion_id(redis_client: redis.Redis, rollout_id: str, insertion_
         True if successful, False otherwise
     """
     try:
-        redis_client.sadd(rollout_id, insertion_id)
+        pipe = redis_client.pipeline()
+        pipe.sadd(rollout_id, insertion_id)
+        pipe.expire(rollout_id, int(ttl_seconds))
+        pipe.execute()
         logger.info(f"Registered insertion_id {insertion_id} for rollout {rollout_id}")
         return True
     except Exception as e:

{eval_protocol-0.3.5.dev3 → eval_protocol-0.3.7}/eval_protocol/pytest/github_action_rollout_processor.py

@@ -1,13 +1,12 @@
 import asyncio
 import os
 import time
-from typing import Any, Callable, Dict, List, Optional
+from typing import Any, Dict, List, Optional
 import json
 import requests
 from datetime import datetime, timezone, timedelta
 from eval_protocol.models import EvaluationRow, Status
 from eval_protocol.data_loader.dynamic_data_loader import DynamicDataLoader
-from eval_protocol.types.remote_rollout_processor import DataLoaderConfig
 
 from .rollout_processor import RolloutProcessor
 from .types import RolloutProcessorConfig
@@ -21,7 +20,7 @@ class GithubActionRolloutProcessor(RolloutProcessor):
     Expected GitHub Actions workflow:
     - Workflow dispatch with inputs: completion_params, metadata (JSON), model_base_url, api_key
     - Workflow makes API calls that get traced (e.g., via Fireworks tracing proxy)
-    - Traces are fetched later via output_data_loader using rollout_id tags
+    - Traces are fetched later via Fireworks tracing proxy using rollout_id tags
 
     NOTE: GHA has a rate limit of 5000 requests per hour.
     """
@@ -38,7 +37,6 @@ class GithubActionRolloutProcessor(RolloutProcessor):
         timeout_seconds: float = 1800.0,
         max_find_workflow_retries: int = 5,
         github_token: Optional[str] = None,
-        output_data_loader: Optional[Callable[[DataLoaderConfig], DynamicDataLoader]] = None,
     ):
         self.owner = owner
         self.repo = repo
@@ -52,7 +50,6 @@ class GithubActionRolloutProcessor(RolloutProcessor):
         self.timeout_seconds = timeout_seconds
         self.max_find_workflow_retries = max_find_workflow_retries
         self.github_token = github_token
-        self._output_data_loader = output_data_loader or default_fireworks_output_data_loader
 
     def _headers(self) -> Dict[str, str]:
         headers = {"Accept": "application/vnd.github+json"}
@@ -200,7 +197,7 @@ class GithubActionRolloutProcessor(RolloutProcessor):
             row.execution_metadata.rollout_duration_seconds = time.perf_counter() - start_time
 
             def _update_with_trace() -> None:
-                return update_row_with_remote_trace(row, self._output_data_loader, self.model_base_url)
+                return update_row_with_remote_trace(row, default_fireworks_output_data_loader, self.model_base_url)
 
             await asyncio.to_thread(_update_with_trace)
 

{eval_protocol-0.3.5.dev3 → eval_protocol-0.3.7}/eval_protocol/pytest/remote_rollout_processor.py

@@ -1,6 +1,6 @@
 import asyncio
 import time
-from typing import Any, Dict, List, Optional, Callable
+from typing import Any, Dict, List, Optional
 
 import requests
 
@@ -26,8 +26,7 @@ class RemoteRolloutProcessor(RolloutProcessor):
     """
     Rollout processor that triggers a remote HTTP server to perform the rollout.
 
-    By default, fetches traces from the Fireworks tracing proxy using rollout_id tags.
-    You can provide a custom output_data_loader for different tracing backends.
+    Fetches traces from the Fireworks tracing proxy using rollout_id tags.
 
     See https://evalprotocol.io/tutorial/remote-rollout-processor for documentation.
     """
@@ -39,7 +38,6 @@ class RemoteRolloutProcessor(RolloutProcessor):
         model_base_url: str = "https://tracing.fireworks.ai",
         poll_interval: float = 1.0,
         timeout_seconds: float = 120.0,
-        output_data_loader: Optional[Callable[[DataLoaderConfig], DynamicDataLoader]] = None,
     ):
         # Prefer constructor-provided configuration. These can be overridden via
         # config.kwargs at call time for backward compatibility.
@@ -52,7 +50,6 @@ class RemoteRolloutProcessor(RolloutProcessor):
             self._model_base_url = _ep_model_base_url
         self._poll_interval = poll_interval
         self._timeout_seconds = timeout_seconds
-        self._output_data_loader = output_data_loader or default_fireworks_output_data_loader
         self._tracing_adapter = FireworksTracingAdapter(base_url=self._model_base_url)
 
     def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) -> List[asyncio.Task[EvaluationRow]]:
@@ -188,7 +185,7 @@ class RemoteRolloutProcessor(RolloutProcessor):
             row.execution_metadata.rollout_duration_seconds = time.perf_counter() - start_time
 
             def _update_with_trace() -> None:
-                return update_row_with_remote_trace(row, self._output_data_loader, model_base_url)
+                return update_row_with_remote_trace(row, default_fireworks_output_data_loader, model_base_url)
 
             await asyncio.to_thread(_update_with_trace)  # Update row with remote trace in-place
             return row

{eval_protocol-0.3.5.dev3 → eval_protocol-0.3.7/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.5.dev3
+Version: 0.3.7
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT