eval-protocol 0.2.43__tar.gz → 0.2.45__tar.gz

{eval_protocol-0.2.43/eval_protocol.egg-info → eval_protocol-0.2.45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.43
+Version: 0.2.45
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.43 → eval_protocol-0.2.45}/eval_protocol/__init__.py

@@ -34,6 +34,7 @@ from .pytest import evaluation_test, SingleTurnRolloutProcessor, RemoteRolloutPr
 from .pytest.parameterize import DefaultParameterIdGenerator
 from .log_utils.elasticsearch_direct_http_handler import ElasticsearchDirectHttpHandler
 from .log_utils.rollout_id_filter import RolloutIdFilter
+from .log_utils.util import setup_rollout_logging_for_elasticsearch_handler
 
 from .types.remote_rollout_processor import (
     InitRequest,
@@ -68,6 +69,7 @@ warnings.filterwarnings("default", category=DeprecationWarning, module="eval_pro
 __all__ = [
     "ElasticsearchDirectHttpHandler",
     "RolloutIdFilter",
+    "setup_rollout_logging_for_elasticsearch_handler",
     "DataLoaderConfig",
     "Status",
     "RemoteRolloutProcessor",

{eval_protocol-0.2.43 → eval_protocol-0.2.45}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-08T08:52:41-0700",
+ "date": "2025-10-08T14:59:37-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "535169e7193e6500d8d323e7dbc31c14dca98b96",
- "version": "0.2.43"
+ "full-revisionid": "b120611112b84df8476cefcc02660c542e61b2a9",
+ "version": "0.2.45"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.43 → eval_protocol-0.2.45}/eval_protocol/adapters/fireworks_tracing.py

@@ -281,9 +281,8 @@ class FireworksTracingAdapter(BaseAdapter):
         from_timestamp: Optional[datetime] = None,
         to_timestamp: Optional[datetime] = None,
         include_tool_calls: bool = True,
-        backend_sleep_between_gets: float = 0.1,
-        backend_max_retries: int = 3,
-        proxy_max_retries: int = 3,
+        sleep_between_gets: float = 0.1,
+        max_retries: int = 3,
         span_name: Optional[str] = None,
         converter: Optional[TraceDictConverter] = None,
     ) -> List[EvaluationRow]:
@@ -305,10 +304,8 @@ class FireworksTracingAdapter(BaseAdapter):
             from_timestamp: Explicit start time (ISO format)
             to_timestamp: Explicit end time (ISO format)
             include_tool_calls: Whether to include tool calling traces
-            backend_sleep_between_gets: Sleep time between backend trace fetches (passed to proxy)
-            backend_max_retries: Maximum retries for backend operations (passed to proxy)
-            proxy_max_retries: Maximum retries when proxy returns 404 (client-side retries with exponential backoff)
-            span_name: If provided, extract messages from generations within this named span
+            sleep_between_gets: Sleep time between polling attempts (default: 2.5s)
+            max_retries: Max retry attempts used by proxy (default: 3)
             converter: Optional custom converter implementing TraceDictConverter protocol.
                 If provided, this will be used instead of the default conversion logic.
 
@@ -318,9 +315,9 @@ class FireworksTracingAdapter(BaseAdapter):
         Raises:
             ValueError: If tags list is empty
         """
-        # Validate that tags are provided (security requirement)
+        # Validate that tags are provided
         if not tags or len(tags) == 0:
-            raise ValueError("At least one tag is required to fetch traces (security: prevents fetching all traces)")
+            raise ValueError("At least one tag is required to fetch traces")
 
         eval_rows = []
 
@@ -339,58 +336,40 @@ class FireworksTracingAdapter(BaseAdapter):
             "hours_back": hours_back,
             "from_timestamp": from_timestamp.isoformat() if from_timestamp else None,
             "to_timestamp": to_timestamp.isoformat() if to_timestamp else None,
-            "sleep_between_gets": backend_sleep_between_gets,
-            "max_retries": backend_max_retries,
+            "sleep_between_gets": sleep_between_gets,
+            "max_retries": max_retries,
         }
 
         # Remove None values
         params = {k: v for k, v in params.items() if v is not None}
 
-        # Make request to proxy with retry logic
+        # Make request to proxy
         if self.project_id:
             url = f"{self.base_url}/v1/project_id/{self.project_id}/traces"
         else:
             url = f"{self.base_url}/v1/traces"
 
-        # Retry loop for handling backend indexing delays (proxy returns 404)
         result = None
-        for attempt in range(proxy_max_retries):
-            try:
-                response = requests.get(url, params=params, timeout=self.timeout)
-                response.raise_for_status()
-                result = response.json()
-                break  # Success, exit retry loop
-            except requests.exceptions.HTTPError as e:
-                error_msg = str(e)
-                should_retry = False
-
-                # Try to extract detail message from response
-                if e.response is not None:
-                    try:
-                        error_detail = e.response.json().get("detail", "")
-                        error_msg = error_detail or e.response.text
-
-                        # Retry on 404 if it's due to incomplete/missing traces (backend still indexing)
-                        if e.response.status_code == 404:
-                            should_retry = True
-                    except Exception:
-                        error_msg = e.response.text
-
-                if should_retry and attempt < proxy_max_retries - 1:
-                    sleep_time = 2 ** (attempt + 1)
-                    logger.warning(error_msg)
-                    time.sleep(sleep_time)
-                else:
-                    # Final retry or non-retryable error
-                    logger.error("Failed to fetch traces from proxy: %s", error_msg)
-                    return eval_rows
-            except requests.exceptions.RequestException as e:
-                # Non-HTTP errors (network issues, timeouts, etc.)
-                logger.error("Failed to fetch traces from proxy: %s", str(e))
-                return eval_rows
-
-        if result is None:
-            logger.error("Failed to fetch traces after %d retries", proxy_max_retries)
+        try:
+            response = requests.get(url, params=params, timeout=self.timeout)
+            response.raise_for_status()
+            result = response.json()
+        except requests.exceptions.HTTPError as e:
+            error_msg = str(e)
+
+            # Try to extract detail message from response
+            if e.response is not None:
+                try:
+                    error_detail = e.response.json().get("detail", {})
+                    error_msg = error_detail or e.response.text
+                except Exception:  # In case e.response.json() fails
+                    error_msg = f"Proxy error: {e.response.text}"
+
+            logger.error("Failed to fetch traces from proxy: %s", error_msg)
+            return eval_rows
+        except requests.exceptions.RequestException as e:
+            # Non-HTTP errors (network issues, timeouts, etc.)
+            logger.error("Failed to fetch traces from proxy: %s", str(e))
             return eval_rows
 
         # Extract traces from response

{eval_protocol-0.2.43 → eval_protocol-0.2.45}/eval_protocol/cli.py

@@ -300,6 +300,7 @@ def parse_args(args=None):
     # Logs command
     logs_parser = subparsers.add_parser("logs", help="Serve logs with file watching and real-time updates")
     logs_parser.add_argument("--port", type=int, default=8000, help="Port to bind to (default: 8000)")
+    logs_parser.add_argument("--debug", action="store_true", help="Enable debug mode")
 
     # Upload command
     upload_parser = subparsers.add_parser(

{eval_protocol-0.2.43 → eval_protocol-0.2.45}/eval_protocol/cli_commands/logs.py

@@ -16,6 +16,7 @@ def logs_command(args):
     print(f"🌐 URL: http://localhost:{port}")
     print(f"🔌 WebSocket: ws://localhost:{port}/ws")
     print(f"👀 Watching paths: {['current directory']}")
+    print(f"🔍 Debug mode: {args.debug}")
     print("Press Ctrl+C to stop the server")
     print("-" * 50)
 
@@ -25,7 +26,7 @@ def logs_command(args):
     elasticsearch_config = ElasticsearchSetup().setup_elasticsearch()
 
     try:
-        serve_logs(port=args.port, elasticsearch_config=elasticsearch_config)
+        serve_logs(port=args.port, elasticsearch_config=elasticsearch_config, debug=args.debug)
         return 0
     except KeyboardInterrupt:
         print("\n🛑 Server stopped by user")

{eval_protocol-0.2.43 → eval_protocol-0.2.45}/eval_protocol/dataset_logger/sqlite_dataset_logger_adapter.py

@@ -23,12 +23,19 @@ class SqliteDatasetLoggerAdapter(DatasetLogger):
 
     def log(self, row: "EvaluationRow") -> None:
         data = row.model_dump(exclude_none=True, mode="json")
+        rollout_id = data.get("execution_metadata", {}).get("rollout_id", "unknown")
+        logger.debug(f"[EVENT_BUS_EMIT] Starting to log row with rollout_id: {rollout_id}")
+
         self._store.upsert_row(data=data)
+        logger.debug(f"[EVENT_BUS_EMIT] Successfully stored row in database for rollout_id: {rollout_id}")
+
         try:
+            logger.debug(f"[EVENT_BUS_EMIT] Emitting event '{LOG_EVENT_TYPE}' for rollout_id: {rollout_id}")
             event_bus.emit(LOG_EVENT_TYPE, EvaluationRow(**data))
+            logger.debug(f"[EVENT_BUS_EMIT] Successfully emitted event for rollout_id: {rollout_id}")
         except Exception as e:
             # Avoid breaking storage due to event emission issues
-            logger.error(f"Failed to emit row_upserted event: {e}")
+            logger.error(f"[EVENT_BUS_EMIT] Failed to emit row_upserted event for rollout_id {rollout_id}: {e}")
             pass
 
     def read(self, rollout_id: Optional[str] = None) -> List["EvaluationRow"]:

eval_protocol-0.2.45/eval_protocol/event_bus/sqlite_event_bus.py

@@ -0,0 +1,126 @@
+import asyncio
+import os
+import threading
+import time
+from typing import Any, Optional
+from uuid import uuid4
+
+from eval_protocol.event_bus.event_bus import EventBus
+from eval_protocol.event_bus.logger import logger
+from eval_protocol.event_bus.sqlite_event_bus_database import SqliteEventBusDatabase
+
+
+class SqliteEventBus(EventBus):
+    """SQLite-based event bus implementation that supports cross-process communication."""
+
+    def __init__(self, db_path: Optional[str] = None):
+        super().__init__()
+
+        # Use the same database as the evaluation row store
+        if db_path is None:
+            from eval_protocol.directory_utils import find_eval_protocol_dir
+
+            eval_protocol_dir = find_eval_protocol_dir()
+            db_path = os.path.join(eval_protocol_dir, "logs.db")
+
+        self._db: SqliteEventBusDatabase = SqliteEventBusDatabase(db_path)
+        self._running = False
+        self._process_id = str(os.getpid())
+
+    def emit(self, event_type: str, data: Any) -> None:
+        """Emit an event to all subscribers.
+
+        Args:
+            event_type: Type of event (e.g., "log")
+            data: Event data
+        """
+        logger.debug(f"[CROSS_PROCESS_EMIT] Emitting event type: {event_type}")
+
+        # Call local listeners immediately
+        logger.debug(f"[CROSS_PROCESS_EMIT] Calling {len(self._listeners)} local listeners")
+        super().emit(event_type, data)
+        logger.debug("[CROSS_PROCESS_EMIT] Completed local listener calls")
+
+        # Publish to cross-process subscribers
+        logger.debug("[CROSS_PROCESS_EMIT] Publishing to cross-process subscribers")
+        self._publish_cross_process(event_type, data)
+        logger.debug("[CROSS_PROCESS_EMIT] Completed cross-process publish")
+
+    def _publish_cross_process(self, event_type: str, data: Any) -> None:
+        """Publish event to cross-process subscribers via database."""
+        logger.debug(f"[CROSS_PROCESS_PUBLISH] Publishing event {event_type} to database")
+        try:
+            self._db.publish_event(event_type, data, self._process_id)
+            logger.debug(f"[CROSS_PROCESS_PUBLISH] Successfully published event {event_type} to database")
+        except Exception as e:
+            logger.error(f"[CROSS_PROCESS_PUBLISH] Failed to publish event {event_type} to database: {e}")
+
+    def start_listening(self) -> None:
+        """Start listening for cross-process events."""
+        if self._running:
+            logger.debug("[CROSS_PROCESS_LISTEN] Already listening, skipping start")
+            return
+
+        logger.debug("[CROSS_PROCESS_LISTEN] Starting cross-process event listening")
+        self._running = True
+        loop = asyncio.get_running_loop()
+        loop.create_task(self._database_listener_task())
+        logger.debug("[CROSS_PROCESS_LISTEN] Started async database listener task")
+
+    def stop_listening(self) -> None:
+        """Stop listening for cross-process events."""
+        logger.debug("[CROSS_PROCESS_LISTEN] Stopping cross-process event listening")
+        self._running = False
+
+    async def _database_listener_task(self) -> None:
+        """Single database listener task that processes events and recreates itself."""
+        if not self._running:
+            # this should end the task loop
+            logger.debug("[CROSS_PROCESS_LISTENER] Stopping database listener task")
+            return
+
+        # Get unprocessed events from other processes
+        events = self._db.get_unprocessed_events(str(self._process_id))
+        if events:
+            logger.debug(f"[CROSS_PROCESS_LISTENER] Found {len(events)} unprocessed events")
+        else:
+            logger.debug(f"[CROSS_PROCESS_LISTENER] No unprocessed events found for process {self._process_id}")
+
+        for event in events:
+            logger.debug(
+                f"[CROSS_PROCESS_LISTENER] Processing event {event['event_id']} of type {event['event_type']}"
+            )
+            # Handle the event
+            self._handle_cross_process_event(event["event_type"], event["data"])
+            logger.debug(f"[CROSS_PROCESS_LISTENER] Successfully processed event {event['event_id']}")
+
+            # Mark as processed
+            self._db.mark_event_processed(event["event_id"])
+            logger.debug(f"[CROSS_PROCESS_LISTENER] Marked event {event['event_id']} as processed")
+
+        # Clean up old events every hour
+        current_time = time.time()
+        if not hasattr(self, "_last_cleanup"):
+            self._last_cleanup = current_time
+        elif current_time - self._last_cleanup >= 3600:
+            logger.debug("[CROSS_PROCESS_LISTENER] Cleaning up old events")
+            self._db.cleanup_old_events()
+            self._last_cleanup = current_time
+
+        # Schedule the next task if still running
+        await asyncio.sleep(1.0)
+        loop = asyncio.get_running_loop()
+        loop.create_task(self._database_listener_task())
+
+    def _handle_cross_process_event(self, event_type: str, data: Any) -> None:
+        """Handle events received from other processes."""
+        logger.debug(f"[CROSS_PROCESS_HANDLE] Handling cross-process event type: {event_type}")
+        logger.debug(f"[CROSS_PROCESS_HANDLE] Calling {len(self._listeners)} listeners")
+
+        for i, listener in enumerate(self._listeners):
+            try:
+                logger.debug(f"[CROSS_PROCESS_HANDLE] Calling listener {i}")
+                listener(event_type, data)
+                logger.debug(f"[CROSS_PROCESS_HANDLE] Successfully called listener {i}")
+            except Exception as e:
+                logger.debug(f"[CROSS_PROCESS_HANDLE] Cross-process event listener {i} failed for {event_type}: {e}")

{eval_protocol-0.2.43 → eval_protocol-0.2.45}/eval_protocol/event_bus/sqlite_event_bus_database.py

@@ -2,7 +2,7 @@ import time
 from typing import Any, List
 from uuid import uuid4
 
-from peewee import CharField, DateTimeField, Model, SqliteDatabase
+from peewee import BooleanField, CharField, DateTimeField, Model, SqliteDatabase
 from playhouse.sqlite_ext import JSONField
 
 from eval_protocol.event_bus.logger import logger
@@ -25,7 +25,7 @@ class SqliteEventBusDatabase:
             data = JSONField()
             timestamp = DateTimeField()
             process_id = CharField()
-            processed = CharField(default="false")  # Track if event has been processed
+            processed = BooleanField(default=False)  # Track if event has been processed
 
         self._Event = Event
         self._db.connect()
@@ -46,7 +46,7 @@ class SqliteEventBusDatabase:
                 data=serialized_data,
                 timestamp=time.time(),
                 process_id=process_id,
-                processed="false",
+                processed=False,
             )
         except Exception as e:
             logger.warning(f"Failed to publish event to database: {e}")
@@ -56,7 +56,7 @@ class SqliteEventBusDatabase:
         try:
             query = (
                 self._Event.select()
-                .where((self._Event.process_id != process_id) & (self._Event.processed == "false"))
+                .where((self._Event.process_id != process_id) & (~self._Event.processed))
                 .order_by(self._Event.timestamp)
             )
 
@@ -80,7 +80,7 @@ class SqliteEventBusDatabase:
     def mark_event_processed(self, event_id: str) -> None:
         """Mark an event as processed."""
         try:
-            self._Event.update(processed="true").where(self._Event.event_id == event_id).execute()
+            self._Event.update(processed=True).where(self._Event.event_id == event_id).execute()
         except Exception as e:
             logger.debug(f"Failed to mark event as processed: {e}")
 
@@ -88,8 +88,6 @@ class SqliteEventBusDatabase:
         """Clean up old processed events."""
         try:
             cutoff_time = time.time() - (max_age_hours * 3600)
-            self._Event.delete().where(
-                (self._Event.processed == "true") & (self._Event.timestamp < cutoff_time)
-            ).execute()
+            self._Event.delete().where((self._Event.processed) & (self._Event.timestamp < cutoff_time)).execute()
         except Exception as e:
             logger.debug(f"Failed to cleanup old events: {e}")

{eval_protocol-0.2.43 → eval_protocol-0.2.45}/eval_protocol/log_utils/elasticsearch_client.py

@@ -100,6 +100,25 @@ class ElasticsearchClient:
         except Exception:
             return False
 
+    def clear_index(self) -> bool:
+        """Clear all documents from the index.
+
+        Returns:
+            bool: True if successful, False otherwise
+        """
+        try:
+            # Delete all documents by query
+            response = self._make_request(
+                "POST", f"{self.index_url}/_delete_by_query", json_data={"query": {"match_all": {}}}
+            )
+            if response.status_code == 200:
+                # Refresh the index to ensure changes are visible
+                refresh_response = self._make_request("POST", f"{self.index_url}/_refresh")
+                return refresh_response.status_code == 200
+            return False
+        except Exception:
+            return False
+
     def get_mapping(self) -> Optional[Dict[str, Any]]:
         """Get the index mapping.
 

{eval_protocol-0.2.43 → eval_protocol-0.2.45}/eval_protocol/log_utils/elasticsearch_direct_http_handler.py

@@ -2,7 +2,7 @@ import logging
 import os
 from concurrent.futures import ThreadPoolExecutor
 from typing import Optional, Any, Dict
-from datetime import datetime
+from datetime import datetime, timezone
 
 from eval_protocol.types.remote_rollout_processor import ElasticsearchConfig
 from .elasticsearch_client import ElasticsearchClient
@@ -36,8 +36,8 @@ class ElasticsearchDirectHttpHandler(logging.Handler):
     def emit(self, record: logging.LogRecord) -> None:
         """Emit a log record by scheduling it for async transmission."""
         try:
-            # Create proper ISO 8601 timestamp
-            timestamp = datetime.fromtimestamp(record.created).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+            # Create proper ISO 8601 timestamp in UTC
+            timestamp = datetime.fromtimestamp(record.created, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
 
             rollout_id = self._get_rollout_id(record)
             logger.debug(f"Emitting log record: {record.getMessage()} with rollout_id: {rollout_id}")

eval_protocol-0.2.45/eval_protocol/log_utils/util.py

@@ -0,0 +1,22 @@
+import os
+from eval_protocol.types.remote_rollout_processor import ElasticsearchConfig
+from .elasticsearch_direct_http_handler import ElasticsearchDirectHttpHandler
+
+
+def setup_rollout_logging_for_elasticsearch_handler(
+    handler: ElasticsearchDirectHttpHandler, rollout_id: str, elastic_search_config: ElasticsearchConfig
+) -> None:
+    """
+    Whenever a new subprocess is created, we need to setup the rollout context
+    for the subprocess. This is useful when implementing your own remote server
+    for rollout processing.
+
+    1. Set the EP_ROLLOUT_ID environment variable
+    2. Configure the Elasticsearch handler with the Elasticsearch config
+    """
+
+    # this should only affect this subprocess so logs from this subprocess can
+    # be correlated to the rollout
+    os.environ["EP_ROLLOUT_ID"] = rollout_id
+
+    handler.configure(elasticsearch_config=elastic_search_config)

{eval_protocol-0.2.43 → eval_protocol-0.2.45}/eval_protocol/pytest/remote_rollout_processor.py

@@ -70,7 +70,7 @@ def _default_output_data_loader(config: DataLoaderConfig) -> DynamicDataLoader:
     def fetch_traces() -> List[EvaluationRow]:
         base_url = config.model_base_url or "https://tracing.fireworks.ai"
         adapter = FireworksTracingAdapter(base_url=base_url)
-        return adapter.get_evaluation_rows(tags=[f"rollout_id:{config.rollout_id}"], proxy_max_retries=5)
+        return adapter.get_evaluation_rows(tags=[f"rollout_id:{config.rollout_id}"], max_retries=5)
 
     return DynamicDataLoader(generators=[fetch_traces], preprocess_fn=filter_longest_conversation)