eval-protocol 0.2.6.dev1__tar.gz → 0.2.6.dev2__tar.gz

{eval_protocol-0.2.6.dev1/eval_protocol.egg-info → eval_protocol-0.2.6.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.6.dev1
+Version: 0.2.6.dev2
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT
@@ -38,10 +38,10 @@ Requires-Dist: litellm>=1.0.0
 Requires-Dist: addict>=2.4.0
 Requires-Dist: deepdiff>=6.0.0
 Requires-Dist: pandas>=1.5.0
-Requires-Dist: watchdog>=2.1.0
 Requires-Dist: websockets>=15.0.1
 Requires-Dist: fastapi>=0.116.1
 Requires-Dist: pytest>=6.0.0
+Requires-Dist: peewee>=3.18.2
 Provides-Extra: dev
 Requires-Dist: build; extra == "dev"
 Requires-Dist: twine; extra == "dev"

{eval_protocol-0.2.6.dev1 → eval_protocol-0.2.6.dev2}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-08-08T10:51:54-0700",
+ "date": "2025-08-10T19:39:17-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "986452fd04442f9a0d1ba902753a83480e413d43",
- "version": "0.2.6-dev1"
+ "full-revisionid": "a50c3f62fdb5be7347741446338d8c1771e92547",
+ "version": "0.2.6-dev2"
 }
 '''  # END VERSION_JSON
 

eval_protocol-0.2.6.dev2/eval_protocol/common_utils.py

@@ -0,0 +1,55 @@
+import json
+import re
+from typing import Any, Dict, List
+
+import requests
+
+
+def load_jsonl(file_path: str) -> List[Dict[str, Any]]:
+    """
+    Reads a JSONL file where each line is a valid JSON object and returns a list of these objects.
+
+    Args:
+        file_path: Path to the JSONL file.
+
+    Returns:
+        A list of dictionaries, where each dictionary is a parsed JSON object from a line.
+        Returns an empty list if the file is not found or if errors occur during parsing. Supports HTTP urls and local file paths.
+    """
+    data: List[Dict[str, Any]] = []
+    if file_path.startswith("http://") or file_path.startswith("https://"):
+        resp = requests.get(file_path, stream=True, timeout=30)
+        resp.raise_for_status()
+        for line_number, raw in enumerate(resp.iter_lines(decode_unicode=True), start=1):
+            if raw is None:
+                continue
+            stripped = raw.strip()
+            if not stripped:
+                continue
+            try:
+                data.append(json.loads(stripped))
+            except json.JSONDecodeError as e:
+                print(f"Error parsing JSON line for URL {file_path} at line {line_number}")
+                row_id_index = stripped.find("row_id")
+                if row_id_index != -1:
+                    row_id = re.search(r'"row_id": (.*),', stripped[row_id_index:])
+                    raise ValueError(f"{e.msg} at line {line_number}: {stripped} ({row_id})") from e
+                raise e
+    else:
+        with open(file_path, "r", encoding="utf-8") as f:
+            for line_number, line in enumerate(f, start=1):
+                # Skip entirely blank or whitespace-only lines to be robust to trailing newlines
+                stripped = line.strip()
+                if not stripped:
+                    continue
+                try:
+                    data.append(json.loads(stripped))
+                except json.JSONDecodeError as e:
+                    print(f"Error parsing JSON line for file {file_path} at line {line_number}")
+                    # attempt to find "row_id" in the line by finding index of "row_id" and performing regex of `"row_id": (.*),`
+                    row_id_index = line.find("row_id")
+                    if row_id_index != -1:
+                        row_id = re.search(r'"row_id": (.*),', line[row_id_index:])
+                        raise ValueError(f"{e.msg} at line {line_number}: {line} ({row_id})") from e
+                    raise e
+    return data

eval_protocol-0.2.6.dev2/eval_protocol/dataset_logger/__init__.py

@@ -0,0 +1,3 @@
+from eval_protocol.dataset_logger.sqlite_dataset_logger_adapter import SqliteDatasetLoggerAdapter
+
+default_logger = SqliteDatasetLoggerAdapter()

{eval_protocol-0.2.6.dev1 → eval_protocol-0.2.6.dev2}/eval_protocol/dataset_logger/dataset_logger.py

@@ -4,6 +4,8 @@ from typing import TYPE_CHECKING, List, Optional
 if TYPE_CHECKING:
     from eval_protocol.models import EvaluationRow
 
+LOG_EVENT_TYPE = "log"
+
 
 class DatasetLogger(ABC):
     """

eval_protocol-0.2.6.dev2/eval_protocol/dataset_logger/sqlite_dataset_logger_adapter.py

@@ -0,0 +1,39 @@
+import os
+from typing import List, Optional
+
+from eval_protocol.dataset_logger.dataset_logger import LOG_EVENT_TYPE, DatasetLogger
+from eval_protocol.dataset_logger.sqlite_evaluation_row_store import SqliteEvaluationRowStore
+from eval_protocol.directory_utils import find_eval_protocol_dir
+from eval_protocol.event_bus import event_bus
+from eval_protocol.event_bus.logger import logger
+from eval_protocol.models import EvaluationRow
+
+
+class SqliteDatasetLoggerAdapter(DatasetLogger):
+    def __init__(self, db_path: Optional[str] = None, store: Optional[SqliteEvaluationRowStore] = None):
+        eval_protocol_dir = find_eval_protocol_dir()
+        if db_path is not None and store is not None:
+            raise ValueError("Provide only one of db_path or store, not both.")
+        if store is not None:
+            self.db_path = store.db_path
+            self._store = store
+        else:
+            self.db_path = db_path if db_path is not None else os.path.join(eval_protocol_dir, "logs.db")
+            self._store = SqliteEvaluationRowStore(self.db_path)
+
+    def log(self, row: "EvaluationRow") -> None:
+        row_id = row.input_metadata.row_id
+        data = row.model_dump(exclude_none=True, mode="json")
+        self._store.upsert_row(row_id=row_id, data=data)
+        try:
+            event_bus.emit(LOG_EVENT_TYPE, EvaluationRow(**data))
+        except Exception as e:
+            # Avoid breaking storage due to event emission issues
+            logger.error(f"Failed to emit row_upserted event: {e}")
+            pass
+
+    def read(self, row_id: Optional[str] = None) -> List["EvaluationRow"]:
+        from eval_protocol.models import EvaluationRow
+
+        results = self._store.read_rows(row_id=row_id)
+        return [EvaluationRow(**data) for data in results]

eval_protocol-0.2.6.dev2/eval_protocol/dataset_logger/sqlite_evaluation_row_store.py

@@ -0,0 +1,57 @@
+import os
+from typing import List, Optional
+
+from peewee import CharField, Model, SqliteDatabase
+from playhouse.sqlite_ext import JSONField
+
+from eval_protocol.models import EvaluationRow
+
+
+class SqliteEvaluationRowStore:
+    """
+    Lightweight reusable SQLite store for evaluation rows.
+
+    Stores arbitrary row data as JSON keyed by a unique string `row_id`.
+    """
+
+    def __init__(self, db_path: str):
+        os.makedirs(os.path.dirname(db_path), exist_ok=True)
+        self._db_path = db_path
+        self._db = SqliteDatabase(self._db_path)
+
+        class BaseModel(Model):
+            class Meta:
+                database = self._db
+
+        class EvaluationRow(BaseModel):  # type: ignore
+            row_id = CharField(unique=True)
+            data = JSONField()
+
+        self._EvaluationRow = EvaluationRow
+
+        self._db.connect()
+        self._db.create_tables([EvaluationRow])
+
+    @property
+    def db_path(self) -> str:
+        return self._db_path
+
+    def upsert_row(self, row_id: str, data: dict) -> None:
+        if self._EvaluationRow.select().where(self._EvaluationRow.row_id == row_id).exists():
+            self._EvaluationRow.update(data=data).where(self._EvaluationRow.row_id == row_id).execute()
+        else:
+            self._EvaluationRow.create(row_id=row_id, data=data)
+
+    def read_rows(self, row_id: Optional[str] = None) -> List[dict]:
+        if row_id is None:
+            query = self._EvaluationRow.select().dicts()
+        else:
+            query = self._EvaluationRow.select().dicts().where(self._EvaluationRow.row_id == row_id)
+        results = list(query)
+        return [result["data"] for result in results]
+
+    def delete_row(self, row_id: str) -> int:
+        return self._EvaluationRow.delete().where(self._EvaluationRow.row_id == row_id).execute()
+
+    def delete_all_rows(self) -> int:
+        return self._EvaluationRow.delete().execute()

eval_protocol-0.2.6.dev2/eval_protocol/event_bus/__init__.py

@@ -0,0 +1,5 @@
+# Global event bus instance - uses SqliteEventBus for cross-process functionality
+from eval_protocol.event_bus.event_bus import EventBus
+from eval_protocol.event_bus.sqlite_event_bus import SqliteEventBus
+
+event_bus: EventBus = SqliteEventBus()

eval_protocol-0.2.6.dev2/eval_protocol/event_bus/event_bus.py

@@ -0,0 +1,50 @@
+from typing import Any, Callable, List
+
+from eval_protocol.event_bus.logger import logger
+
+
+class EventBus:
+    """Core event bus interface for decoupling components in the evaluation system."""
+
+    def __init__(self):
+        self._listeners: List[Callable[[str, Any], None]] = []
+
+    def subscribe(self, callback: Callable[[str, Any], None]) -> None:
+        """Subscribe to events.
+
+        Args:
+            callback: Function that takes (event_type, data) parameters
+        """
+        self._listeners.append(callback)
+
+    def unsubscribe(self, callback: Callable[[str, Any], None]) -> None:
+        """Unsubscribe from events.
+
+        Args:
+            callback: The callback function to remove
+        """
+        try:
+            self._listeners.remove(callback)
+        except ValueError:
+            pass  # Callback wasn't subscribed
+
+    def emit(self, event_type: str, data: Any) -> None:
+        """Emit an event to all subscribers.
+
+        Args:
+            event_type: Type of event (e.g., "row_upserted")
+            data: Event data
+        """
+        for listener in self._listeners:
+            try:
+                listener(event_type, data)
+            except Exception as e:
+                logger.debug(f"Event listener failed for {event_type}: {e}")
+
+    def start_listening(self) -> None:
+        """Start listening for cross-process events. Override in subclasses."""
+        pass
+
+    def stop_listening(self) -> None:
+        """Stop listening for cross-process events. Override in subclasses."""
+        pass

eval_protocol-0.2.6.dev2/eval_protocol/event_bus/logger.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger(__name__)

eval_protocol-0.2.6.dev2/eval_protocol/event_bus/sqlite_event_bus.py

@@ -0,0 +1,109 @@
+import threading
+import time
+from typing import Any, Optional
+from uuid import uuid4
+
+from eval_protocol.event_bus.event_bus import EventBus
+from eval_protocol.event_bus.logger import logger
+from eval_protocol.event_bus.sqlite_event_bus_database import SqliteEventBusDatabase
+
+
+class SqliteEventBus(EventBus):
+    """SQLite-based event bus implementation that supports cross-process communication."""
+
+    def __init__(self, db_path: Optional[str] = None):
+        super().__init__()
+
+        # Use the same database as the evaluation row store
+        if db_path is None:
+            import os
+
+            from eval_protocol.directory_utils import find_eval_protocol_dir
+
+            eval_protocol_dir = find_eval_protocol_dir()
+            db_path = os.path.join(eval_protocol_dir, "logs.db")
+
+        self._db = SqliteEventBusDatabase(db_path)
+        self._running = False
+        self._listener_thread: Optional[threading.Thread] = None
+        self._process_id = str(uuid4())
+
+    def emit(self, event_type: str, data: Any) -> None:
+        """Emit an event to all subscribers.
+
+        Args:
+            event_type: Type of event (e.g., "log")
+            data: Event data
+        """
+        # Call local listeners immediately
+        super().emit(event_type, data)
+
+        # Publish to cross-process subscribers
+        self._publish_cross_process(event_type, data)
+
+    def _publish_cross_process(self, event_type: str, data: Any) -> None:
+        """Publish event to cross-process subscribers via database."""
+        self._db.publish_event(event_type, data, self._process_id)
+
+    def start_listening(self) -> None:
+        """Start listening for cross-process events."""
+        if self._running:
+            return
+
+        self._running = True
+        self._start_database_listener()
+
+    def stop_listening(self) -> None:
+        """Stop listening for cross-process events."""
+        self._running = False
+        if self._listener_thread and self._listener_thread.is_alive():
+            self._listener_thread.join(timeout=1)
+
+    def _start_database_listener(self) -> None:
+        """Start database-based event listener."""
+
+        def database_listener():
+            last_cleanup = time.time()
+
+            while self._running:
+                try:
+                    # Get unprocessed events from other processes
+                    events = self._db.get_unprocessed_events(self._process_id)
+
+                    for event in events:
+                        if not self._running:
+                            break
+
+                        try:
+                            # Handle the event
+                            self._handle_cross_process_event(event["event_type"], event["data"])
+
+                            # Mark as processed
+                            self._db.mark_event_processed(event["event_id"])
+
+                        except Exception as e:
+                            logger.debug(f"Failed to process event {event['event_id']}: {e}")
+
+                    # Clean up old events every hour
+                    current_time = time.time()
+                    if current_time - last_cleanup >= 3600:
+                        self._db.cleanup_old_events()
+                        last_cleanup = current_time
+
+                    # Small sleep to prevent busy waiting
+                    time.sleep(0.1)
+
+                except Exception as e:
+                    logger.debug(f"Database listener error: {e}")
+                    time.sleep(1)
+
+        self._listener_thread = threading.Thread(target=database_listener, daemon=True)
+        self._listener_thread.start()
+
+    def _handle_cross_process_event(self, event_type: str, data: Any) -> None:
+        """Handle events received from other processes."""
+        for listener in self._listeners:
+            try:
+                listener(event_type, data)
+            except Exception as e:
+                logger.debug(f"Cross-process event listener failed for {event_type}: {e}")

eval_protocol-0.2.6.dev2/eval_protocol/event_bus/sqlite_event_bus_database.py

@@ -0,0 +1,95 @@
+import time
+from typing import Any, List
+from uuid import uuid4
+
+from peewee import CharField, DateTimeField, Model, SqliteDatabase
+from playhouse.sqlite_ext import JSONField
+
+from eval_protocol.event_bus.logger import logger
+
+
+class SqliteEventBusDatabase:
+    """SQLite database for cross-process event communication."""
+
+    def __init__(self, db_path: str):
+        self._db_path = db_path
+        self._db = SqliteDatabase(db_path)
+
+        class BaseModel(Model):
+            class Meta:
+                database = self._db
+
+        class Event(BaseModel):  # type: ignore
+            event_id = CharField(unique=True)
+            event_type = CharField()
+            data = JSONField()
+            timestamp = DateTimeField()
+            process_id = CharField()
+            processed = CharField(default="false")  # Track if event has been processed
+
+        self._Event = Event
+        self._db.connect()
+        self._db.create_tables([Event])
+
+    def publish_event(self, event_type: str, data: Any, process_id: str) -> None:
+        """Publish an event to the database."""
+        try:
+            # Serialize data, handling pydantic models
+            if hasattr(data, "model_dump"):
+                serialized_data = data.model_dump(mode="json", exclude_none=True)
+            else:
+                serialized_data = data
+
+            self._Event.create(
+                event_id=str(uuid4()),
+                event_type=event_type,
+                data=serialized_data,
+                timestamp=time.time(),
+                process_id=process_id,
+                processed="false",
+            )
+        except Exception as e:
+            logger.warning(f"Failed to publish event to database: {e}")
+
+    def get_unprocessed_events(self, process_id: str) -> List[dict]:
+        """Get unprocessed events from other processes."""
+        try:
+            query = (
+                self._Event.select()
+                .where((self._Event.process_id != process_id) & (self._Event.processed == "false"))
+                .order_by(self._Event.timestamp)
+            )
+
+            events = []
+            for event in query:
+                events.append(
+                    {
+                        "event_id": event.event_id,
+                        "event_type": event.event_type,
+                        "data": event.data,
+                        "timestamp": event.timestamp,
+                        "process_id": event.process_id,
+                    }
+                )
+
+            return events
+        except Exception as e:
+            logger.warning(f"Failed to get unprocessed events: {e}")
+            return []
+
+    def mark_event_processed(self, event_id: str) -> None:
+        """Mark an event as processed."""
+        try:
+            self._Event.update(processed="true").where(self._Event.event_id == event_id).execute()
+        except Exception as e:
+            logger.debug(f"Failed to mark event as processed: {e}")
+
+    def cleanup_old_events(self, max_age_hours: int = 24) -> None:
+        """Clean up old processed events."""
+        try:
+            cutoff_time = time.time() - (max_age_hours * 3600)
+            self._Event.delete().where(
+                (self._Event.processed == "true") & (self._Event.timestamp < cutoff_time)
+            ).execute()
+        except Exception as e:
+            logger.debug(f"Failed to cleanup old events: {e}")

{eval_protocol-0.2.6.dev1 → eval_protocol-0.2.6.dev2}/eval_protocol/generation/clients.py

@@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional
 
 import aiohttp
 from omegaconf import DictConfig
-from pydantic import BaseModel, Field  # Added for new models
+from pydantic import BaseModel  # Added for new models
 
 logger = logging.getLogger(__name__)
 
@@ -83,6 +83,9 @@ class FireworksModelClient(ModelClient):
         }
         if self.top_p is not None:
             payload["top_p"] = self.top_p
+        # Include reasoning settings if configured (for reasoning-capable models)
+        if self.reasoning_effort:
+            payload["reasoning_effort"] = self.reasoning_effort
 
         if tools:
             payload["tools"] = tools

{eval_protocol-0.2.6.dev1 → eval_protocol-0.2.6.dev2}/eval_protocol/logging_utils.py

@@ -12,7 +12,7 @@ import sys
 from pathlib import Path
 from typing import Optional
 
-from eval_protocol.dataset_logger.directory_utils import find_eval_protocol_dir
+from eval_protocol.directory_utils import find_eval_protocol_dir
 
 
 def setup_logger(

{eval_protocol-0.2.6.dev1 → eval_protocol-0.2.6.dev2}/eval_protocol/mcp/client/connection.py

@@ -9,14 +9,16 @@ import asyncio
 import hashlib
 import json
 import logging
+import time
 from contextlib import AsyncExitStack
 from typing import Any, Dict, List, Optional, Tuple
 
+import httpx
 from mcp.client.session import ClientSession
 from mcp.client.streamable_http import streamablehttp_client
+from mcp.types import Implementation
 
 from ...types import MCPSession
-from mcp.types import Implementation
 
 logger = logging.getLogger(__name__)
 
@@ -109,15 +111,13 @@ class MCPConnectionManager:
         """
         Clean session data in remote mcp server for the given session
         """
-        import httpx
-
         base_url = session.base_url.rstrip("/").removesuffix("/mcp")
         url = f"{base_url}/control/reset_session"
 
         headers = {"mcp-session-id": session.session_id}
         body = {"seed": session.seed}
 
-        timeout = httpx.Timeout(3.0)
+        timeout = httpx.Timeout(15.0)
         async with httpx.AsyncClient(timeout=timeout) as client:
             resp = await client.post(url, headers=headers, json=body)
             resp.raise_for_status()
@@ -202,8 +202,6 @@ class MCPConnectionManager:
         initial_observation = None
 
         try:
-            import httpx
-
             # Extract base URL and session ID from the MCP session
             base_url = session.base_url.rstrip("/").removesuffix("/mcp")
             session_id = session.session_id
@@ -459,9 +457,6 @@ class MCPConnectionManager:
         control_plane_info = {}
 
         try:
-            # Query control plane endpoints following the new architecture
-            import httpx
-
             # Extract base URL and session ID from the MCP session
             base_url = session.base_url.rstrip("/").removesuffix("/mcp")
             # Use the session ID from the established MCP session
@@ -544,10 +539,10 @@ class MCPConnectionManager:
                 await session._exit_stack.aclose()
             except asyncio.CancelledError:
                 # Handle cancellation gracefully (especially important for Python 3.12)
-                logger.debug(f"Session {session.session_id} close was cancelled")
+                logger.error(f"Session {session.session_id} close was cancelled")
             except Exception as e:
                 # Hitting this error, probably because of use of threads: "Attempted to exit cancel scope in a different task than it was entered in"
-                logger.debug(f"Error closing session {session.session_id}: {e}")
+                logger.error(f"Error closing session {session.session_id}: {e}")
             finally:
                 session._exit_stack = None
                 session._mcp_session = None

{eval_protocol-0.2.6.dev1 → eval_protocol-0.2.6.dev2}/eval_protocol/mcp/execution/base_policy.py

@@ -220,7 +220,7 @@ class LLMBasePolicy(PlaybackPolicyBase, ABC):
             return mcp_tool_calls, usage_stats
         else:
             # No tool calls in response - this is normal when episode ends or LLM provides only text
-            logger.info(f"No tool calls in response for env {env_index}, message content: {message.get('content')}")
+            logger.debug(f"No tool calls in response for env {env_index}, message content: {message.get('content')}")
             return [
                 MCPToolCall(
                     tool_name="_no_tool_call",