eval-protocol 0.2.35.dev1__py3-none-any.whl → 0.2.35.dev2__py3-none-any.whl

eval_protocol/__init__.py

@@ -24,7 +24,7 @@ from .mcp_env import (
 )
 from .data_loader import DynamicDataLoader, InlineDataLoader
 from . import mcp, rewards
-from .models import EvaluateResult, Message, MetricResult, EvaluationRow, InputMetadata
+from .models import EvaluateResult, Message, MetricResult, EvaluationRow, InputMetadata, Status
 from .playback_policy import PlaybackPolicyBase
 from .resources import create_llm_resource
 from .reward_function import RewardFunction
@@ -63,6 +63,7 @@ except ImportError:
 warnings.filterwarnings("default", category=DeprecationWarning, module="eval_protocol")
 
 __all__ = [
+    "Status",
     "RemoteRolloutProcessor",
     "InputMetadata",
     "EvaluationRow",

eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-02T01:54:12-0700",
+ "date": "2025-10-02T12:04:07-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "c4108ba1d87ba6fb76100c63e6ee16f48bc06598",
- "version": "0.2.35-dev1"
+ "full-revisionid": "52178b3b90bb27a7f53fcbbba0bfbb50e7ebb416",
+ "version": "0.2.35-dev2"
 }
 '''  # END VERSION_JSON
 

eval_protocol/cli.py

@@ -15,14 +15,8 @@ from pathlib import Path
 logger = logging.getLogger(__name__)
 
 
-from eval_protocol.evaluation import create_evaluation, preview_evaluation
-
 from .cli_commands.agent_eval_cmd import agent_eval_command
-from .cli_commands.common import (
-    check_agent_environment,
-    check_environment,
-    setup_logging,
-)
+from .cli_commands.common import setup_logging
 from .cli_commands.deploy import deploy_command
 from .cli_commands.deploy_mcp import deploy_mcp_command
 from .cli_commands.logs import logs_command

eval_protocol/cli_commands/agent_eval_cmd.py

@@ -27,11 +27,7 @@ import logging  # For logger instance
 import os  # For environment variables
 from pathlib import Path
 
-from pydantic import ValidationError
-
-from eval_protocol.agent import Orchestrator
 from eval_protocol.agent.task_manager import TaskManager
-from eval_protocol.models import TaskDefinitionModel  # Import the new Pydantic model
 
 # setup_logging is already called in cli.py's main, but good for standalone use if any
 # from .common import setup_logging

eval_protocol/cli_commands/run_eval_cmd.py

@@ -17,7 +17,6 @@ from omegaconf import (  # Ensure MISSING is imported if used in configs
     OmegaConf,
 )
 
-from eval_protocol.execution.pipeline import EvaluationPipeline
 
 logger = logging.getLogger(__name__)
 
@@ -26,6 +25,8 @@ def run_evaluation_command_logic(cfg: DictConfig) -> None:
     """
     Main logic for the 'run-evaluation' command.
     """
+    from eval_protocol.execution.pipeline import EvaluationPipeline
+
     logger.info("Starting 'run-evaluation' command with resolved Hydra config.")
 
     # Make Hydra's runtime output directory available to the pipeline if needed

eval_protocol/logging/elasticsearch_client.py

@@ -0,0 +1,286 @@
+"""
+Centralized Elasticsearch client for all Elasticsearch API operations.
+
+This module provides a unified interface for all Elasticsearch operations
+used throughout the codebase, including index management, document operations,
+and search functionality.
+"""
+
+import json
+import requests
+from typing import Any, Dict, List, Optional, Union
+from urllib.parse import urlparse
+from eval_protocol.types.remote_rollout_processor import ElasticsearchConfig
+
+
+class ElasticsearchClient:
+    """Centralized client for all Elasticsearch operations."""
+
+    def __init__(self, config: ElasticsearchConfig):
+        """Initialize the Elasticsearch client.
+
+        Args:
+            config: Elasticsearch configuration
+        """
+        self.config = config
+        self.base_url = config.url.rstrip("/")
+        self.index_url = f"{self.base_url}/{config.index_name}"
+        self._headers = {"Content-Type": "application/json", "Authorization": f"ApiKey {config.api_key}"}
+
+    def _make_request(
+        self,
+        method: str,
+        url: str,
+        json_data: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
+        params: Optional[Dict[str, Any]] = None,
+        timeout: int = 30,
+    ) -> requests.Response:
+        """Make an HTTP request to Elasticsearch.
+
+        Args:
+            method: HTTP method (GET, POST, PUT, DELETE, HEAD)
+            url: Full URL for the request
+            json_data: JSON data to send in request body
+            params: Query parameters
+            timeout: Request timeout in seconds
+
+        Returns:
+            requests.Response object
+
+        Raises:
+            requests.RequestException: If the request fails
+        """
+        return requests.request(
+            method=method,
+            url=url,
+            headers=self._headers,
+            json=json_data,
+            params=params,
+            verify=self.config.verify_ssl,
+            timeout=timeout,
+        )
+
+    # Index Management Operations
+
+    def create_index(self, mapping: Dict[str, Any]) -> bool:
+        """Create an index with the specified mapping.
+
+        Args:
+            mapping: Index mapping configuration
+
+        Returns:
+            bool: True if successful, False otherwise
+        """
+        try:
+            response = self._make_request("PUT", self.index_url, json_data=mapping)
+            return response.status_code in [200, 201]
+        except Exception:
+            return False
+
+    def index_exists(self) -> bool:
+        """Check if the index exists.
+
+        Returns:
+            bool: True if index exists, False otherwise
+        """
+        try:
+            response = self._make_request("HEAD", self.index_url)
+            return response.status_code == 200
+        except Exception:
+            return False
+
+    def delete_index(self) -> bool:
+        """Delete the index.
+
+        Returns:
+            bool: True if successful, False otherwise
+        """
+        try:
+            response = self._make_request("DELETE", self.index_url)
+            return response.status_code in [200, 404]  # 404 means index doesn't exist
+        except Exception:
+            return False
+
+    def get_mapping(self) -> Optional[Dict[str, Any]]:
+        """Get the index mapping.
+
+        Returns:
+            Dict containing mapping data, or None if failed
+        """
+        try:
+            response = self._make_request("GET", f"{self.index_url}/_mapping")
+            if response.status_code == 200:
+                return response.json()
+            return None
+        except Exception:
+            return None
+
+    def get_index_stats(self) -> Optional[Dict[str, Any]]:
+        """Get index statistics.
+
+        Returns:
+            Dict containing index statistics, or None if failed
+        """
+        try:
+            response = self._make_request("GET", f"{self.index_url}/_stats")
+            if response.status_code == 200:
+                return response.json()
+            return None
+        except Exception:
+            return None
+
+    # Document Operations
+
+    def index_document(self, document: Dict[str, Any], doc_id: Optional[str] = None) -> bool:
+        """Index a document.
+
+        Args:
+            document: Document to index
+            doc_id: Optional document ID
+
+        Returns:
+            bool: True if successful, False otherwise
+        """
+        try:
+            if doc_id:
+                url = f"{self.index_url}/_doc/{doc_id}"
+            else:
+                url = f"{self.index_url}/_doc"
+
+            response = self._make_request("POST", url, json_data=document)
+            return response.status_code in [200, 201]
+        except Exception:
+            return False
+
+    def bulk_index_documents(self, documents: List[Dict[str, Any]]) -> bool:
+        """Bulk index multiple documents.
+
+        Args:
+            documents: List of documents to index
+
+        Returns:
+            bool: True if successful, False otherwise
+        """
+        try:
+            # Prepare bulk request body
+            bulk_body = []
+            for doc in documents:
+                bulk_body.append({"index": {}})
+                bulk_body.append(doc)
+
+            response = self._make_request("POST", f"{self.index_url}/_bulk", json_data=bulk_body)
+            return response.status_code == 200
+        except Exception:
+            return False
+
+    # Search Operations
+
+    def search(
+        self, query: Dict[str, Any], size: int = 10, from_: int = 0, sort: Optional[List[Dict[str, Any]]] = None
+    ) -> Optional[Dict[str, Any]]:
+        """Search documents in the index.
+
+        Args:
+            query: Elasticsearch query
+            size: Number of results to return
+            from_: Starting offset
+            sort: Sort specification
+
+        Returns:
+            Dict containing search results, or None if failed
+        """
+        try:
+            search_body = {"query": query, "size": size, "from": from_}
+
+            if sort:
+                search_body["sort"] = sort
+
+            response = self._make_request("POST", f"{self.index_url}/_search", json_data=search_body)
+
+            if response.status_code == 200:
+                return response.json()
+            return None
+        except Exception:
+            return None
+
+    def search_by_term(self, field: str, value: Any, size: int = 10) -> Optional[Dict[str, Any]]:
+        """Search documents by exact term match.
+
+        Args:
+            field: Field name to search
+            value: Value to match
+            size: Number of results to return
+
+        Returns:
+            Dict containing search results, or None if failed
+        """
+        query = {"term": {field: value}}
+        return self.search(query, size=size)
+
+    def search_by_match(self, field: str, value: str, size: int = 10) -> Optional[Dict[str, Any]]:
+        """Search documents by text match.
+
+        Args:
+            field: Field name to search
+            value: Text to match
+            size: Number of results to return
+
+        Returns:
+            Dict containing search results, or None if failed
+        """
+        query = {"match": {field: value}}
+        return self.search(query, size=size)
+
+    def search_by_match_phrase_prefix(self, field: str, value: str, size: int = 10) -> Optional[Dict[str, Any]]:
+        """Search documents by phrase prefix match.
+
+        Args:
+            field: Field name to search
+            value: Phrase prefix to match
+            size: Number of results to return
+
+        Returns:
+            Dict containing search results, or None if failed
+        """
+        query = {"match_phrase_prefix": {field: value}}
+        return self.search(query, size=size)
+
+    def search_all(self, size: int = 10) -> Optional[Dict[str, Any]]:
+        """Search all documents in the index.
+
+        Args:
+            size: Number of results to return
+
+        Returns:
+            Dict containing search results, or None if failed
+        """
+        query = {"match_all": {}}
+        return self.search(query, size=size)
+
+    # Health and Status Operations
+
+    def health_check(self) -> bool:
+        """Check if Elasticsearch is healthy.
+
+        Returns:
+            bool: True if healthy, False otherwise
+        """
+        try:
+            response = self._make_request("GET", f"{self.base_url}/_cluster/health")
+            return response.status_code == 200
+        except Exception:
+            return False
+
+    def get_cluster_info(self) -> Optional[Dict[str, Any]]:
+        """Get cluster information.
+
+        Returns:
+            Dict containing cluster info, or None if failed
+        """
+        try:
+            response = self._make_request("GET", f"{self.base_url}/_cluster/health")
+            if response.status_code == 200:
+                return response.json()
+            return None
+        except Exception:
+            return None

eval_protocol/logging/elasticsearch_direct_http_handler.py

@@ -1,50 +1,92 @@
 import json
 import logging
 import asyncio
+import os
 import threading
 from concurrent.futures import ThreadPoolExecutor
 from typing import Optional, Tuple, Any, Dict
 from datetime import datetime
-from urllib.parse import urlparse
-import requests
 
-from eval_protocol.types.remote_rollout_processor import ElasticSearchConfig
+from eval_protocol.types.remote_rollout_processor import ElasticsearchConfig
+from .elasticsearch_client import ElasticsearchClient
 
 
 class ElasticsearchDirectHttpHandler(logging.Handler):
-    def __init__(self, elasticsearch_config: ElasticSearchConfig) -> None:
+    def __init__(self, elasticsearch_config: ElasticsearchConfig) -> None:
         super().__init__()
-        self.base_url: str = elasticsearch_config.url.rstrip("/")
-        self.index_name: str = elasticsearch_config.index_name
-        self.api_key: str = elasticsearch_config.api_key
-        self.url: str = f"{self.base_url}/{self.index_name}/_doc"
+        self.config = ElasticsearchConfig(
+            url=elasticsearch_config.url,
+            api_key=elasticsearch_config.api_key,
+            index_name=elasticsearch_config.index_name,
+        )
+        self.client = ElasticsearchClient(self.config)
         self.formatter: logging.Formatter = logging.Formatter()
         self._executor = None
 
-        # Parse URL to determine if we should verify SSL
-        parsed_url = urlparse(elasticsearch_config.url)
-        self.verify_ssl = parsed_url.scheme == "https"
-
     def emit(self, record: logging.LogRecord) -> None:
         """Emit a log record by scheduling it for async transmission."""
         try:
             # Create proper ISO 8601 timestamp
             timestamp = datetime.fromtimestamp(record.created).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
 
+            rollout_id = self._get_rollout_id(record)
+            status_info = self._get_status_info(record)
+
             data: Dict[str, Any] = {
                 "@timestamp": timestamp,
                 "level": record.levelname,
                 "message": record.getMessage(),
                 "logger_name": record.name,
-                # Add other relevant record attributes if needed
+                "rollout_id": rollout_id,
             }
 
+            # Add status information if present
+            if status_info:
+                data.update(status_info)
+
             # Schedule the HTTP request to run asynchronously
             self._schedule_async_send(data, record)
         except Exception as e:
             self.handleError(record)
             print(f"Error preparing log for Elasticsearch: {e}")
 
+    def _get_rollout_id(self, record: logging.LogRecord) -> str:
+        """Get the rollout ID from environment variables."""
+        rollout_id = os.getenv("EP_ROLLOUT_ID")
+        if rollout_id is None:
+            raise ValueError(
+                "EP_ROLLOUT_ID environment variable is not set but needed for ElasticsearchDirectHttpHandler"
+            )
+        return rollout_id
+
+    def _get_status_info(self, record: logging.LogRecord) -> Optional[Dict[str, Any]]:
+        """Extract status information from the log record's extra data."""
+        # Check if 'status' is in the extra data (passed via extra parameter)
+        if hasattr(record, "status") and record.status is not None:  # type: ignore
+            status = record.status  # type: ignore
+
+            # Handle Status class instances (Pydantic BaseModel)
+            if hasattr(status, "code") and hasattr(status, "message"):
+                # Status object - extract code and message
+                status_code = status.code
+                # Handle both enum values and direct integer values
+                if hasattr(status_code, "value"):
+                    status_code = status_code.value
+
+                return {
+                    "status_code": status_code,
+                    "status_message": status.message,
+                    "status_details": getattr(status, "details", []),
+                }
+            elif isinstance(status, dict):
+                # Dictionary representation of status
+                return {
+                    "status_code": status.get("code"),
+                    "status_message": status.get("message"),
+                    "status_details": status.get("details", []),
+                }
+        return None
+
     def _schedule_async_send(self, data: Dict[str, Any], record: logging.LogRecord) -> None:
         """Schedule an async task to send the log data to Elasticsearch."""
         if self._executor is None:
@@ -59,13 +101,9 @@ class ElasticsearchDirectHttpHandler(logging.Handler):
     def _send_to_elasticsearch(self, data: Dict[str, Any], record: logging.LogRecord) -> None:
         """Send data to Elasticsearch (runs in thread pool)."""
         try:
-            response: requests.Response = requests.post(
-                self.url,
-                headers={"Content-Type": "application/json", "Authorization": f"ApiKey {self.api_key}"},
-                data=json.dumps(data),
-                verify=self.verify_ssl,  # If using HTTPS, verify SSL certificate
-            )
-            response.raise_for_status()  # Raise an exception for HTTP errors
+            success = self.client.index_document(data)
+            if not success:
+                raise Exception("Failed to index document to Elasticsearch")
         except Exception as e:
             # Re-raise to be handled by the callback
             raise e

eval_protocol/logging/elasticsearch_index_manager.py

@@ -1,6 +1,6 @@
-import requests
 from typing import Dict, Any, Optional
-from urllib.parse import urlparse
+from .elasticsearch_client import ElasticsearchClient
+from eval_protocol.types.remote_rollout_processor import ElasticsearchConfig
 
 
 class ElasticsearchIndexManager:
@@ -14,16 +14,10 @@ class ElasticsearchIndexManager:
             index_name: Name of the index to manage
             api_key: API key for authentication
         """
-        self.base_url: str = base_url.rstrip("/")
-        self.index_name: str = index_name
-        self.api_key: str = api_key
-        self.index_url: str = f"{self.base_url}/{self.index_name}"
+        self.config = ElasticsearchConfig(url=base_url, api_key=api_key, index_name=index_name)
+        self.client = ElasticsearchClient(self.config)
         self._mapping_created: bool = False
 
-        # Parse URL to determine if we should verify SSL
-        parsed_url = urlparse(base_url)
-        self.verify_ssl = parsed_url.scheme == "https"
-
     def create_logging_index_mapping(self) -> bool:
         """Create index with proper mapping for logging data.
 
@@ -41,25 +35,22 @@ class ElasticsearchIndexManager:
 
             # If index exists but has wrong mapping, delete and recreate it
             if self.index_exists():
-                print(f"Warning: Index {self.index_name} exists with incorrect mapping. Deleting and recreating...")
+                print(
+                    f"Warning: Index {self.config.index_name} exists with incorrect mapping. Deleting and recreating..."
+                )
                 if not self.delete_index():
-                    print(f"Warning: Failed to delete existing index {self.index_name}")
+                    print(f"Warning: Failed to delete existing index {self.config.index_name}")
                     return False
 
             # Create index with proper mapping
             mapping = self._get_logging_mapping()
-            response = requests.put(
-                self.index_url,
-                headers={"Content-Type": "application/json", "Authorization": f"ApiKey {self.api_key}"},
-                json=mapping,
-                verify=self.verify_ssl,
-            )
-
-            if response.status_code in [200, 201]:
+            success = self.client.create_index(mapping)
+
+            if success:
                 self._mapping_created = True
                 return True
             else:
-                print(f"Warning: Failed to create index mapping: {response.status_code} - {response.text}")
+                print("Warning: Failed to create index mapping")
                 return False
 
         except Exception as e:
@@ -74,46 +65,50 @@ class ElasticsearchIndexManager:
         """
         try:
             # Check if index exists
-            response = requests.head(
-                self.index_url, headers={"Authorization": f"ApiKey {self.api_key}"}, verify=self.verify_ssl
-            )
-
-            if response.status_code != 200:
+            if not self.client.index_exists():
                 return False
 
             # Check if mapping is correct
-            mapping_response = requests.get(
-                f"{self.index_url}/_mapping",
-                headers={"Authorization": f"ApiKey {self.api_key}"},
-                verify=self.verify_ssl,
-            )
-
-            if mapping_response.status_code != 200:
+            mapping_data = self.client.get_mapping()
+            if mapping_data is None:
                 return False
 
-            mapping_data = mapping_response.json()
             return self._has_correct_timestamp_mapping(mapping_data)
 
         except Exception:
             return False
 
     def _has_correct_timestamp_mapping(self, mapping_data: Dict[str, Any]) -> bool:
-        """Check if the mapping has @timestamp as a date field.
+        """Check if the mapping has @timestamp as a date field, rollout_id as a keyword field, and status fields.
 
         Args:
             mapping_data: Elasticsearch mapping response data
 
         Returns:
-            bool: True if @timestamp is correctly mapped as date field
+            bool: True if all required fields are correctly mapped
         """
         try:
-            return (
-                self.index_name in mapping_data
-                and "mappings" in mapping_data[self.index_name]
-                and "properties" in mapping_data[self.index_name]["mappings"]
-                and "@timestamp" in mapping_data[self.index_name]["mappings"]["properties"]
-                and mapping_data[self.index_name]["mappings"]["properties"]["@timestamp"].get("type") == "date"
-            )
+            if not (
+                self.config.index_name in mapping_data
+                and "mappings" in mapping_data[self.config.index_name]
+                and "properties" in mapping_data[self.config.index_name]["mappings"]
+            ):
+                return False
+
+            properties = mapping_data[self.config.index_name]["mappings"]["properties"]
+
+            # Check @timestamp is mapped as date
+            timestamp_ok = "@timestamp" in properties and properties["@timestamp"].get("type") == "date"
+
+            # Check rollout_id is mapped as keyword
+            rollout_id_ok = "rollout_id" in properties and properties["rollout_id"].get("type") == "keyword"
+
+            # Check status fields are mapped correctly
+            status_code_ok = "status_code" in properties and properties["status_code"].get("type") == "integer"
+            status_message_ok = "status_message" in properties and properties["status_message"].get("type") == "text"
+            status_details_ok = "status_details" in properties and properties["status_details"].get("type") == "object"
+
+            return timestamp_ok and rollout_id_ok and status_code_ok and status_message_ok and status_details_ok
         except (KeyError, TypeError):
             return False
 
@@ -130,6 +125,10 @@ class ElasticsearchIndexManager:
                     "level": {"type": "keyword"},
                     "message": {"type": "text"},
                     "logger_name": {"type": "keyword"},
+                    "rollout_id": {"type": "keyword"},
+                    "status_code": {"type": "integer"},
+                    "status_message": {"type": "text"},
+                    "status_details": {"type": "object"},
                 }
             }
         }
@@ -141,14 +140,12 @@ class ElasticsearchIndexManager:
             bool: True if index was deleted successfully, False otherwise.
         """
         try:
-            response = requests.delete(
-                self.index_url, headers={"Authorization": f"ApiKey {self.api_key}"}, verify=self.verify_ssl
-            )
-            if response.status_code in [200, 404]:  # 404 means index doesn't exist, which is fine
+            success = self.client.delete_index()
+            if success:
                 self._mapping_created = False
                 return True
             else:
-                print(f"Warning: Failed to delete index: {response.status_code} - {response.text}")
+                print("Warning: Failed to delete index")
                 return False
         except Exception as e:
             print(f"Warning: Failed to delete index: {e}")
@@ -160,13 +157,7 @@ class ElasticsearchIndexManager:
         Returns:
             bool: True if index exists, False otherwise.
         """
-        try:
-            response = requests.head(
-                self.index_url, headers={"Authorization": f"ApiKey {self.api_key}"}, verify=self.verify_ssl
-            )
-            return response.status_code == 200
-        except Exception:
-            return False
+        return self.client.index_exists()
 
     def get_index_stats(self) -> Optional[Dict[str, Any]]:
         """Get statistics about the index.
@@ -174,14 +165,4 @@ class ElasticsearchIndexManager:
         Returns:
             Dict containing index statistics, or None if failed
         """
-        try:
-            response = requests.get(
-                f"{self.index_url}/_stats",
-                headers={"Authorization": f"ApiKey {self.api_key}"},
-                verify=self.verify_ssl,
-            )
-            if response.status_code == 200:
-                return response.json()
-            return None
-        except Exception:
-            return None
+        return self.client.get_index_stats()

eval_protocol/pytest/elasticsearch_setup.py

@@ -6,7 +6,7 @@ from typing import Optional
 
 from dotenv import load_dotenv
 from eval_protocol.directory_utils import find_eval_protocol_dir
-from eval_protocol.types.remote_rollout_processor import ElasticSearchConfig
+from eval_protocol.types.remote_rollout_processor import ElasticsearchConfig
 from eval_protocol.logging.elasticsearch_index_manager import ElasticsearchIndexManager
 
 logger = logging.getLogger(__name__)
@@ -24,7 +24,7 @@ class ElasticsearchSetup:
     def __init__(self):
         self.eval_protocol_dir = find_eval_protocol_dir()
 
-    def setup_elasticsearch(self, index_name: str = "default-logs") -> ElasticSearchConfig:
+    def setup_elasticsearch(self, index_name: str = "default-logs") -> ElasticsearchConfig:
         """
         Set up Elasticsearch, handling both local and remote scenarios.
 
@@ -32,7 +32,7 @@ class ElasticsearchSetup:
             index_name: Name of the Elasticsearch index to use for logging
 
         Returns:
-            ElasticSearchConfig for the running instance with the specified index name.
+            ElasticsearchConfig for the running instance with the specified index name.
         """
         elastic_start_local_dir = os.path.join(self.eval_protocol_dir, "elastic-start-local")
         env_file_path = os.path.join(elastic_start_local_dir, ".env")
@@ -48,11 +48,11 @@ class ElasticsearchSetup:
         self.create_logging_index(index_name)
 
         # Return config with the specified index name
-        return ElasticSearchConfig(url=config.url, api_key=config.api_key, index_name=index_name)
+        return ElasticsearchConfig(url=config.url, api_key=config.api_key, index_name=index_name)
 
     def _setup_existing_docker_elasticsearch(
         self, elastic_start_local_dir: str, env_file_path: str
-    ) -> ElasticSearchConfig:
+    ) -> ElasticsearchConfig:
         """Set up Elasticsearch using existing Docker start.sh script."""
         from eval_protocol.utils.subprocess_utils import run_script_and_wait
 
@@ -63,7 +63,7 @@ class ElasticsearchSetup:
         )
         return self._parse_elastic_env_file(env_file_path)
 
-    def _setup_initialized_docker_elasticsearch(self, env_file_path: str) -> ElasticSearchConfig:
+    def _setup_initialized_docker_elasticsearch(self, env_file_path: str) -> ElasticsearchConfig:
         """Set up Elasticsearch by initializing Docker setup from scratch with retry logic."""
         max_retries = 2
         for attempt in range(max_retries):
@@ -126,7 +126,7 @@ class ElasticsearchSetup:
                 return False
         return False
 
-    def _parse_elastic_env_file(self, env_file_path: str) -> ElasticSearchConfig:
+    def _parse_elastic_env_file(self, env_file_path: str) -> ElasticsearchConfig:
         """Parse ES_LOCAL_API_KEY and ES_LOCAL_URL from .env file."""
         loaded = load_dotenv(env_file_path)
         if not loaded:
@@ -138,7 +138,7 @@ class ElasticsearchSetup:
         if not url or not api_key:
             raise ElasticsearchSetupError("Failed to parse ES_LOCAL_API_KEY and ES_LOCAL_URL from .env file")
 
-        return ElasticSearchConfig(url=url, api_key=api_key, index_name="default-logs")
+        return ElasticsearchConfig(url=url, api_key=api_key, index_name="default-logs")
 
     def create_logging_index(self, index_name: str) -> bool:
         """Create an Elasticsearch index with proper mapping for logging data.

eval_protocol/pytest/remote_rollout_processor.py

@@ -4,9 +4,10 @@ from typing import Any, Dict, List, Optional, Callable
 
 import requests
 
+from eval_protocol.logging.elasticsearch_client import ElasticsearchClient
 from eval_protocol.models import EvaluationRow, Status
 from eval_protocol.data_loader.dynamic_data_loader import DynamicDataLoader
-from eval_protocol.types.remote_rollout_processor import ElasticSearchConfig, InitRequest, RolloutMetadata
+from eval_protocol.types.remote_rollout_processor import ElasticsearchConfig, InitRequest, RolloutMetadata
 from .rollout_processor import RolloutProcessor
 from .types import RolloutProcessorConfig
 from .elasticsearch_setup import ElasticsearchSetup
@@ -33,7 +34,7 @@ class RemoteRolloutProcessor(RolloutProcessor):
         timeout_seconds: float = 120.0,
         output_data_loader: Callable[[str], DynamicDataLoader],
         disable_elastic_search: bool = False,
-        elastic_search_config: Optional[ElasticSearchConfig] = None,
+        elastic_search_config: Optional[ElasticsearchConfig] = None,
     ):
         # Prefer constructor-provided configuration. These can be overridden via
         # config.kwargs at call time for backward compatibility.
@@ -56,7 +57,7 @@ class RemoteRolloutProcessor(RolloutProcessor):
         self._elastic_search_config = self._setup_elastic_search()
         logger.info("Elasticsearch setup complete")
 
-    def _setup_elastic_search(self) -> ElasticSearchConfig:
+    def _setup_elastic_search(self) -> ElasticsearchConfig:
         """Set up Elasticsearch using the dedicated setup module."""
         setup = ElasticsearchSetup()
         return setup.setup_elasticsearch()
@@ -183,6 +184,10 @@ class RemoteRolloutProcessor(RolloutProcessor):
                 r.raise_for_status()
                 return r.json()
 
+            elasticsearch_client = (
+                ElasticsearchClient(self._elastic_search_config) if self._elastic_search_config else None
+            )
+
             while time.time() < deadline:
                 try:
                     status = await asyncio.to_thread(_get_status)

eval_protocol/types/remote_rollout_processor.py

@@ -4,10 +4,11 @@ Request and response models for remote rollout processor servers.
 
 from typing import Any, Dict, List, Optional
 from pydantic import BaseModel, Field
+from urllib.parse import urlparse
 from eval_protocol.models import Message, Status
 
 
-class ElasticSearchConfig(BaseModel):
+class ElasticsearchConfig(BaseModel):
     """
     Configuration for Elasticsearch.
     """
@@ -16,6 +17,12 @@ class ElasticSearchConfig(BaseModel):
     api_key: str
     index_name: str
 
+    @property
+    def verify_ssl(self) -> bool:
+        """Infer verify_ssl from URL scheme."""
+        parsed_url = urlparse(self.url)
+        return parsed_url.scheme == "https"
+
 
 class RolloutMetadata(BaseModel):
     """Metadata for rollout execution."""
@@ -31,7 +38,7 @@ class InitRequest(BaseModel):
     """Request model for POST /init endpoint."""
 
     model: str
-    elastic_search_config: Optional[ElasticSearchConfig] = None
+    elastic_search_config: Optional[ElasticsearchConfig] = None
     messages: Optional[List[Message]] = None
     tools: Optional[List[Dict[str, Any]]] = None
 

{eval_protocol-0.2.35.dev1.dist-info → eval_protocol-0.2.35.dev2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.35.dev1
+Version: 0.2.35.dev2
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.35.dev1.dist-info → eval_protocol-0.2.35.dev2.dist-info}/RECORD

@@ -3,11 +3,11 @@ development/normalize_sandbox_fusion.py,sha256=1598fX-W6MNM2QZusUEurOLvlMv0-a8-T
 development/utils/__init__.py,sha256=5_3ae0whpBdWIYreHlWCc2H9BCCsQfuVFSKZbxXID60,79
 development/utils/generate_api_key.py,sha256=hHCMFkzW4yxqwcn2ct5diDm-PR9cMX9XP7IYieZvH5U,869
 development/utils/subprocess_manager.py,sha256=7n7rT9ji7h93i79SMrGS5RNesrnLFjdFON9_eQCmYNE,18937
-eval_protocol/__init__.py,sha256=KSI4bHfk9SoXWXqMBWDyjvz5RW2kFT6zt1pAC70ZEgA,3321
+eval_protocol/__init__.py,sha256=n0wwL4ueoazupJc8Enjqt8C917hHT4FcrsjqjZ_e9XU,3343
 eval_protocol/__main__.py,sha256=FIW5fo2X2rsPThurSlZEuqvE1u0XNwJW1uoej_OhAAs,161
-eval_protocol/_version.py,sha256=m3jeKhOXL0PZa1Bvksj29LbXTi8qC6PJXuT42NtlB8w,503
+eval_protocol/_version.py,sha256=BtvZ7LNFVsPmnGr9QV8lMcCn8Yl3uTtapUP8aIqg6Us,503
 eval_protocol/auth.py,sha256=6KOY-IIfrYkWmvZj3N0qrVHHCGbOFmcr2F73ZxIoYTg,5854
-eval_protocol/cli.py,sha256=A1acUJz6IVSFcVU5CrGSUXSOax0wj4LP4VOF5hCzga4,18905
+eval_protocol/cli.py,sha256=otIJKB_TjmuO9KPCBvfHqjkmySZW2XTMZgqXIoVSzAE,18768
 eval_protocol/common_utils.py,sha256=eRzO_SQNfxBz-vIF9PdTVFAzuAb2zfuCnKSKY_xdKQY,2491
 eval_protocol/config.py,sha256=b6mWbw-IJhu8iWgCJqcFp14vPJl7iXH7FnKe7cNIgSc,6438
 eval_protocol/directory_utils.py,sha256=6sYCgXt84qrQk1fGNtOZKj_Z7z9XK1e5jyMDsbmaiAQ,1869
@@ -63,13 +63,13 @@ eval_protocol/benchmarks/test_tau_bench_retail.py,sha256=-rGNA_biYO4_3d9S2JMHTQ6
 eval_protocol/benchmarks/data/airline_dataset.jsonl,sha256=jVvKx51hy4eUSia5zXZRI_N5_X6RRHi192_YaaBH7n4,79086
 eval_protocol/benchmarks/data/retail_dataset.jsonl,sha256=9fGs1U6lvnpKBNjERWPmJHOnrO3cPoDtGwvdxv11mD4,179853
 eval_protocol/cli_commands/__init__.py,sha256=XYAFq-qJ787pmrMiouG_jd9uwZcRhxLtNLSTxKcHzCM,85
-eval_protocol/cli_commands/agent_eval_cmd.py,sha256=7ouxAWU-1OzcR3O_MuCEvCy1oLy8dxNPuNmKrl7YqRs,11201
+eval_protocol/cli_commands/agent_eval_cmd.py,sha256=ezENJzLsUHvpm-FXrDpIV4fBHZbr3WqQOQiIL_rcVEw,11032
 eval_protocol/cli_commands/common.py,sha256=UuV5AZ02Rp6oioOblr9mfIrFcfTAKJOuxzhOqadBcHE,10155
 eval_protocol/cli_commands/deploy.py,sha256=WwwyK80exi1dBFjcIXEj_odaGOfe1byiWgnLGiUUQvM,24054
 eval_protocol/cli_commands/deploy_mcp.py,sha256=_6S7NcbCb_hGVqaBUgKLURO53i2UAI7xXUkBVXP6cLo,9961
 eval_protocol/cli_commands/logs.py,sha256=gh-HIQ51SH8nyvmdpHBfzFSJg7TnL0QwvVN9vpDa-20,799
 eval_protocol/cli_commands/preview.py,sha256=1iXru9V8QBGx7tLDJ28iG7bjV2ICFroEKaQ8EJsGciU,8072
-eval_protocol/cli_commands/run_eval_cmd.py,sha256=rJn-pGlPbWcXljywH78kif7r_m_CW84LjGa7xyKgNvo,9725
+eval_protocol/cli_commands/run_eval_cmd.py,sha256=xLf7Adfjt-5jmfKHh9ioay3n0MDo78tofeZgTnZ-C7w,9730
 eval_protocol/cli_commands/upload.py,sha256=al9CU4syDf5m0KOtlIol3tDb8WSMibNZV4-fqYqBEUU,19776
 eval_protocol/data_loader/__init__.py,sha256=tUntIfAZ0D2yjwfYMLzkDvdDsWqsnYS73JtngubrjFo,153
 eval_protocol/data_loader/dynamic_data_loader.py,sha256=tMSqKiECls1ySHv5eHpAlc9PwsRLzE5Rp4Ko3kIYUuA,1442
@@ -99,8 +99,9 @@ eval_protocol/integrations/__init__.py,sha256=mTs7Wypur0uG9r0GSE-MtdIx0eXZjecplm
 eval_protocol/integrations/deepeval.py,sha256=BZsm_EYKFIIoJdLjp29tcpKnSWpGB0x1uuf3LxsoBjc,4837
 eval_protocol/integrations/openeval.py,sha256=bHn0PRvtHMWmrKbGHqZOvJfx7lmvjt4vzeGdx6oqmck,1443
 eval_protocol/integrations/trl.py,sha256=yaF7GAq9xRhIhNvr0m0lul-_WZ8H6uiY_e3qDCUQYMY,9404
-eval_protocol/logging/elasticsearch_direct_http_handler.py,sha256=x6Y_BUE9KHsA7btS0EdPmTaJPIAZnHmuZ68p7tslHw0,3962
-eval_protocol/logging/elasticsearch_index_manager.py,sha256=ddeXO8zyXIkw27m9aVKLb1UJzUdxk39zGIU9BXAGeTA,6806
+eval_protocol/logging/elasticsearch_client.py,sha256=OhecOYRF2fkneDFGRnvTHgsXtbP3Z3rfd3BXeZzauNA,8857
+eval_protocol/logging/elasticsearch_direct_http_handler.py,sha256=ZYfgKod54pLMw3f__u6m8tHdSgojNS-4JmWiSLz-ewk,5503
+eval_protocol/logging/elasticsearch_index_manager.py,sha256=nZvieLATW_QLy1d6DL824JYY7V04IxSal--IGYCS1R4,6339
 eval_protocol/mcp/__init__.py,sha256=H2lau7Ch0-CBTy0zFBeyKJ-a1lHxia6WlXWLlu-rflM,1353
 eval_protocol/mcp/adapter.py,sha256=aXETSXkOKFg8QdYcfcpJoDMqdwgmaEKrjCkHui-aO3o,4020
 eval_protocol/mcp/clients.py,sha256=K00KrQdaSnHy0SgSQzjAmJjhHVnnaKT7tqtgAeV13lY,8982
@@ -146,7 +147,7 @@ eval_protocol/pytest/default_no_op_rollout_processor.py,sha256=d6Uh-lRQVtLHbb2BC
 eval_protocol/pytest/default_pydantic_ai_rollout_processor.py,sha256=L_gFvDNTKr9cFV8ngpg-iIrEycB79nUVowRvPkxysm8,7746
 eval_protocol/pytest/default_single_turn_rollout_process.py,sha256=hG8a605vOuzd-ITcpFOU2ysU2v8-Bshs-LyLeM4Tfig,5906
 eval_protocol/pytest/dual_mode_wrapper.py,sha256=wMRD0G4-GpFvh2ZaBxhcrRdNSxCWIi0RqhZ5eBJVMio,3969
-eval_protocol/pytest/elasticsearch_setup.py,sha256=3GJmohHfsLsP9gAVfioqwI7C6aONEWAK1XSkVTdEUxQ,7057
+eval_protocol/pytest/elasticsearch_setup.py,sha256=6NEljVFyeyrJXfS0TUmq1kEoJGr5vJZ4_Kw00UmrJes,7057
 eval_protocol/pytest/evaluation_test.py,sha256=tQ_lPrzb14tYOuhIPhHJYDRTk_4PYdzFHb7CIJ9j628,34729
 eval_protocol/pytest/evaluation_test_postprocess.py,sha256=OyLTqvHDgvV4cbzyJnH0AC_L_k3vepe8XQus0ZTtgLE,9449
 eval_protocol/pytest/exception_config.py,sha256=VvoVmUYkWZi_ZtpdFyTtJlHjpdtr-MbvW5fKZzLpRUQ,4219
@@ -155,7 +156,7 @@ eval_protocol/pytest/generate_parameter_combinations.py,sha256=LTVn6zxR4pPdKB-fp
 eval_protocol/pytest/handle_persist_flow.py,sha256=YBqXEP8gZ6noJK1AGOONpRRZMB3dQNEZLpM1kEHt7CE,10925
 eval_protocol/pytest/parameterize.py,sha256=Ry74Kr_ciSLcVg1AeKIKUNqnlkjZZ4xuIv5kmd_mm3s,16091
 eval_protocol/pytest/plugin.py,sha256=TxnWls2plEbm5EJx3gJRBNqkDMv-paWdjg0wyM9-fb8,14371
-eval_protocol/pytest/remote_rollout_processor.py,sha256=wDkdFthkbEW-LruG6rKXVPYi2M_YP3f7ZaMBSbAhWh8,11744
+eval_protocol/pytest/remote_rollout_processor.py,sha256=NjYJUpWT_1Wk8h_W5LHQfcMS6UKm9Uj9FraO4qgB8Ew,11977
 eval_protocol/pytest/rollout_processor.py,sha256=eKm44_sNU3yIVcfk7xsIUTXvAnCzjd0CNSw6PuYQAGw,791
 eval_protocol/pytest/store_experiment_link.py,sha256=Sb7DTXrDUcPUp1LBE746R7vulyEtDszPPTLa21AzFgs,1447
 eval_protocol/pytest/store_results_url.py,sha256=SmAPVDD--u_hZ6GuB_L3GVWUvKxkczF1etjRTnztqu0,1726
@@ -196,7 +197,7 @@ eval_protocol/stats/__init__.py,sha256=FtH_CJCY4SLhJeTJmBrH4ISYJX8PpRYpL4R8NQSeU
 eval_protocol/stats/confidence_intervals.py,sha256=zreh2-8dTq2OsiShMBZezezu6NobskXqXrznMS6srm0,4007
 eval_protocol/types/__init__.py,sha256=UisnRwa-7haDyLZg6U6qiVNWh3eJ-xmsPk5nNycOtkY,243
 eval_protocol/types/errors.py,sha256=4SZFJbXaruWg6nsTLg2HezEP5MNA9zJSrCxLnaxUfN4,514
-eval_protocol/types/remote_rollout_processor.py,sha256=BYCaMz-rRWwqLzuvLQBxd6tqdpeFtaMt9T8XgrsMOBA,1801
+eval_protocol/types/remote_rollout_processor.py,sha256=2S9PgcZC7lTWym9EvIzIWlSZOfrU11llkuI_jlWwS1U,2016
 eval_protocol/types/types.py,sha256=FPpnMu8Q1DTyq3EcG1WtpkHaDMPuodG1zNpqnExIPbw,3523
 eval_protocol/utils/__init__.py,sha256=bP-yexMWhOz40d2-vI8ZvA84uZOL3o0XWilc_H5dT8g,429
 eval_protocol/utils/batch_evaluation.py,sha256=tQfDmvsgTPT8H9t3j-cIpmkHXP2ib2iyLbvI8Yx1xKY,9260
@@ -210,7 +211,7 @@ eval_protocol/utils/show_results_url.py,sha256=PHM6dWtCUiuV5WQgvHegnxY7ofkE4b9wO
 eval_protocol/utils/static_policy.py,sha256=fiKnOS06EG5OB6p5An_yY_dLAvVboYnC4Sqx5z_v3-g,10716
 eval_protocol/utils/subprocess_utils.py,sha256=2EcoVNLSlfdxwQn-2pscqjiGpBR4Ho8kfRnmzmew-1w,3504
 eval_protocol/utils/vite_server.py,sha256=oKUEfOYbNQoq90i7nrk1xiDWpfK_9BS5eMNe0QEBSi4,5064
-eval_protocol-0.2.35.dev1.dist-info/licenses/LICENSE,sha256=OzeIb507xW9AVhGMqqHpoL_EFRJUo8Sb7A3LN5NqFfQ,1075
+eval_protocol-0.2.35.dev2.dist-info/licenses/LICENSE,sha256=OzeIb507xW9AVhGMqqHpoL_EFRJUo8Sb7A3LN5NqFfQ,1075
 vendor/tau2/__init__.py,sha256=EQMX_v8x-YBV24ia35_nLkf5MrC6aAuT_M5m7IJcl3k,541
 vendor/tau2/cli.py,sha256=lhJocXCDxEfdv7gIxya5b0w5J5qebpgrg_ZTpjGp_ww,7515
 vendor/tau2/config.py,sha256=LrkKRGSFH4Cvf9CNO-MttJMvIia0a2zP1uKVnUQi6B8,1278
@@ -310,8 +311,8 @@ vite-app/dist/assets/index-C8woq7EO.js,sha256=UQiM5fwp8CGPSEnS5uAUQBzyOhfpTiMz8L
 vite-app/dist/assets/index-C8woq7EO.js.map,sha256=s3rEcnsitUjfuvHIg7nl4UHHG4wNIySAT3s-2YGDBFY,3849814
 vite-app/dist/assets/index-CSKGq1w7.css,sha256=o0IZC5dbIyMb7S9-gw8t5eK60RCXf3jsEPx9q2CYOAQ,24505
 vite-app/dist/assets/logo-light-BprIBJQW.png,sha256=rRXC24eqrQO3y--N493THrD48WQVAhSVMHM_iDKy250,21694
-eval_protocol-0.2.35.dev1.dist-info/METADATA,sha256=OSa-5gSfprsnvVRX8dDw_9MhO7fRqRrkiMkEabJ4Fuo,7450
-eval_protocol-0.2.35.dev1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-eval_protocol-0.2.35.dev1.dist-info/entry_points.txt,sha256=CebRaxbWXly21zPN1fbyAw26kNUU2dv7zZyGkXxtFVw,183
-eval_protocol-0.2.35.dev1.dist-info/top_level.txt,sha256=8jjn7dpvLPL4RX2JBeAfPPMOR6x6f7E4o4yFiKLEHuw,33
-eval_protocol-0.2.35.dev1.dist-info/RECORD,,
+eval_protocol-0.2.35.dev2.dist-info/METADATA,sha256=U1rm2RbgpeOYYvJ8CuhE6GrvWe0CyjBe2DN9iL2bAC0,7450
+eval_protocol-0.2.35.dev2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+eval_protocol-0.2.35.dev2.dist-info/entry_points.txt,sha256=CebRaxbWXly21zPN1fbyAw26kNUU2dv7zZyGkXxtFVw,183
+eval_protocol-0.2.35.dev2.dist-info/top_level.txt,sha256=8jjn7dpvLPL4RX2JBeAfPPMOR6x6f7E4o4yFiKLEHuw,33
+eval_protocol-0.2.35.dev2.dist-info/RECORD,,