eval-protocol 0.2.39__tar.gz → 0.2.41__tar.gz

{eval_protocol-0.2.39/eval_protocol.egg-info → eval_protocol-0.2.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.39
+Version: 0.2.41
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.39 → eval_protocol-0.2.41}/eval_protocol/__init__.py

@@ -32,12 +32,15 @@ from .typed_interface import reward_function
 from .quickstart import aha_judge, multi_turn_assistant_to_ground_truth, assistant_to_ground_truth
 from .pytest import evaluation_test, SingleTurnRolloutProcessor, RemoteRolloutProcessor
 from .pytest.parameterize import DefaultParameterIdGenerator
+from .log_utils.elasticsearch_direct_http_handler import ElasticsearchDirectHttpHandler
+from .log_utils.rollout_id_filter import RolloutIdFilter
 
 from .types.remote_rollout_processor import (
     InitRequest,
     RolloutMetadata,
     StatusResponse,
     create_langfuse_config_tags,
+    DataLoaderConfig,
 )
 
 try:
@@ -63,6 +66,9 @@ except ImportError:
 warnings.filterwarnings("default", category=DeprecationWarning, module="eval_protocol")
 
 __all__ = [
+    "ElasticsearchDirectHttpHandler",
+    "RolloutIdFilter",
+    "DataLoaderConfig",
     "Status",
     "RemoteRolloutProcessor",
     "InputMetadata",

{eval_protocol-0.2.39 → eval_protocol-0.2.41}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-03T13:05:13-0700",
+ "date": "2025-10-07T15:43:37-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "d88e1553ce310279cab0261df1035f27713e2e5f",
- "version": "0.2.39"
+ "full-revisionid": "289abc56fc56935b45b11da011712fe48d956af1",
+ "version": "0.2.41"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.39 → eval_protocol-0.2.41}/eval_protocol/adapters/__init__.py

@@ -6,6 +6,7 @@ and converting them to EvaluationRow format for use in evaluation pipelines.
 Available adapters:
 - BaseAdapter: Abstract base class for all adapters
 - LangfuseAdapter: Pull data from Langfuse deployments
+- FireworksTracingAdapter: Pull data from Langfuse via Fireworks tracing proxy
 - HuggingFaceAdapter: Load datasets from HuggingFace Hub
 - BigQueryAdapter: Query data from Google BigQuery
 - TRL integration (legacy)
@@ -24,6 +25,10 @@ try:
 except ImportError:
     pass
 
+from .fireworks_tracing import FireworksTracingAdapter
+
+__all__.extend(["FireworksTracingAdapter"])
+
 try:
     from .huggingface import (
         HuggingFaceAdapter,

eval_protocol-0.2.41/eval_protocol/adapters/fireworks_tracing.py

@@ -0,0 +1,415 @@
+"""Fireworks Tracing adapter for Eval Protocol.
+
+This adapter uses the Fireworks tracing proxy at tracing.fireworks.ai
+to pull data from Langfuse deployments with simplified retry logic handling.
+"""
+
+from __future__ import annotations
+import logging
+import requests
+import time
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Protocol
+
+from eval_protocol.models import EvaluationRow, InputMetadata, ExecutionMetadata, Message
+from .base import BaseAdapter
+from .utils import extract_messages_from_data
+
+logger = logging.getLogger(__name__)
+
+
+class TraceDictConverter(Protocol):
+    """Protocol for custom trace dictionary-to-EvaluationRow converter functions.
+
+    A converter function should take a trace dictionary along with processing
+    options and return an EvaluationRow or None to skip the trace.
+    """
+
+    def __call__(
+        self,
+        trace: Dict[str, Any],
+        include_tool_calls: bool,
+        span_name: Optional[str],
+    ) -> Optional[EvaluationRow]:
+        """Convert a trace dictionary to an EvaluationRow.
+
+        Args:
+            trace: The trace dictionary to convert
+            include_tool_calls: Whether to include tool calling information
+            span_name: Optional span name to extract messages from
+
+        Returns:
+            EvaluationRow or None if the trace should be skipped
+        """
+        ...
+
+
+def convert_trace_dict_to_evaluation_row(
+    trace: Dict[str, Any], include_tool_calls: bool = True, span_name: Optional[str] = None
+) -> Optional[EvaluationRow]:
+    """Convert a trace dictionary (from proxy API) to EvaluationRow format.
+
+    Args:
+        trace: Trace dictionary from Fireworks proxy API
+        include_tool_calls: Whether to include tool calling information
+        span_name: If provided, extract messages from generations within this named span
+
+    Returns:
+        EvaluationRow or None if conversion fails
+    """
+    try:
+        # Extract messages from trace input and output
+        messages = extract_messages_from_trace_dict(trace, include_tool_calls, span_name)
+
+        # Extract tools if available
+        tools = None
+        if include_tool_calls and isinstance(trace.get("input"), dict) and "tools" in trace["input"]:
+            tools = trace["input"]["tools"]
+
+        if not messages:
+            return None
+
+        execution_metadata = ExecutionMetadata()
+        row_id = None
+
+        # Extract metadata from tags
+        tags = trace.get("tags", [])
+        if tags:
+            for tag in tags:
+                if tag.startswith("invocation_id:"):
+                    execution_metadata.invocation_id = tag.split(":", 1)[1]
+                elif tag.startswith("experiment_id:"):
+                    execution_metadata.experiment_id = tag.split(":", 1)[1]
+                elif tag.startswith("rollout_id:"):
+                    execution_metadata.rollout_id = tag.split(":", 1)[1]
+                elif tag.startswith("run_id:"):
+                    execution_metadata.run_id = tag.split(":", 1)[1]
+                elif tag.startswith("row_id:"):
+                    row_id = tag.split(":", 1)[1]
+
+                if (
+                    execution_metadata.invocation_id
+                    and execution_metadata.experiment_id
+                    and execution_metadata.rollout_id
+                    and execution_metadata.run_id
+                    and row_id
+                ):
+                    break  # Break early if we've found all the metadata we need
+
+        return EvaluationRow(
+            messages=messages,
+            tools=tools,
+            input_metadata=InputMetadata(
+                row_id=row_id,
+                session_data={
+                    "langfuse_trace_id": trace.get("id"),  # Store the trace ID here
+                },
+            ),
+            execution_metadata=execution_metadata,
+        )
+
+    except (AttributeError, ValueError, KeyError) as e:
+        logger.error("Error converting trace %s: %s", trace.get("id"), e)
+        return None
+
+
+def extract_messages_from_trace_dict(
+    trace: Dict[str, Any], include_tool_calls: bool = True, span_name: Optional[str] = None
+) -> List[Message]:
+    """Extract messages from trace dictionary.
+
+    Args:
+        trace: Trace dictionary from proxy API
+        include_tool_calls: Whether to include tool calling information
+        span_name: If provided, extract messages from generations within this named span
+
+    Returns:
+        List of Message objects
+    """
+    messages = []
+
+    if span_name:  # Look for a generation tied to a span name
+        try:
+            # Find the final generation in the named span
+            gen = get_final_generation_in_span_dict(trace, span_name)
+            if not gen:
+                return messages
+
+            # Extract messages from generation input and output
+            if gen.get("input"):
+                messages.extend(extract_messages_from_data(gen["input"], include_tool_calls))
+            if gen.get("output"):
+                messages.extend(extract_messages_from_data(gen["output"], include_tool_calls))
+
+            return messages
+
+        except Exception as e:
+            logger.error("Failed to extract messages from span '%s' in trace %s: %s", span_name, trace.get("id"), e)
+            return messages
+
+    else:
+        try:
+            # Extract messages from trace input and output
+            if trace.get("input"):
+                messages.extend(extract_messages_from_data(trace["input"], include_tool_calls))
+            if trace.get("output"):
+                messages.extend(extract_messages_from_data(trace["output"], include_tool_calls))
+        except (AttributeError, ValueError, KeyError) as e:
+            logger.warning("Error processing trace %s: %s", trace.get("id"), e)
+
+        # Fallback: use the last GENERATION observation which typically contains full chat history
+        if not messages:
+            try:
+                all_observations = trace.get("observations", [])
+                gens = [obs for obs in all_observations if obs.get("type") == "GENERATION"]
+                if gens:
+                    gens.sort(key=lambda x: x.get("start_time", ""))
+                    last_gen = gens[-1]
+                    if last_gen.get("input"):
+                        messages.extend(extract_messages_from_data(last_gen["input"], include_tool_calls))
+                    if last_gen.get("output"):
+                        messages.extend(extract_messages_from_data(last_gen["output"], include_tool_calls))
+            except Exception as e:
+                logger.warning("Failed to extract from last generation for trace %s: %s", trace.get("id"), e)
+
+    return messages
+
+
+def get_final_generation_in_span_dict(trace: Dict[str, Any], span_name: str) -> Optional[Dict[str, Any]]:
+    """Get the final generation within a named span from trace dictionary.
+
+    Args:
+        trace: Trace dictionary
+        span_name: Name of the span to search for
+
+    Returns:
+        The final generation dictionary, or None if not found
+    """
+    # Get all observations from the trace
+    all_observations = trace.get("observations", [])
+
+    # Find a span with the given name that has generation children
+    parent_span = None
+    for obs in all_observations:
+        if obs.get("name") == span_name and obs.get("type") == "SPAN":
+            # Check if this span has generation children
+            has_generations = any(
+                child.get("type") == "GENERATION" and child.get("parent_observation_id") == obs.get("id")
+                for child in all_observations
+            )
+            if has_generations:
+                parent_span = obs
+                break
+
+    if not parent_span:
+        logger.warning("No span named '%s' found in trace %s", span_name, trace.get("id"))
+        return None
+
+    # Find all generations within this span
+    generations = []
+    for obs in all_observations:
+        if obs.get("type") == "GENERATION" and obs.get("parent_observation_id") == parent_span.get("id"):
+            generations.append(obs)
+
+    if not generations:
+        logger.warning("No generations found in span '%s' in trace %s", span_name, trace.get("id"))
+        return None
+
+    # Sort generations by start time for chronological order
+    generations.sort(key=lambda x: x.get("start_time", ""))
+
+    # Return the final generation (contains full message history)
+    return generations[-1]
+
+
+class FireworksTracingAdapter(BaseAdapter):
+    """Adapter to pull data from Langfuse via Fireworks tracing proxy.
+
+    This adapter uses the Fireworks tracing proxy API which handles retry logic
+    and rate limiting internally, simplifying the client-side implementation.
+
+    Examples:
+        Basic usage (default project):
+        >>> adapter = FireworksTracingAdapter()
+        >>> rows = list(adapter.get_evaluation_rows(tags=["rollout_id:xyz"], limit=10))
+
+        With explicit project ID:
+        >>> adapter = FireworksTracingAdapter(
+        ...     project_id="your_project_id",
+        ...     base_url="https://tracing.fireworks.ai"
+        ... )
+        >>> rows = list(adapter.get_evaluation_rows(tags=["production"], limit=10))
+
+        Filter by specific criteria:
+        >>> rows = list(adapter.get_evaluation_rows(
+        ...     tags=["production"],
+        ...     limit=50,
+        ...     hours_back=24
+        ... ))
+    """
+
+    def __init__(
+        self,
+        project_id: Optional[str] = None,
+        base_url: str = "https://tracing.fireworks.ai",
+        timeout: int = 300,
+    ):
+        """Initialize the Fireworks Tracing adapter.
+
+        Args:
+            project_id: Optional project ID. If not provided, uses the default project configured on the server.
+            base_url: The base URL of the tracing proxy (default: https://tracing.fireworks.ai)
+            timeout: Request timeout in seconds (default: 300)
+        """
+        self.project_id = project_id
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+
+    def get_evaluation_rows(
+        self,
+        tags: List[str],
+        limit: int = 100,
+        sample_size: Optional[int] = None,
+        user_id: Optional[str] = None,
+        session_id: Optional[str] = None,
+        name: Optional[str] = None,
+        environment: Optional[str] = None,
+        version: Optional[str] = None,
+        release: Optional[str] = None,
+        fields: Optional[str] = None,
+        hours_back: Optional[int] = None,
+        from_timestamp: Optional[datetime] = None,
+        to_timestamp: Optional[datetime] = None,
+        include_tool_calls: bool = True,
+        backend_sleep_between_gets: float = 0.1,
+        backend_max_retries: int = 3,
+        proxy_max_retries: int = 3,
+        span_name: Optional[str] = None,
+        converter: Optional[TraceDictConverter] = None,
+    ) -> List[EvaluationRow]:
+        """Pull traces from Langfuse via proxy and convert to EvaluationRow format.
+
+        Args:
+            tags: REQUIRED - Filter by specific tags (prevents fetching all traces).
+                Must provide at least one tag (e.g., ['rollout_id:xyz'], ['production'])
+            limit: Max number of trace summaries to collect via pagination
+            sample_size: Optional number of traces to randomly sample (if None, process all)
+            user_id: Filter by user ID
+            session_id: Filter by session ID
+            name: Filter by trace name
+            environment: Filter by environment (e.g., production, staging, development)
+            version: Filter by trace version
+            release: Filter by trace release
+            fields: Comma-separated list of fields to include
+            hours_back: Filter traces from this many hours ago
+            from_timestamp: Explicit start time (ISO format)
+            to_timestamp: Explicit end time (ISO format)
+            include_tool_calls: Whether to include tool calling traces
+            backend_sleep_between_gets: Sleep time between backend trace fetches (passed to proxy)
+            backend_max_retries: Maximum retries for backend operations (passed to proxy)
+            proxy_max_retries: Maximum retries when proxy returns 404 (client-side retries with exponential backoff)
+            span_name: If provided, extract messages from generations within this named span
+            converter: Optional custom converter implementing TraceDictConverter protocol.
+                If provided, this will be used instead of the default conversion logic.
+
+        Returns:
+            List[EvaluationRow]: Converted evaluation rows
+
+        Raises:
+            ValueError: If tags list is empty
+        """
+        # Validate that tags are provided (security requirement)
+        if not tags or len(tags) == 0:
+            raise ValueError("At least one tag is required to fetch traces (security: prevents fetching all traces)")
+
+        eval_rows = []
+
+        # Build query parameters for GET request
+        params = {
+            "limit": limit,
+            "sample_size": sample_size,
+            "tags": tags,
+            "user_id": user_id,
+            "session_id": session_id,
+            "name": name,
+            "environment": environment,
+            "version": version,
+            "release": release,
+            "fields": fields,
+            "hours_back": hours_back,
+            "from_timestamp": from_timestamp.isoformat() if from_timestamp else None,
+            "to_timestamp": to_timestamp.isoformat() if to_timestamp else None,
+            "sleep_between_gets": backend_sleep_between_gets,
+            "max_retries": backend_max_retries,
+        }
+
+        # Remove None values
+        params = {k: v for k, v in params.items() if v is not None}
+
+        # Make request to proxy with retry logic
+        if self.project_id:
+            url = f"{self.base_url}/v1/project_id/{self.project_id}/traces"
+        else:
+            url = f"{self.base_url}/v1/traces"
+
+        # Retry loop for handling backend indexing delays (proxy returns 404)
+        result = None
+        for attempt in range(proxy_max_retries):
+            try:
+                response = requests.get(url, params=params, timeout=self.timeout)
+                response.raise_for_status()
+                result = response.json()
+                break  # Success, exit retry loop
+            except requests.exceptions.HTTPError as e:
+                error_msg = str(e)
+                should_retry = False
+
+                # Try to extract detail message from response
+                if e.response is not None:
+                    try:
+                        error_detail = e.response.json().get("detail", "")
+                        error_msg = error_detail or e.response.text
+
+                        # Retry on 404 if it's due to incomplete/missing traces (backend still indexing)
+                        if e.response.status_code == 404 and (
+                            "Incomplete traces" in error_detail or "No traces found" in error_detail
+                        ):
+                            should_retry = True
+                    except Exception:
+                        error_msg = e.response.text
+
+                if should_retry and attempt < proxy_max_retries - 1:
+                    sleep_time = 2 ** (attempt + 1)
+                    logger.warning(error_msg)
+                    time.sleep(sleep_time)
+                else:
+                    # Final retry or non-retryable error
+                    logger.error("Failed to fetch traces from proxy: %s", error_msg)
+                    return eval_rows
+            except requests.exceptions.RequestException as e:
+                # Non-HTTP errors (network issues, timeouts, etc.)
+                logger.error("Failed to fetch traces from proxy: %s", str(e))
+                return eval_rows
+
+        if result is None:
+            logger.error("Failed to fetch traces after %d retries", proxy_max_retries)
+            return eval_rows
+
+        # Extract traces from response
+        traces = result.get("traces", [])
+
+        # Convert each trace to EvaluationRow
+        for trace in traces:
+            try:
+                if converter:
+                    eval_row = converter(trace, include_tool_calls, span_name)
+                else:
+                    eval_row = convert_trace_dict_to_evaluation_row(trace, include_tool_calls, span_name)
+                if eval_row:
+                    eval_rows.append(eval_row)
+            except (AttributeError, ValueError, KeyError) as e:
+                logger.warning("Failed to convert trace %s: %s", trace.get("id"), e)
+                continue
+
+        logger.info("Successfully converted %d traces to evaluation rows", len(eval_rows))
+        return eval_rows

{eval_protocol-0.2.39 → eval_protocol-0.2.41}/eval_protocol/adapters/langfuse.py

@@ -355,7 +355,7 @@ class LangfuseAdapter(BaseAdapter):
 
                     # If no results, possible due to indexing delay--remote rollout processor just finished pushing rows to Langfuse
                     if traces and traces.meta and traces.meta.total_items == 0 and page == 1:
-                        raise Exception("Empty results - indexing delay")
+                        raise Exception("Empty results")
 
                     break
                 except Exception as e:

{eval_protocol-0.2.39/eval_protocol/logging → eval_protocol-0.2.41/eval_protocol/log_utils}/elasticsearch_client.py

@@ -6,10 +6,9 @@ used throughout the codebase, including index management, document operations,
 and search functionality.
 """
 
-import json
 import requests
 from typing import Any, Dict, List, Optional, Union
-from urllib.parse import urlparse
+from eval_protocol.models import Status
 from eval_protocol.types.remote_rollout_processor import ElasticsearchConfig
 
 
@@ -203,33 +202,39 @@ class ElasticsearchClient:
         except Exception:
             return None
 
-    def search_by_term(self, field: str, value: Any, size: int = 10) -> Optional[Dict[str, Any]]:
+    def search_by_term(
+        self, field: str, value: Any, size: int = 10, sort: Optional[List[Dict[str, Any]]] = None
+    ) -> Optional[Dict[str, Any]]:
         """Search documents by exact term match.
 
         Args:
             field: Field name to search
             value: Value to match
             size: Number of results to return
+            sort: Sort specification
 
         Returns:
             Dict containing search results, or None if failed
         """
         query = {"term": {field: value}}
-        return self.search(query, size=size)
+        return self.search(query, size=size, sort=sort)
 
-    def search_by_match(self, field: str, value: str, size: int = 10) -> Optional[Dict[str, Any]]:
+    def search_by_match(
+        self, field: str, value: str, size: int = 10, sort: Optional[List[Dict[str, Any]]] = None
+    ) -> Optional[Dict[str, Any]]:
         """Search documents by text match.
 
         Args:
             field: Field name to search
             value: Text to match
             size: Number of results to return
+            sort: Sort specification (e.g., [{"@timestamp": {"order": "desc"}}])
 
         Returns:
             Dict containing search results, or None if failed
         """
         query = {"match": {field: value}}
-        return self.search(query, size=size)
+        return self.search(query, size=size, sort=sort)
 
     def search_by_match_phrase_prefix(self, field: str, value: str, size: int = 10) -> Optional[Dict[str, Any]]:
         """Search documents by phrase prefix match.
@@ -257,6 +262,34 @@ class ElasticsearchClient:
         query = {"match_all": {}}
         return self.search(query, size=size)
 
+    def search_by_status_code_not_in(
+        self,
+        rollout_id: str,
+        excluded_codes: list[Status.Code],
+        size: int = 10,
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Search documents where status_code does NOT match any of the provided status codes.
+
+        Args:
+            excluded_codes: List of status codes to exclude (i.e., find logs NOT having these codes)
+            size: Number of results to return
+            rollout_id: Optional rollout ID to filter by
+
+        Returns:
+            Dict containing search results, or None if failed
+        """
+        # Build the query with must_not for status code exclusion
+        bool_query: dict[str, list[dict[str, Any]]] = {
+            "must_not": [{"terms": {"status_code": [code.value for code in excluded_codes]}}]
+        }
+
+        # Add rollout_id filter and ensure status_code exists
+        bool_query["must"] = [{"term": {"rollout_id": rollout_id}}, {"exists": {"field": "status_code"}}]
+
+        query = {"bool": bool_query}
+        return self.search(query, size=size)
+
     # Health and Status Operations
 
     def health_check(self) -> bool:

{eval_protocol-0.2.39/eval_protocol/logging → eval_protocol-0.2.41/eval_protocol/log_utils}/elasticsearch_direct_http_handler.py

@@ -1,28 +1,38 @@
-import json
 import logging
-import asyncio
 import os
-import threading
 from concurrent.futures import ThreadPoolExecutor
-from typing import Optional, Tuple, Any, Dict
+from typing import Optional, Any, Dict
 from datetime import datetime
 
 from eval_protocol.types.remote_rollout_processor import ElasticsearchConfig
 from .elasticsearch_client import ElasticsearchClient
 
+import logging
+
+logger = logging.getLogger(__name__)
+
+# do not inherit root logger since we are a handler ourselves
+logger.propagate = False
+
+logger.addHandler(logging.StreamHandler())
+
+if os.environ.get("EP_DEBUG") == "true":
+    logger.setLevel(logging.DEBUG)
+    logger.debug("EP_DEBUG=true detected, set log level to DEBUG")
+
 
 class ElasticsearchDirectHttpHandler(logging.Handler):
-    def __init__(self, elasticsearch_config: ElasticsearchConfig) -> None:
+    def __init__(self, elasticsearch_config: ElasticsearchConfig | None = None) -> None:
         super().__init__()
-        self.config = ElasticsearchConfig(
-            url=elasticsearch_config.url,
-            api_key=elasticsearch_config.api_key,
-            index_name=elasticsearch_config.index_name,
-        )
-        self.client = ElasticsearchClient(self.config)
+        self.config = elasticsearch_config
+        self.client = ElasticsearchClient(self.config) if self.config else None
         self.formatter: logging.Formatter = logging.Formatter()
         self._executor = None
 
+    def configure(self, elasticsearch_config: ElasticsearchConfig) -> None:
+        self.config = elasticsearch_config
+        self.client = ElasticsearchClient(self.config)
+
     def emit(self, record: logging.LogRecord) -> None:
         """Emit a log record by scheduling it for async transmission."""
         try:
@@ -30,6 +40,12 @@ class ElasticsearchDirectHttpHandler(logging.Handler):
             timestamp = datetime.fromtimestamp(record.created).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
 
             rollout_id = self._get_rollout_id(record)
+            logger.debug(f"Emitting log record: {record.getMessage()} with rollout_id: {rollout_id}")
+            if not rollout_id:
+                logger.debug(
+                    "No rollout_id provided in extra data for ElasticsearchDirectHttpHandler through EP_ROLLOUT_ID environment variable or rollout_id extra data. Skipping log record."
+                )
+                return
             status_info = self._get_status_info(record)
 
             data: Dict[str, Any] = {
@@ -50,7 +66,7 @@ class ElasticsearchDirectHttpHandler(logging.Handler):
             self.handleError(record)
             print(f"Error preparing log for Elasticsearch: {e}")
 
-    def _get_rollout_id(self, record: logging.LogRecord) -> str:
+    def _get_rollout_id(self, record: logging.LogRecord) -> str | None:
         """Get the rollout ID from record extra data or environment variables."""
         # Check if rollout_id is provided in the extra data first
         if hasattr(record, "rollout_id") and record.rollout_id is not None:  # type: ignore
@@ -58,10 +74,6 @@ class ElasticsearchDirectHttpHandler(logging.Handler):
 
         # Fall back to environment variable
         rollout_id = os.getenv("EP_ROLLOUT_ID")
-        if rollout_id is None:
-            raise ValueError(
-                "EP_ROLLOUT_ID environment variable is not set and no rollout_id provided in extra data for ElasticsearchDirectHttpHandler"
-            )
         return rollout_id
 
     def _get_status_info(self, record: logging.LogRecord) -> Optional[Dict[str, Any]]:
@@ -105,6 +117,9 @@ class ElasticsearchDirectHttpHandler(logging.Handler):
 
     def _send_to_elasticsearch(self, data: Dict[str, Any], record: logging.LogRecord) -> None:
         """Send data to Elasticsearch (runs in thread pool)."""
+        if not self.client:
+            logger.warning("No Elasticsearch client configured, skipping log record")
+            return
         try:
             success = self.client.index_document(data)
             if not success: