eval-protocol 0.2.57__tar.gz → 0.2.57.dev2__tar.gz

{eval_protocol-0.2.57/eval_protocol.egg-info → eval_protocol-0.2.57.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.57
+Version: 0.2.57.dev2
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.57 → eval_protocol-0.2.57.dev2}/eval_protocol/__init__.py

@@ -36,7 +36,6 @@ from .utils.evaluation_row_utils import (
     filter_longest_conversation,
 )
 from .pytest import evaluation_test, SingleTurnRolloutProcessor, RemoteRolloutProcessor, GithubActionRolloutProcessor
-from .pytest.remote_rollout_processor import create_elasticsearch_config_from_env
 from .pytest.parameterize import DefaultParameterIdGenerator
 from .log_utils.elasticsearch_direct_http_handler import ElasticsearchDirectHttpHandler
 from .log_utils.rollout_id_filter import RolloutIdFilter
@@ -90,7 +89,6 @@ except ImportError:
 warnings.filterwarnings("default", category=DeprecationWarning, module="eval_protocol")
 
 __all__ = [
-    "create_elasticsearch_config_from_env",
     "ElasticsearchConfig",
     "ElasticsearchDirectHttpHandler",
     "RolloutIdFilter",

{eval_protocol-0.2.57 → eval_protocol-0.2.57.dev2}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-15T12:11:42-0700",
+ "date": "2025-10-21T14:44:45-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "e3a7e386b88f6bc402fb82042664cbb4ded44eca",
- "version": "0.2.57"
+ "full-revisionid": "5a0eb89e557f1362bc17acd8a02c25a072dc3092",
+ "version": "0.2.57-dev2"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.57 → eval_protocol-0.2.57.dev2}/eval_protocol/adapters/fireworks_tracing.py

@@ -265,6 +265,55 @@ class FireworksTracingAdapter(BaseAdapter):
         self.base_url = base_url.rstrip("/")
         self.timeout = timeout
 
+    def search_logs(self, tags: List[str], limit: int = 100, hours_back: int = 24) -> List[Dict[str, Any]]:
+        """Fetch logs from Fireworks tracing gateway /logs endpoint.
+
+        Returns entries with keys: timestamp, message, severity, tags.
+        """
+        if not tags:
+            raise ValueError("At least one tag is required to fetch logs")
+
+        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
+        params: Dict[str, Any] = {"tags": tags, "limit": limit, "hours_back": hours_back, "program": "eval_protocol"}
+
+        # Try /logs first, fall back to /v1/logs if not found
+        urls_to_try = [f"{self.base_url}/logs", f"{self.base_url}/v1/logs"]
+        data: Dict[str, Any] = {}
+        last_error: Optional[str] = None
+        for url in urls_to_try:
+            try:
+                response = requests.get(url, params=params, timeout=self.timeout, headers=headers)
+                if response.status_code == 404:
+                    # Try next variant
+                    last_error = f"404 for {url}"
+                    continue
+                response.raise_for_status()
+                data = response.json() or {}
+                break
+            except requests.exceptions.RequestException as e:
+                last_error = str(e)
+                continue
+        else:
+            # All attempts failed
+            if last_error:
+                logger.error("Failed to fetch logs from Fireworks (tried %s): %s", urls_to_try, last_error)
+            return []
+
+        entries: List[Dict[str, Any]] = data.get("entries", []) or []
+        # Normalize minimal shape
+        results: List[Dict[str, Any]] = []
+        for e in entries:
+            results.append(
+                {
+                    "timestamp": e.get("timestamp"),
+                    "message": e.get("message"),
+                    "severity": e.get("severity", "INFO"),
+                    "tags": e.get("tags", []),
+                    "status": e.get("status"),
+                }
+            )
+        return results
+
     def get_evaluation_rows(
         self,
         tags: List[str],

{eval_protocol-0.2.57 → eval_protocol-0.2.57.dev2}/eval_protocol/cli.py

@@ -307,6 +307,16 @@ def parse_args(args=None):
         action="store_true",
         help="Use env vars for Elasticsearch config (requires ELASTICSEARCH_URL, ELASTICSEARCH_API_KEY, ELASTICSEARCH_INDEX_NAME)",
     )
+    logs_parser.add_argument(
+        "--use-fireworks",
+        action="store_true",
+        help="Force Fireworks tracing backend for logs UI (overrides env auto-detection)",
+    )
+    logs_parser.add_argument(
+        "--use-elasticsearch",
+        action="store_true",
+        help="Force Elasticsearch backend for logs UI (overrides env auto-detection)",
+    )
 
     # Upload command
     upload_parser = subparsers.add_parser(

eval_protocol-0.2.57.dev2/eval_protocol/cli_commands/logs.py

@@ -0,0 +1,57 @@
+"""
+CLI command for serving logs with file watching and real-time updates.
+"""
+
+import sys
+from pathlib import Path
+
+import os
+from ..utils.logs_server import serve_logs
+
+
+def logs_command(args):
+    """Serve logs with file watching and real-time updates"""
+
+    port = args.port
+    print("🚀 Starting Eval Protocol Logs Server")
+    print(f"🌐 URL: http://localhost:{port}")
+    print(f"🔌 WebSocket: ws://localhost:{port}/ws")
+    print(f"👀 Watching paths: {['current directory']}")
+    print(f"🔍 Debug mode: {args.debug}")
+    print("Press Ctrl+C to stop the server")
+    print("-" * 50)
+
+    # Backend selection: Fireworks first when API key present, unless overridden
+    use_fireworks = False
+    if getattr(args, "use_fireworks", False):
+        use_fireworks = True
+    elif getattr(args, "use_elasticsearch", False):
+        use_fireworks = False
+    else:
+        use_fireworks = bool(os.environ.get("FIREWORKS_API_KEY"))
+
+    # Setup backend configs
+    elasticsearch_config = None
+    # Prefer explicit FW_TRACING_GATEWAY_BASE_URL, then GATEWAY_URL from env (remote validation),
+    # finally default to public tracing.fireworks.ai
+    fireworks_base_url = (
+        os.environ.get("FW_TRACING_GATEWAY_BASE_URL")
+        or os.environ.get("GATEWAY_URL")
+        or "https://tracing.fireworks.ai"
+    )
+
+    try:
+        serve_logs(
+            port=args.port,
+            elasticsearch_config=elasticsearch_config,
+            debug=args.debug,
+            backend="fireworks" if use_fireworks else "elasticsearch",
+            fireworks_base_url=fireworks_base_url if use_fireworks else None,
+        )
+        return 0
+    except KeyboardInterrupt:
+        print("\n🛑 Server stopped by user")
+        return 0
+    except Exception as e:
+        print(f"❌ Error starting server: {e}")
+        return 1

{eval_protocol-0.2.57 → eval_protocol-0.2.57.dev2}/eval_protocol/cli_commands/upload.py

@@ -267,32 +267,29 @@ def _parse_entry(entry: str, cwd: str) -> tuple[str, str]:
 def _resolve_entry_to_qual_and_source(entry: str, cwd: str) -> tuple[str, str]:
     target, func = _parse_entry(entry, cwd)
 
-    # Check if target looks like a file path
+    # Determine the file path to load
     if "/" in target or "\\" in target or os.path.exists(target):
-        # It's a file path - convert to absolute and load as module
+        # It's a file path - convert to absolute
         if not os.path.isabs(target):
             target = os.path.abspath(os.path.join(cwd, target))
-
         if not target.endswith(".py"):
             target = target + ".py"
-
         if not os.path.isfile(target):
             raise ValueError(f"File not found: {target}")
-
-        # Import module from file path
-        spec = importlib.util.spec_from_file_location(Path(target).stem, target)
-        if not spec or not spec.loader:
-            raise ValueError(f"Unable to load module from path: {target}")
-        module = importlib.util.module_from_spec(spec)
-        sys.modules[spec.name] = module
-        spec.loader.exec_module(module)  # type: ignore[attr-defined]
-        module_name = spec.name
         source_file_path = target
     else:
-        # Treat as module path (e.g., "my_package.my_module")
-        module_name = target
-        module = importlib.import_module(module_name)
-        source_file_path = getattr(module, "__file__", "") or ""
+        # Treat dotted name as a file path
+        dotted_as_path = target.replace(".", "/") + ".py"
+        source_file_path = os.path.join(cwd, dotted_as_path)
+
+    # Load the module from the file path
+    spec = importlib.util.spec_from_file_location(Path(source_file_path).stem, source_file_path)
+    if not spec or not spec.loader:
+        raise ValueError(f"Unable to load module from path: {source_file_path}")
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)  # type: ignore[attr-defined]
+    module_name = spec.name
 
     if not hasattr(module, func):
         raise ValueError(f"Function '{func}' not found in module '{module_name}'")
@@ -591,8 +588,7 @@ def upload_command(args: argparse.Namespace) -> int:
 
         print(f"\nUploading evaluator '{evaluator_id}' for {qualname.split('.')[-1]}...")
         try:
-            # Always treat as a single evaluator (single-metric) even if folder has helper modules
-            test_dir = os.path.dirname(source_file_path) if source_file_path else root
+            test_dir = root
             metric_name = os.path.basename(test_dir) or "metric"
             result = create_evaluation(
                 evaluator_id=evaluator_id,

eval_protocol-0.2.57.dev2/eval_protocol/event_bus/__init__.py

@@ -0,0 +1,25 @@
+# Global event bus instance - uses SqliteEventBus for cross-process functionality
+from eval_protocol.event_bus.event_bus import EventBus
+
+
+def _get_default_event_bus():
+    from eval_protocol.event_bus.sqlite_event_bus import SqliteEventBus
+
+    return SqliteEventBus()
+
+
+# Lazy property that creates the event bus only when accessed
+class _LazyEventBus(EventBus):
+    def __init__(self):
+        self._event_bus: EventBus | None = None
+
+    def _get_event_bus(self):
+        if self._event_bus is None:
+            self._event_bus = _get_default_event_bus()
+        return self._event_bus
+
+    def __getattr__(self, name):
+        return getattr(self._get_event_bus(), name)
+
+
+event_bus: EventBus = _LazyEventBus()

{eval_protocol-0.2.57 → eval_protocol-0.2.57.dev2}/eval_protocol/log_utils/elasticsearch_direct_http_handler.py

@@ -60,6 +60,17 @@ class ElasticsearchDirectHttpHandler(logging.Handler):
             if status_info:
                 data.update(status_info)
 
+            # Optional correlation enrichment
+            experiment_id = getattr(record, "experiment_id", None)
+            if experiment_id is not None:
+                data["experiment_id"] = experiment_id
+            run_id = getattr(record, "run_id", None)
+            if run_id is not None:
+                data["run_id"] = run_id
+            rollout_ids = getattr(record, "rollout_ids", None)
+            if rollout_ids is not None:
+                data["rollout_ids"] = rollout_ids
+
             # Schedule the HTTP request to run asynchronously
             self._schedule_async_send(data, record)
         except Exception as e:

eval_protocol-0.2.57.dev2/eval_protocol/log_utils/fireworks_tracing_http_handler.py

@@ -0,0 +1,138 @@
+import logging
+import os
+import threading
+from datetime import datetime, timezone
+from typing import Optional, Any, Dict, List, cast
+
+import requests
+
+
+class FireworksTracingHttpHandler(logging.Handler):
+    """Logging handler that posts structured logs to tracing.fireworks gateway /logs endpoint."""
+
+    def __init__(self, gateway_base_url: Optional[str] = None, rollout_id_env: str = "EP_ROLLOUT_ID") -> None:
+        super().__init__()
+        self.gateway_base_url = (
+            gateway_base_url or os.getenv("FW_TRACING_GATEWAY_BASE_URL") or "https://tracing.fireworks.ai"
+        )
+        self.rollout_id_env = rollout_id_env
+        self._session = requests.Session()
+        self._lock = threading.Lock()
+        # Include Authorization header if FIREWORKS_API_KEY is available
+        api_key = os.environ.get("FIREWORKS_API_KEY")
+        if api_key:
+            try:
+                self._session.headers.update({"Authorization": f"Bearer {api_key}"})
+            except Exception:
+                pass
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            if not self.gateway_base_url:
+                return
+            rollout_id = self._get_rollout_id(record)
+            if not rollout_id:
+                return
+            payload = self._build_payload(record, rollout_id)
+            base = self.gateway_base_url.rstrip("/")
+            url = f"{base}/logs"
+            # Optional debug prints to aid local diagnostics
+            if os.environ.get("EP_DEBUG") == "true":
+                try:
+                    tags_val = payload.get("tags")
+                    tags_len = len(tags_val) if isinstance(tags_val, list) else 0
+                    msg_val = payload.get("message")
+                    msg_preview = msg_val[:80] if isinstance(msg_val, str) else msg_val
+                    print(f"[FW_LOG] POST {url} rollout_id={rollout_id} tags={tags_len} msg={msg_preview}")
+                except Exception:
+                    pass
+            with self._lock:
+                resp = self._session.post(url, json=payload, timeout=5)
+            if os.environ.get("EP_DEBUG") == "true":
+                try:
+                    print(f"[FW_LOG] resp={resp.status_code}")
+                except Exception:
+                    pass
+            # Fallback to /v1/logs if /logs is not found
+            if resp is not None and getattr(resp, "status_code", None) == 404:
+                alt = f"{base}/v1/logs"
+                if os.environ.get("EP_DEBUG") == "true":
+                    try:
+                        tags_val = payload.get("tags")
+                        tags_len = len(tags_val) if isinstance(tags_val, list) else 0
+                        print(f"[FW_LOG] RETRY POST {alt} rollout_id={rollout_id} tags={tags_len}")
+                    except Exception:
+                        pass
+                with self._lock:
+                    resp2 = self._session.post(alt, json=payload, timeout=5)
+                if os.environ.get("EP_DEBUG") == "true":
+                    try:
+                        print(f"[FW_LOG] retry resp={resp2.status_code}")
+                    except Exception:
+                        pass
+        except Exception:
+            # Avoid raising exceptions from logging
+            self.handleError(record)
+
+    def _get_rollout_id(self, record: logging.LogRecord) -> Optional[str]:
+        if hasattr(record, "rollout_id") and cast(Any, getattr(record, "rollout_id")) is not None:
+            return str(cast(Any, getattr(record, "rollout_id")))
+        return os.getenv(self.rollout_id_env)
+
+    def _get_status_info(self, record: logging.LogRecord) -> Optional[Dict[str, Any]]:
+        """Extract status information from the log record's extra data."""
+        # Check if 'status' is in the extra data (passed via extra parameter)
+        if hasattr(record, "status") and record.status is not None:  # type: ignore
+            status = record.status  # type: ignore
+
+            # Handle Status class instances (Pydantic BaseModel)
+            if hasattr(status, "code") and hasattr(status, "message"):
+                # Status object - extract code and message
+                status_code = status.code
+                # Handle both enum values and direct integer values
+                if hasattr(status_code, "value"):
+                    status_code = status_code.value
+
+                return {
+                    "code": status_code,
+                    "message": status.message,
+                    "details": getattr(status, "details", []),
+                }
+            elif isinstance(status, dict):
+                # Dictionary representation of status
+                return {
+                    "code": status.get("code"),
+                    "message": status.get("message"),
+                    "details": status.get("details", []),
+                }
+        return None
+
+    def _build_payload(self, record: logging.LogRecord, rollout_id: str) -> Dict[str, Any]:
+        timestamp = datetime.fromtimestamp(record.created, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+        message = record.getMessage()
+        tags: List[str] = [f"rollout_id:{rollout_id}"]
+        # Optional additional tags
+        if hasattr(record, "experiment_id") and cast(Any, getattr(record, "experiment_id")):
+            tags.append(f"experiment_id:{cast(Any, getattr(record, 'experiment_id'))}")
+        if hasattr(record, "run_id") and cast(Any, getattr(record, "run_id")):
+            tags.append(f"run_id:{cast(Any, getattr(record, 'run_id'))}")
+        # Groupwise list of rollout_ids
+        if hasattr(record, "rollout_ids") and cast(Any, getattr(record, "rollout_ids")):
+            try:
+                for rid in cast(List[str], getattr(record, "rollout_ids")):
+                    tags.append(f"rollout_id:{rid}")
+            except Exception:
+                pass
+        program = cast(Optional[str], getattr(record, "program", None)) or "eval_protocol"
+
+        return {
+            "program": program,
+            "status": self._get_status_info(record),
+            "message": message,
+            "tags": tags,
+            "extras": {
+                "logger_name": record.name,
+                "level": record.levelname,
+                "timestamp": timestamp,
+            },
+        }

eval_protocol-0.2.57.dev2/eval_protocol/log_utils/init.py

@@ -0,0 +1,69 @@
+import logging
+import os
+from typing import Optional
+
+from eval_protocol.log_utils.fireworks_tracing_http_handler import (
+    FireworksTracingHttpHandler,
+)
+from eval_protocol.log_utils.elasticsearch_direct_http_handler import (
+    ElasticsearchDirectHttpHandler,
+)
+from eval_protocol.log_utils.rollout_context import ContextRolloutIdFilter
+from eval_protocol.types.remote_rollout_processor import ElasticsearchConfig
+
+
+_INITIALIZED = False
+
+
+def _get_env(name: str) -> Optional[str]:
+    val = os.getenv(name)
+    return val if val and val.strip() else None
+
+
+def init_external_logging_from_env() -> None:
+    """
+    Initialize external logging sinks (Fireworks tracing, optional Elasticsearch) from env vars.
+
+    Idempotent: safe to call multiple times.
+
+    Environment variables:
+      - FW_TRACING_GATEWAY_BASE_URL: enable Fireworks tracing handler when set
+      - EP_ELASTICSEARCH_URL, EP_ELASTICSEARCH_API_KEY, EP_ELASTICSEARCH_INDEX: enable ES when all set
+    """
+    global _INITIALIZED
+    if _INITIALIZED:
+        return
+
+    root_logger = logging.getLogger()
+
+    # Ensure we do not add duplicate handlers if already present
+    existing_handler_types = {type(h).__name__ for h in root_logger.handlers}
+
+    # Fireworks tracing: prefer if FIREWORKS_API_KEY is present; default base URL if not provided
+    fw_key = _get_env("FIREWORKS_API_KEY")
+    # Allow remote validation gateway to act as tracing base when provided
+    fw_url = _get_env("FW_TRACING_GATEWAY_BASE_URL") or _get_env("GATEWAY_URL") or "https://tracing.fireworks.ai"
+    if fw_key and "FireworksTracingHttpHandler" not in existing_handler_types:
+        fw_handler = FireworksTracingHttpHandler(gateway_base_url=fw_url)
+        fw_handler.setLevel(logging.INFO)
+        fw_handler.addFilter(ContextRolloutIdFilter())
+        root_logger.addHandler(fw_handler)
+
+    # Elasticsearch
+    es_url = _get_env("EP_ELASTICSEARCH_URL")
+    es_api_key = _get_env("EP_ELASTICSEARCH_API_KEY")
+    es_index = _get_env("EP_ELASTICSEARCH_INDEX")
+    if (
+        not fw_key
+        and es_url
+        and es_api_key
+        and es_index
+        and "ElasticsearchDirectHttpHandler" not in existing_handler_types
+    ):
+        es_config = ElasticsearchConfig(url=es_url, api_key=es_api_key, index_name=es_index)
+        es_handler = ElasticsearchDirectHttpHandler(elasticsearch_config=es_config)
+        es_handler.setLevel(logging.INFO)
+        es_handler.addFilter(ContextRolloutIdFilter())
+        root_logger.addHandler(es_handler)
+
+    _INITIALIZED = True

eval_protocol-0.2.57.dev2/eval_protocol/log_utils/rollout_context.py

@@ -0,0 +1,84 @@
+import logging
+import os
+from contextlib import asynccontextmanager
+from typing import List, Optional
+
+import contextvars
+
+
+# Context variables used to correlate logs with rollouts under concurrency
+current_rollout_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar("ep_rollout_id", default=None)
+current_rollout_ids: contextvars.ContextVar[Optional[List[str]]] = contextvars.ContextVar(
+    "ep_rollout_ids", default=None
+)
+current_experiment_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar("ep_experiment_id", default=None)
+current_run_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar("ep_run_id", default=None)
+
+
+class ContextRolloutIdFilter(logging.Filter):
+    """
+    Logging filter that injects correlation fields into a LogRecord from ContextVars.
+
+    The filter is intended to be attached ONLY to external sink handlers (e.g.,
+    Fireworks or Elasticsearch). If there is no active rollout context, it drops
+    the record for that handler to avoid shipping uncorrelated logs.
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:  # type: ignore[override]
+        rollout_id = current_rollout_id.get()
+        if not rollout_id:
+            # Allow explicit rollout IDs on the record or via environment fallback.
+            rollout_id = getattr(record, "rollout_id", None) or os.getenv("EP_ROLLOUT_ID")
+        if not rollout_id:
+            # No correlation context → do not emit to external sink
+            return False
+
+        # Inject primary correlation fields
+        setattr(record, "rollout_id", rollout_id)
+
+        rollout_ids = current_rollout_ids.get()
+        if rollout_ids:
+            setattr(record, "rollout_ids", rollout_ids)
+
+        experiment_id = current_experiment_id.get()
+        if experiment_id:
+            setattr(record, "experiment_id", experiment_id)
+
+        run_id = current_run_id.get()
+        if run_id:
+            setattr(record, "run_id", run_id)
+
+        return True
+
+
+@asynccontextmanager
+async def rollout_logging_context(
+    rollout_id: str,
+    *,
+    experiment_id: Optional[str] = None,
+    run_id: Optional[str] = None,
+    rollout_ids: Optional[List[str]] = None,
+):
+    """
+    Async context manager to set correlation ContextVars for the current task.
+
+    Args:
+        rollout_id: Primary rollout identifier for correlation.
+        experiment_id: Optional experiment ID for tagging.
+        run_id: Optional run ID for tagging.
+        rollout_ids: Optional list of related rollout IDs (e.g., groupwise mode).
+    """
+    t_rollout = current_rollout_id.set(rollout_id)
+    t_rollouts = current_rollout_ids.set(rollout_ids) if rollout_ids is not None else None
+    t_experiment = current_experiment_id.set(experiment_id) if experiment_id is not None else None
+    t_run = current_run_id.set(run_id) if run_id is not None else None
+    try:
+        yield
+    finally:
+        current_rollout_id.reset(t_rollout)
+        if t_rollouts is not None:
+            current_rollout_ids.reset(t_rollouts)
+        if t_experiment is not None:
+            current_experiment_id.reset(t_experiment)
+        if t_run is not None:
+            current_run_id.reset(t_run)

{eval_protocol-0.2.57 → eval_protocol-0.2.57.dev2}/eval_protocol/proxy/proxy_core/redis_utils.py

@@ -3,7 +3,7 @@ Redis utilities for tracking chat completions via insertion IDs.
 """
 
 import logging
-from typing import Set
+from typing import Set, cast
 import redis
 
 logger = logging.getLogger(__name__)
@@ -40,7 +40,16 @@ def get_insertion_ids(redis_client: redis.Redis, rollout_id: str) -> Set[str]:
         Set of insertion_id strings, empty set if none found or on error
     """
     try:
-        insertion_ids = redis_client.smembers(rollout_id)
+        raw = redis_client.smembers(rollout_id)
+        # Typing in redis stubs may be Awaitable[Set[Any]] | Set[Any]; at runtime this is a Set[bytes]
+        raw_ids = cast(Set[object], raw)
+        # Normalize to set[str]
+        insertion_ids: Set[str] = set()
+        for b in raw_ids:
+            try:
+                insertion_ids.add(b.decode("utf-8") if isinstance(b, (bytes, bytearray)) else cast(str, b))
+            except Exception:
+                continue
         logger.debug(f"Found {len(insertion_ids)} expected insertion_ids for rollout {rollout_id}")
         return insertion_ids
     except Exception as e: