eval-protocol 0.2.69.dev3__tar.gz → 0.2.70.dev1__tar.gz

{eval_protocol-0.2.69.dev3/eval_protocol.egg-info → eval_protocol-0.2.70.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.69.dev3
+Version: 0.2.70.dev1
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.69.dev3 → eval_protocol-0.2.70.dev1}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-29T03:48:45-0700",
+ "date": "2025-10-29T17:18:36-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "f84133471cd09ac683c082262720f30b9dfaaa2d",
- "version": "0.2.69-dev3"
+ "full-revisionid": "0ebd0177dafc55bfa302a49b2d674c0487516eff",
+ "version": "0.2.70-dev1"
 }
 '''  # END VERSION_JSON
 

eval_protocol-0.2.70.dev1/eval_protocol/exceptions.py

@@ -0,0 +1,176 @@
+"""
+Custom exceptions for Eval Protocol that map to gRPC Status codes.
+
+These exceptions provide a clean way to handle errors and map them to appropriate
+Status objects following the AIP-193 standard.
+"""
+
+from typing import Optional
+
+
+class EvalProtocolError(Exception):
+    """
+    Base exception for all Eval Protocol specific errors.
+
+    Maps to Status.Code and can be converted to Status objects for structured logging.
+    """
+
+    pass
+
+
+# Standard gRPC status code exceptions
+class CancelledError(EvalProtocolError):
+    """Operation was cancelled (Status.Code.CANCELLED = 1)"""
+
+    status_code = 1
+
+
+class UnknownError(EvalProtocolError):
+    """Unknown error occurred (Status.Code.UNKNOWN = 2)"""
+
+    status_code = 2
+
+
+class InvalidArgumentError(EvalProtocolError):
+    """Invalid argument provided (Status.Code.INVALID_ARGUMENT = 3)"""
+
+    status_code = 3
+
+
+class DeadlineExceededError(EvalProtocolError):
+    """Deadline exceeded (Status.Code.DEADLINE_EXCEEDED = 4)"""
+
+    status_code = 4
+
+
+class NotFoundError(EvalProtocolError):
+    """Resource not found (Status.Code.NOT_FOUND = 5)"""
+
+    status_code = 5
+
+
+class AlreadyExistsError(EvalProtocolError):
+    """Resource already exists (Status.Code.ALREADY_EXISTS = 6)"""
+
+    status_code = 6
+
+
+class PermissionDeniedError(EvalProtocolError):
+    """Permission denied (Status.Code.PERMISSION_DENIED = 7)"""
+
+    status_code = 7
+
+
+class ResourceExhaustedError(EvalProtocolError):
+    """Resource exhausted (Status.Code.RESOURCE_EXHAUSTED = 8)"""
+
+    status_code = 8
+
+
+class FailedPreconditionError(EvalProtocolError):
+    """Failed precondition (Status.Code.FAILED_PRECONDITION = 9)"""
+
+    status_code = 9
+
+
+class AbortedError(EvalProtocolError):
+    """Operation was aborted (Status.Code.ABORTED = 10)"""
+
+    status_code = 10
+
+
+class OutOfRangeError(EvalProtocolError):
+    """Value out of range (Status.Code.OUT_OF_RANGE = 11)"""
+
+    status_code = 11
+
+
+class UnimplementedError(EvalProtocolError):
+    """Operation is not implemented (Status.Code.UNIMPLEMENTED = 12)"""
+
+    status_code = 12
+
+
+class InternalError(EvalProtocolError):
+    """Internal server error (Status.Code.INTERNAL = 13)"""
+
+    status_code = 13
+
+
+class UnavailableError(EvalProtocolError):
+    """Service unavailable (Status.Code.UNAVAILABLE = 14)"""
+
+    status_code = 14
+
+
+class DataLossError(EvalProtocolError):
+    """Unrecoverable data loss (Status.Code.DATA_LOSS = 15)"""
+
+    status_code = 15
+
+
+class UnauthenticatedError(EvalProtocolError):
+    """Request lacks valid authentication (Status.Code.UNAUTHENTICATED = 16)"""
+
+    status_code = 16
+
+
+# Custom EP exceptions
+class RolloutFinishedError(EvalProtocolError):
+    """Rollout completed successfully (Status.Code.FINISHED = 100)"""
+
+    status_code = 100
+
+
+class RolloutRunningError(EvalProtocolError):
+    """Rollout is still running (Status.Code.RUNNING = 101)"""
+
+    status_code = 101
+
+
+class ScoreInvalidError(EvalProtocolError):
+    """Score is invalid (Status.Code.SCORE_INVALID = 102)"""
+
+    status_code = 102
+
+
+# Convenience mapping from status codes to exception classes
+# Only actual error conditions should raise exceptions
+STATUS_CODE_TO_EXCEPTION = {
+    0: None,  # OK - success, no exception
+    1: CancelledError,
+    2: UnknownError,
+    3: InvalidArgumentError,
+    4: DeadlineExceededError,
+    5: NotFoundError,
+    6: AlreadyExistsError,
+    7: PermissionDeniedError,
+    8: ResourceExhaustedError,
+    9: FailedPreconditionError,
+    10: AbortedError,
+    11: OutOfRangeError,
+    12: UnimplementedError,
+    13: InternalError,
+    14: UnavailableError,
+    15: DataLossError,
+    16: UnauthenticatedError,
+    100: None,  # FINISHED - success, no exception
+    101: None,  # RUNNING - in progress, no exception
+    102: None,  # SCORE_INVALID - success, no exception
+}
+
+
+def exception_for_status_code(code: int) -> Optional[EvalProtocolError]:
+    """
+    Create an exception instance for a given status code.
+
+    Args:
+        code: Status code from Status.Code enum
+
+    Returns:
+        Exception instance or None if code is OK (0)
+    """
+    exception_class = STATUS_CODE_TO_EXCEPTION.get(code)
+    if exception_class is None:
+        return None
+    return exception_class()

{eval_protocol-0.2.69.dev3 → eval_protocol-0.2.70.dev1}/eval_protocol/models.py

@@ -136,6 +136,13 @@ class Status(BaseModel):
         """Create a status indicating the evaluation finished."""
         return cls(code=cls.Code.FINISHED, message="Evaluation finished", details=[])
 
+    @staticmethod
+    def _build_details_with_extra_info(extra_info: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Helper to build details list from extra_info."""
+        if extra_info:
+            return [ErrorInfo.extra_info(extra_info).to_aip193_format()]
+        return []
+
     @classmethod
     def aborted(cls, message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
         """Create a status indicating the evaluation was aborted."""
@@ -160,148 +167,191 @@ class Status(BaseModel):
         """Create a status indicating the rollout finished."""
         return cls(code=cls.Code.FINISHED, message=message, details=details or [])
 
+    # Error methods organized by Status.Code enum values (1-16)
+
+    # CANCELLED = 1
     @classmethod
-    def rollout_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
-        """Create a status indicating the rollout failed with an error."""
-        details = []
-        if extra_info:
-            details.append(ErrorInfo.extra_info(extra_info).to_aip193_format())
-        return cls.error(error_message, details)
+    def rollout_cancelled_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout was cancelled."""
+        return cls.cancelled_error(error_message, cls._build_details_with_extra_info(extra_info))
+
+    @classmethod
+    def cancelled_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating the operation was cancelled."""
+        return cls(code=cls.Code.CANCELLED, message=error_message, details=details or [])
 
+    # UNKNOWN = 2
     @classmethod
-    def rollout_error_from_exception(
-        cls, exception: Exception, extra_info: Optional[Dict[str, Any]] = None
+    def rollout_unknown_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with an unknown error."""
+        return cls.unknown_error(error_message, cls._build_details_with_extra_info(extra_info))
+
+    @classmethod
+    def unknown_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an unknown error occurred."""
+        return cls(code=cls.Code.UNKNOWN, message=error_message, details=details or [])
+
+    # INVALID_ARGUMENT = 3
+    @classmethod
+    def rollout_invalid_argument_error(
+        cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None
     ) -> "Status":
-        """
-        Create a status indicating the rollout failed with an exception.
-        Simple approach that stores exception info directly in details.
-        """
-        details = []
+        """Create a status indicating the rollout failed with an invalid argument error."""
+        return cls.invalid_argument_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-        details.append(
-            {
-                "exception_type": f"{type(exception).__module__}.{type(exception).__name__}",
-                "exception_message": str(exception),
-            }
-        )
+    @classmethod
+    def invalid_argument_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an invalid argument error occurred."""
+        return cls(code=cls.Code.INVALID_ARGUMENT, message=error_message, details=details or [])
 
-        if extra_info:
-            details.append({"extra_info": extra_info})
+    # DEADLINE_EXCEEDED = 4
+    @classmethod
+    def rollout_deadline_exceeded_error(
+        cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None
+    ) -> "Status":
+        """Create a status indicating the rollout failed with a deadline exceeded error."""
+        return cls.deadline_exceeded_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-        return cls(code=cls.Code.INTERNAL, message=str(exception), details=details)
+    @classmethod
+    def deadline_exceeded_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating a deadline exceeded error occurred."""
+        return cls(code=cls.Code.DEADLINE_EXCEEDED, message=error_message, details=details or [])
 
+    # NOT_FOUND = 5
     @classmethod
-    def raise_from_status_details(cls, status_details: List[Dict[str, Any]]) -> bool:
-        """
-        Try to raise original exception from simple status details using dynamic imports.
-        """
+    def rollout_not_found_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with a not found error."""
+        return cls.not_found_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-        for detail in status_details:
-            # Look for simple exception info
-            if "exception_type" in detail and "exception_message" in detail:
-                exception_type = detail["exception_type"]
-                exception_message = detail["exception_message"]
-
-                logger.info(f"Found exception info: {exception_type}")
-
-                # Dynamically import and raise the exception
-                exception_class = cls._import_exception_class(exception_type)
-                if exception_class:
-                    logger.info(f"Found exception class: {exception_class}")
-                    # Try different constructor patterns
-                    exception_to_raise = cls._create_exception_instance(exception_class, exception_message)
-                    if exception_to_raise:
-                        logger.info(f"Re-raising {exception_type} from status details")
-                        raise exception_to_raise
-                    else:
-                        logger.debug(f"Could not create instance of {exception_type}")
-                        continue
-                else:
-                    logger.debug(f"Could not import exception type: {exception_type}")
-                    continue
+    @classmethod
+    def not_found_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating a not found error occurred."""
+        return cls(code=cls.Code.NOT_FOUND, message=error_message, details=details or [])
 
-        return False
+    # ALREADY_EXISTS = 6
+    @classmethod
+    def rollout_already_exists_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with an already exists error."""
+        return cls.already_exists_error(error_message, cls._build_details_with_extra_info(extra_info))
 
     @classmethod
-    def _create_exception_instance(cls, exception_class: type, message: str) -> Optional[Exception]:
-        """
-        Try to create an exception instance using different constructor patterns.
+    def already_exists_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an already exists error occurred."""
+        return cls(code=cls.Code.ALREADY_EXISTS, message=error_message, details=details or [])
 
-        Args:
-            exception_class: The exception class to instantiate
-            message: The error message
+    # PERMISSION_DENIED = 7
+    @classmethod
+    def rollout_permission_denied_error(
+        cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None
+    ) -> "Status":
+        """Create a status indicating the rollout failed with a permission denied error."""
+        return cls.permission_denied_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-        Returns:
-            Exception instance if successful, None otherwise
-        """
-        # Common constructor patterns to try
-        patterns = [
-            # Pattern 1: Just message
-            lambda: exception_class(message),
-            # Pattern 2: Message as named parameter
-            lambda: exception_class(message=message),
-            # Pattern 3: Message + common litellm parameters
-            # NOTE: we are losing some diagnostic information here by not passing the model and llm_provider. We could try to capture full exception state in rollout_error_from_exception.
-            lambda: exception_class(message, model="unknown", llm_provider="unknown"),
-            lambda: exception_class(message=message, model="unknown", llm_provider="unknown"),
-            # Pattern 4: No arguments (fallback)
-            lambda: exception_class(),
-        ]
-
-        for i, pattern in enumerate(patterns):
-            try:
-                instance = pattern()
-                logger.debug(f"Successfully created {exception_class.__name__} using pattern {i + 1}")
-                return instance
-            except (TypeError, ValueError) as e:
-                logger.debug(f"Pattern {i + 1} failed for {exception_class.__name__}: {e}")
-                continue
+    @classmethod
+    def permission_denied_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating a permission denied error occurred."""
+        return cls(code=cls.Code.PERMISSION_DENIED, message=error_message, details=details or [])
 
-        logger.debug(f"All constructor patterns failed for {exception_class.__name__}")
-        return None
+    # RESOURCE_EXHAUSTED = 8
+    @classmethod
+    def rollout_resource_exhausted_error(
+        cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None
+    ) -> "Status":
+        """Create a status indicating the rollout failed with a resource exhausted error."""
+        return cls.resource_exhausted_error(error_message, cls._build_details_with_extra_info(extra_info))
 
     @classmethod
-    def _import_exception_class(cls, exception_type: str) -> Optional[type]:
-        """
-        Dynamically import an exception class from a string.
+    def resource_exhausted_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating a resource exhausted error occurred."""
+        return cls(code=cls.Code.RESOURCE_EXHAUSTED, message=error_message, details=details or [])
 
-        Args:
-            exception_type: Exception type string like "litellm.exceptions.NotFoundError",
-                           "openai.BadRequestError", "requests.exceptions.ConnectionError", etc.
+    # FAILED_PRECONDITION = 9
+    @classmethod
+    def rollout_failed_precondition_error(
+        cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None
+    ) -> "Status":
+        """Create a status indicating the rollout failed with a failed precondition error."""
+        return cls.failed_precondition_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-        Returns:
-            The exception class if found, None otherwise
-        """
-        try:
-            # Require fully qualified names (no automatic prefixing)
-            if "." not in exception_type:
-                logging.getLogger(__name__).debug(f"Exception type must be fully qualified: {exception_type}")
-                return None
+    @classmethod
+    def failed_precondition_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating a failed precondition error occurred."""
+        return cls(code=cls.Code.FAILED_PRECONDITION, message=error_message, details=details or [])
 
-            # Parse module and class name
-            module_name, class_name = exception_type.rsplit(".", 1)
+    # ABORTED = 10
+    @classmethod
+    def rollout_aborted_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout was aborted."""
+        return cls.aborted(error_message, cls._build_details_with_extra_info(extra_info))
 
-            # Import the module
-            module = importlib.import_module(module_name)
+    # OUT_OF_RANGE = 11
+    @classmethod
+    def rollout_out_of_range_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with an out of range error."""
+        return cls.out_of_range_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-            # Get the exception class
-            exception_class = getattr(module, class_name, None)
+    @classmethod
+    def out_of_range_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an out of range error occurred."""
+        return cls(code=cls.Code.OUT_OF_RANGE, message=error_message, details=details or [])
 
-            # Verify it's actually an exception class
-            if exception_class and issubclass(exception_class, BaseException):
-                return exception_class
+    # UNIMPLEMENTED = 12
+    @classmethod
+    def rollout_unimplemented_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with an unimplemented error."""
+        return cls.unimplemented_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-            return None
+    @classmethod
+    def unimplemented_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an unimplemented error occurred."""
+        return cls(code=cls.Code.UNIMPLEMENTED, message=error_message, details=details or [])
 
-        except (ImportError, AttributeError, ValueError) as e:
-            logging.getLogger(__name__).debug(f"Could not import exception class {exception_type}: {e}")
-            return None
+    # INTERNAL = 13
+    @classmethod
+    def rollout_internal_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with an internal error."""
+        return cls.internal_error(error_message, cls._build_details_with_extra_info(extra_info))
 
     @classmethod
-    def error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
-        """Create a status indicating the rollout failed with an error."""
+    def internal_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an internal error occurred."""
         return cls(code=cls.Code.INTERNAL, message=error_message, details=details or [])
 
+    # UNAVAILABLE = 14
+    @classmethod
+    def rollout_unavailable_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with an unavailable error."""
+        return cls.unavailable_error(error_message, cls._build_details_with_extra_info(extra_info))
+
+    @classmethod
+    def unavailable_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an unavailable error occurred."""
+        return cls(code=cls.Code.UNAVAILABLE, message=error_message, details=details or [])
+
+    # DATA_LOSS = 15
+    @classmethod
+    def rollout_data_loss_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with a data loss error."""
+        return cls.data_loss_error(error_message, cls._build_details_with_extra_info(extra_info))
+
+    @classmethod
+    def data_loss_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating a data loss error occurred."""
+        return cls(code=cls.Code.DATA_LOSS, message=error_message, details=details or [])
+
+    # UNAUTHENTICATED = 16
+    @classmethod
+    def rollout_unauthenticated_error(
+        cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None
+    ) -> "Status":
+        """Create a status indicating the rollout failed with an unauthenticated error."""
+        return cls.unauthenticated_error(error_message, cls._build_details_with_extra_info(extra_info))
+
+    @classmethod
+    def unauthenticated_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an unauthenticated error occurred."""
+        return cls(code=cls.Code.UNAUTHENTICATED, message=error_message, details=details or [])
+
     @classmethod
     def score_invalid(
         cls, message: str = "Score is invalid", details: Optional[List[Dict[str, Any]]] = None

{eval_protocol-0.2.69.dev3 → eval_protocol-0.2.70.dev1}/eval_protocol/pytest/default_single_turn_rollout_process.py

@@ -21,6 +21,16 @@ logger = logging.getLogger(__name__)
 class SingleTurnRolloutProcessor(RolloutProcessor):
     """Single turn rollout processor for direct LLM calls."""
 
+    def __init__(self, *, drop_trailing_assistant_messages: bool = True) -> None:
+        """
+        Args:
+            drop_trailing_assistant_messages: When True (default), strip any trailing
+                assistant messages from the input conversation before calling the model.
+                This helps when datasets include previous assistant turns and you want
+                the model to answer the latest user query.
+        """
+        self.drop_trailing_assistant_messages = drop_trailing_assistant_messages
+
     def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) -> List[asyncio.Task[EvaluationRow]]:
         """Generate single turn rollout tasks and return them for external handling."""
         # Do not modify global LiteLLM cache. Disable caching per-request instead.
@@ -32,7 +42,13 @@ class SingleTurnRolloutProcessor(RolloutProcessor):
             if len(row.messages) == 0:
                 raise ValueError("Messages is empty. Please provide a non-empty dataset")
 
-            messages_payload = [message.model_dump() for message in row.messages]
+            # Optionally drop trailing assistant messages for single-turn prompts
+            messages_for_request: List[Message] = list(row.messages)
+            if self.drop_trailing_assistant_messages:
+                while messages_for_request and messages_for_request[-1].role == "assistant":
+                    messages_for_request.pop()
+
+            messages_payload = [message.model_dump() for message in messages_for_request]
 
             request_params = {"messages": messages_payload, **config.completion_params}
             # Ensure caching is disabled only for this request (review feedback)
@@ -114,7 +130,7 @@ class SingleTurnRolloutProcessor(RolloutProcessor):
                         except Exception:
                             pass
 
-            messages = list(row.messages) + [
+            messages = list(messages_for_request) + [
                 Message(
                     role="assistant",
                     content=assistant_content,

{eval_protocol-0.2.69.dev3 → eval_protocol-0.2.70.dev1}/eval_protocol/pytest/exception_config.py

@@ -11,7 +11,9 @@ import backoff
 import litellm
 import requests
 import httpx
-import openai
+
+import eval_protocol.exceptions
+
 
 # Default exceptions that should be retried with backoff
 DEFAULT_RETRYABLE_EXCEPTIONS: Set[Type[Exception]] = {
@@ -29,14 +31,22 @@ DEFAULT_RETRYABLE_EXCEPTIONS: Set[Type[Exception]] = {
     httpx.TimeoutException,
     httpx.NetworkError,
     httpx.RemoteProtocolError,
+    # LiteLLM library exceptions
     litellm.exceptions.RateLimitError,
     litellm.exceptions.InternalServerError,
     litellm.exceptions.Timeout,
     litellm.exceptions.NotFoundError,
-    litellm.exceptions.BadRequestError,  # remove this once we have a long term solution
+    # litellm.exceptions.BadRequestError,  # remove this once we have a long term solution
     litellm.exceptions.ServiceUnavailableError,
-    openai.NotFoundError,
-    openai.BadRequestError,  # remove this once we have a long term solution
+    litellm.exceptions.APIError,
+    # Eval Protocol exceptions
+    eval_protocol.exceptions.UnknownError,
+    eval_protocol.exceptions.DeadlineExceededError,
+    eval_protocol.exceptions.NotFoundError,
+    eval_protocol.exceptions.PermissionDeniedError,
+    eval_protocol.exceptions.UnavailableError,
+    eval_protocol.exceptions.UnauthenticatedError,
+    eval_protocol.exceptions.ResourceExhaustedError,
 }
 
 

{eval_protocol-0.2.69.dev3 → eval_protocol-0.2.70.dev1}/eval_protocol/pytest/remote_rollout_processor.py

@@ -10,6 +10,7 @@ from eval_protocol.types.remote_rollout_processor import (
     DataLoaderConfig,
 )
 from eval_protocol.adapters.fireworks_tracing import FireworksTracingAdapter
+from eval_protocol.exceptions import exception_for_status_code
 
 from .rollout_processor import RolloutProcessor
 from .types import RolloutProcessorConfig
@@ -97,13 +98,7 @@ class RemoteRolloutProcessor(RolloutProcessor):
                     r.raise_for_status()
                 except requests.exceptions.Timeout:
                     raise TimeoutError(
-                        "The /init endpoint timed out after 30 seconds. "
-                        "CRITICAL: The /init endpoint must return immediately (within 30s) and NOT block on rollout execution. "
-                        "Your remote server should:\n"
-                        "1. Accept the /init request and return a 200 response immediately\n"
-                        "2. Process the actual rollout asynchronously in the background\n"
-                        "3. Use the /status endpoint to report progress\n"
-                        "For Python/Node.js: Start a separate process per rollout to avoid blocking the /init response."
+                        f"The /init endpoint tried {url} with {init_payload.model_dump()} but timed out after 30 seconds."
                     )
 
             await asyncio.to_thread(_post_init)
@@ -166,7 +161,10 @@ class RemoteRolloutProcessor(RolloutProcessor):
                         f"Found Fireworks log for rollout {row.execution_metadata.rollout_id} with status code {status_code}"
                     )
 
-                    Status.raise_from_status_details(status_details)
+                    # Create and raise exception if appropriate
+                    exception = exception_for_status_code(status_code)
+                    if exception is not None:
+                        raise exception
 
                     row.rollout_status = Status(
                         code=Status.Code(status_code),
@@ -183,7 +181,7 @@ class RemoteRolloutProcessor(RolloutProcessor):
                     f"Loop completed without breaking for {row.execution_metadata.rollout_id}, which means we timed out"
                 )
                 # Loop completed without breaking, which means we timed out
-                row.rollout_status = Status.rollout_error(
+                row.rollout_status = Status.rollout_deadline_exceeded_error(
                     f"Rollout {row.execution_metadata.rollout_id} timed out after {timeout_seconds} seconds"
                 )
 

{eval_protocol-0.2.69.dev3 → eval_protocol-0.2.70.dev1}/eval_protocol/pytest/tracing_utils.py

@@ -151,14 +151,14 @@ def update_row_with_remote_trace(
     output_rows: List[EvaluationRow] = [r for result in results for r in result.rows]
 
     if len(output_rows) == 0:  # Fallback to original row if no remote data found
-        row.rollout_status = Status(code=Status.Code.NOT_FOUND, message="No remote data found for rollout")
+        row.rollout_status = Status.rollout_not_found_error("No remote data found for rollout")
         return None
     elif len(output_rows) == 1:  # Return the remote row
         remote_row = output_rows[0]
 
         # if the remote_row has the same number of messages as the original row, something went wrong
         if len(remote_row.messages) == len(row.messages):
-            row.rollout_status = Status.rollout_error(
+            row.rollout_status = Status.rollout_internal_error(
                 "Rollout finished with the same number of messages as the original row"
             )
             return None