eval-protocol 0.2.70__tar.gz → 0.2.71__tar.gz

{eval_protocol-0.2.70/eval_protocol.egg-info → eval_protocol-0.2.71}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.70
+Version: 0.2.71
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.70 → eval_protocol-0.2.71}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-29T04:00:08-0700",
+ "date": "2025-10-29T17:44:43-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "c705cb8d88a8d5966f22c84172d885a4352debc0",
- "version": "0.2.70"
+ "full-revisionid": "4c855e76a696143f7d7b40b40d336f3f7a76ffcf",
+ "version": "0.2.71"
 }
 '''  # END VERSION_JSON
 

eval_protocol-0.2.71/eval_protocol/exceptions.py

@@ -0,0 +1,176 @@
+"""
+Custom exceptions for Eval Protocol that map to gRPC Status codes.
+
+These exceptions provide a clean way to handle errors and map them to appropriate
+Status objects following the AIP-193 standard.
+"""
+
+from typing import Optional
+
+
+class EvalProtocolError(Exception):
+    """
+    Base exception for all Eval Protocol specific errors.
+
+    Maps to Status.Code and can be converted to Status objects for structured logging.
+    """
+
+    pass
+
+
+# Standard gRPC status code exceptions
+class CancelledError(EvalProtocolError):
+    """Operation was cancelled (Status.Code.CANCELLED = 1)"""
+
+    status_code = 1
+
+
+class UnknownError(EvalProtocolError):
+    """Unknown error occurred (Status.Code.UNKNOWN = 2)"""
+
+    status_code = 2
+
+
+class InvalidArgumentError(EvalProtocolError):
+    """Invalid argument provided (Status.Code.INVALID_ARGUMENT = 3)"""
+
+    status_code = 3
+
+
+class DeadlineExceededError(EvalProtocolError):
+    """Deadline exceeded (Status.Code.DEADLINE_EXCEEDED = 4)"""
+
+    status_code = 4
+
+
+class NotFoundError(EvalProtocolError):
+    """Resource not found (Status.Code.NOT_FOUND = 5)"""
+
+    status_code = 5
+
+
+class AlreadyExistsError(EvalProtocolError):
+    """Resource already exists (Status.Code.ALREADY_EXISTS = 6)"""
+
+    status_code = 6
+
+
+class PermissionDeniedError(EvalProtocolError):
+    """Permission denied (Status.Code.PERMISSION_DENIED = 7)"""
+
+    status_code = 7
+
+
+class ResourceExhaustedError(EvalProtocolError):
+    """Resource exhausted (Status.Code.RESOURCE_EXHAUSTED = 8)"""
+
+    status_code = 8
+
+
+class FailedPreconditionError(EvalProtocolError):
+    """Failed precondition (Status.Code.FAILED_PRECONDITION = 9)"""
+
+    status_code = 9
+
+
+class AbortedError(EvalProtocolError):
+    """Operation was aborted (Status.Code.ABORTED = 10)"""
+
+    status_code = 10
+
+
+class OutOfRangeError(EvalProtocolError):
+    """Value out of range (Status.Code.OUT_OF_RANGE = 11)"""
+
+    status_code = 11
+
+
+class UnimplementedError(EvalProtocolError):
+    """Operation is not implemented (Status.Code.UNIMPLEMENTED = 12)"""
+
+    status_code = 12
+
+
+class InternalError(EvalProtocolError):
+    """Internal server error (Status.Code.INTERNAL = 13)"""
+
+    status_code = 13
+
+
+class UnavailableError(EvalProtocolError):
+    """Service unavailable (Status.Code.UNAVAILABLE = 14)"""
+
+    status_code = 14
+
+
+class DataLossError(EvalProtocolError):
+    """Unrecoverable data loss (Status.Code.DATA_LOSS = 15)"""
+
+    status_code = 15
+
+
+class UnauthenticatedError(EvalProtocolError):
+    """Request lacks valid authentication (Status.Code.UNAUTHENTICATED = 16)"""
+
+    status_code = 16
+
+
+# Custom EP exceptions
+class RolloutFinishedError(EvalProtocolError):
+    """Rollout completed successfully (Status.Code.FINISHED = 100)"""
+
+    status_code = 100
+
+
+class RolloutRunningError(EvalProtocolError):
+    """Rollout is still running (Status.Code.RUNNING = 101)"""
+
+    status_code = 101
+
+
+class ScoreInvalidError(EvalProtocolError):
+    """Score is invalid (Status.Code.SCORE_INVALID = 102)"""
+
+    status_code = 102
+
+
+# Convenience mapping from status codes to exception classes
+# Only actual error conditions should raise exceptions
+STATUS_CODE_TO_EXCEPTION = {
+    0: None,  # OK - success, no exception
+    1: CancelledError,
+    2: UnknownError,
+    3: InvalidArgumentError,
+    4: DeadlineExceededError,
+    5: NotFoundError,
+    6: AlreadyExistsError,
+    7: PermissionDeniedError,
+    8: ResourceExhaustedError,
+    9: FailedPreconditionError,
+    10: AbortedError,
+    11: OutOfRangeError,
+    12: UnimplementedError,
+    13: InternalError,
+    14: UnavailableError,
+    15: DataLossError,
+    16: UnauthenticatedError,
+    100: None,  # FINISHED - success, no exception
+    101: None,  # RUNNING - in progress, no exception
+    102: None,  # SCORE_INVALID - success, no exception
+}
+
+
+def exception_for_status_code(code: int) -> Optional[EvalProtocolError]:
+    """
+    Create an exception instance for a given status code.
+
+    Args:
+        code: Status code from Status.Code enum
+
+    Returns:
+        Exception instance or None if code is OK (0)
+    """
+    exception_class = STATUS_CODE_TO_EXCEPTION.get(code)
+    if exception_class is None:
+        return None
+    return exception_class()

{eval_protocol-0.2.70 → eval_protocol-0.2.71}/eval_protocol/models.py

@@ -136,6 +136,13 @@ class Status(BaseModel):
         """Create a status indicating the evaluation finished."""
         return cls(code=cls.Code.FINISHED, message="Evaluation finished", details=[])
 
+    @staticmethod
+    def _build_details_with_extra_info(extra_info: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Helper to build details list from extra_info."""
+        if extra_info:
+            return [ErrorInfo.extra_info(extra_info).to_aip193_format()]
+        return []
+
     @classmethod
     def aborted(cls, message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
         """Create a status indicating the evaluation was aborted."""
@@ -160,179 +167,196 @@ class Status(BaseModel):
         """Create a status indicating the rollout finished."""
         return cls(code=cls.Code.FINISHED, message=message, details=details or [])
 
+    # Error methods organized by Status.Code enum values (1-16)
+
+    # CANCELLED = 1
     @classmethod
-    def rollout_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
-        """Create a status indicating the rollout failed with an error."""
-        details = []
-        if extra_info:
-            details.append(ErrorInfo.extra_info(extra_info).to_aip193_format())
-        return cls.error(error_message, details)
+    def rollout_cancelled_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout was cancelled."""
+        return cls.cancelled_error(error_message, cls._build_details_with_extra_info(extra_info))
+
+    @classmethod
+    def cancelled_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating the operation was cancelled."""
+        return cls(code=cls.Code.CANCELLED, message=error_message, details=details or [])
+
+    # UNKNOWN = 2
+    @classmethod
+    def rollout_unknown_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with an unknown error."""
+        return cls.unknown_error(error_message, cls._build_details_with_extra_info(extra_info))
+
+    @classmethod
+    def unknown_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an unknown error occurred."""
+        return cls(code=cls.Code.UNKNOWN, message=error_message, details=details or [])
 
+    # INVALID_ARGUMENT = 3
     @classmethod
-    def rollout_error_from_exception(
-        cls, exception: Exception, extra_info: Optional[Dict[str, Any]] = None
+    def rollout_invalid_argument_error(
+        cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None
     ) -> "Status":
-        """
-        Create a status indicating the rollout failed with an exception.
-        Simple approach that stores exception info directly in details.
-        """
-        details = []
+        """Create a status indicating the rollout failed with an invalid argument error."""
+        return cls.invalid_argument_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-        details.append(
-            {
-                "exception_type": f"{type(exception).__module__}.{type(exception).__name__}",
-                "exception_message": str(exception),
-            }
-        )
+    @classmethod
+    def invalid_argument_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an invalid argument error occurred."""
+        return cls(code=cls.Code.INVALID_ARGUMENT, message=error_message, details=details or [])
 
-        if extra_info:
-            details.append({"extra_info": extra_info})
+    # DEADLINE_EXCEEDED = 4
+    @classmethod
+    def rollout_deadline_exceeded_error(
+        cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None
+    ) -> "Status":
+        """Create a status indicating the rollout failed with a deadline exceeded error."""
+        return cls.deadline_exceeded_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-        return cls(code=cls.Code.INTERNAL, message=str(exception), details=details)
+    @classmethod
+    def deadline_exceeded_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating a deadline exceeded error occurred."""
+        return cls(code=cls.Code.DEADLINE_EXCEEDED, message=error_message, details=details or [])
 
+    # NOT_FOUND = 5
     @classmethod
-    def raise_from_status_details(cls, status_details: List[Dict[str, Any]]) -> bool:
-        """
-        Try to raise original exception from simple status details using dynamic imports.
-        """
+    def rollout_not_found_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with a not found error."""
+        return cls.not_found_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-        for detail in status_details:
-            # Look for simple exception info
-            if "exception_type" in detail and "exception_message" in detail:
-                exception_type = detail["exception_type"]
-                exception_message = detail["exception_message"]
-
-                logger.info(f"Found exception info: {exception_type}")
-
-                # Dynamically import and raise the exception
-                exception_class = cls._import_exception_class(exception_type)
-                if exception_class:
-                    logger.info(f"Found exception class: {exception_class}")
-                    # Try different constructor patterns
-                    exception_to_raise = cls._create_exception_instance(exception_class, exception_message)
-                    if exception_to_raise:
-                        logger.info(f"Re-raising {exception_type} from status details")
-                        raise exception_to_raise
-                    else:
-                        logger.info(f"Could not create instance of {exception_type}")
-                        continue
-                else:
-                    logger.info(f"Could not import exception type: {exception_type}")
-                    continue
+    @classmethod
+    def not_found_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating a not found error occurred."""
+        return cls(code=cls.Code.NOT_FOUND, message=error_message, details=details or [])
 
-        return False
+    # ALREADY_EXISTS = 6
+    @classmethod
+    def rollout_already_exists_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with an already exists error."""
+        return cls.already_exists_error(error_message, cls._build_details_with_extra_info(extra_info))
 
     @classmethod
-    def _create_exception_instance(cls, exception_class: type, message: str) -> Optional[Exception]:
-        """
-        Try to create an exception instance using different constructor patterns.
+    def already_exists_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an already exists error occurred."""
+        return cls(code=cls.Code.ALREADY_EXISTS, message=error_message, details=details or [])
 
-        Args:
-            exception_class: The exception class to instantiate
-            message: The error message
+    # PERMISSION_DENIED = 7
+    @classmethod
+    def rollout_permission_denied_error(
+        cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None
+    ) -> "Status":
+        """Create a status indicating the rollout failed with a permission denied error."""
+        return cls.permission_denied_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-        Returns:
-            Exception instance if successful, None otherwise
-        """
-        # Common constructor patterns to try
-        patterns = [
-            # Pattern 1: Just message
-            lambda: exception_class(message),
-            # Pattern 2: Message as named parameter
-            lambda: exception_class(message=message),
-            # Pattern 3: Message + common litellm parameters
-            # NOTE: we are losing some diagnostic information here by not passing the model and llm_provider. We could try to capture full exception state in rollout_error_from_exception.
-            lambda: exception_class(message, model="unknown", llm_provider="unknown"),
-            lambda: exception_class(message=message, model="unknown", llm_provider="unknown"),
-            # Pattern 5: OpenAI exceptions - create mock response object
-            lambda: cls._create_openai_exception(exception_class, message),
-            # Pattern 7: No arguments (fallback)
-            lambda: exception_class(),
-        ]
-
-        for i, pattern in enumerate(patterns):
-            try:
-                instance = pattern()
-                logger.debug(f"Successfully created {exception_class.__name__} using pattern {i + 1}")
-                return instance
-            except (TypeError, ValueError) as e:
-                logger.debug(f"Pattern {i + 1} failed for {exception_class.__name__}: {e}")
-                continue
+    @classmethod
+    def permission_denied_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating a permission denied error occurred."""
+        return cls(code=cls.Code.PERMISSION_DENIED, message=error_message, details=details or [])
 
-        logger.debug(f"All constructor patterns failed for {exception_class.__name__}")
-        return None
+    # RESOURCE_EXHAUSTED = 8
+    @classmethod
+    def rollout_resource_exhausted_error(
+        cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None
+    ) -> "Status":
+        """Create a status indicating the rollout failed with a resource exhausted error."""
+        return cls.resource_exhausted_error(error_message, cls._build_details_with_extra_info(extra_info))
 
     @classmethod
-    def _create_openai_exception(cls, exception_class: type, message: str) -> Optional[Exception]:
-        """
-        Create OpenAI exception with a mock response object.
+    def resource_exhausted_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating a resource exhausted error occurred."""
+        return cls(code=cls.Code.RESOURCE_EXHAUSTED, message=error_message, details=details or [])
 
-        OpenAI exceptions require httpx.Response objects which are complex to create,
-        so we create a minimal mock that satisfies the basic requirements.
-        """
-        try:
-            import httpx
+    # FAILED_PRECONDITION = 9
+    @classmethod
+    def rollout_failed_precondition_error(
+        cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None
+    ) -> "Status":
+        """Create a status indicating the rollout failed with a failed precondition error."""
+        return cls.failed_precondition_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-            # Create a minimal mock response object
-            class MockRequest:
-                def __init__(self):
-                    self.method = "POST"
-                    self.url = "https://api.openai.com/v1/chat/completions"
+    @classmethod
+    def failed_precondition_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating a failed precondition error occurred."""
+        return cls(code=cls.Code.FAILED_PRECONDITION, message=error_message, details=details or [])
 
-            class MockResponse:
-                def __init__(self):
-                    self.status_code = 404
-                    self.headers = {"x-request-id": "mock-request-id"}
-                    self.request = MockRequest()
+    # ABORTED = 10
+    @classmethod
+    def rollout_aborted_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout was aborted."""
+        return cls.aborted(error_message, cls._build_details_with_extra_info(extra_info))
 
-            mock_response = MockResponse()
-            return exception_class(message, response=mock_response, body=None)
+    # OUT_OF_RANGE = 11
+    @classmethod
+    def rollout_out_of_range_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with an out of range error."""
+        return cls.out_of_range_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-        except Exception as e:
-            logging.getLogger(__name__).debug(f"Failed to create OpenAI exception with mock response: {e}")
-            return None
+    @classmethod
+    def out_of_range_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an out of range error occurred."""
+        return cls(code=cls.Code.OUT_OF_RANGE, message=error_message, details=details or [])
 
+    # UNIMPLEMENTED = 12
     @classmethod
-    def _import_exception_class(cls, exception_type: str) -> Optional[type]:
-        """
-        Dynamically import an exception class from a string.
+    def rollout_unimplemented_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with an unimplemented error."""
+        return cls.unimplemented_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-        Args:
-            exception_type: Exception type string like "litellm.exceptions.NotFoundError",
-                           "openai.BadRequestError", "requests.exceptions.ConnectionError", etc.
+    @classmethod
+    def unimplemented_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an unimplemented error occurred."""
+        return cls(code=cls.Code.UNIMPLEMENTED, message=error_message, details=details or [])
 
-        Returns:
-            The exception class if found, None otherwise
-        """
-        try:
-            # Require fully qualified names (no automatic prefixing)
-            if "." not in exception_type:
-                logging.getLogger(__name__).debug(f"Exception type must be fully qualified: {exception_type}")
-                return None
+    # INTERNAL = 13
+    @classmethod
+    def rollout_internal_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with an internal error."""
+        return cls.internal_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-            # Parse module and class name
-            module_name, class_name = exception_type.rsplit(".", 1)
+    # For backwards compatibility
+    @classmethod
+    def rollout_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with an error."""
+        return cls.internal_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-            # Import the module
-            module = importlib.import_module(module_name)
+    @classmethod
+    def internal_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an internal error occurred."""
+        return cls(code=cls.Code.INTERNAL, message=error_message, details=details or [])
 
-            # Get the exception class
-            exception_class = getattr(module, class_name, None)
+    # UNAVAILABLE = 14
+    @classmethod
+    def rollout_unavailable_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with an unavailable error."""
+        return cls.unavailable_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-            # Verify it's actually an exception class
-            if exception_class and issubclass(exception_class, BaseException):
-                return exception_class
+    @classmethod
+    def unavailable_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an unavailable error occurred."""
+        return cls(code=cls.Code.UNAVAILABLE, message=error_message, details=details or [])
 
-            return None
+    # DATA_LOSS = 15
+    @classmethod
+    def rollout_data_loss_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
+        """Create a status indicating the rollout failed with a data loss error."""
+        return cls.data_loss_error(error_message, cls._build_details_with_extra_info(extra_info))
 
-        except (ImportError, AttributeError, ValueError) as e:
-            logging.getLogger(__name__).debug(f"Could not import exception class {exception_type}: {e}")
-            return None
+    @classmethod
+    def data_loss_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating a data loss error occurred."""
+        return cls(code=cls.Code.DATA_LOSS, message=error_message, details=details or [])
 
+    # UNAUTHENTICATED = 16
     @classmethod
-    def error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
-        """Create a status indicating the rollout failed with an error."""
-        return cls(code=cls.Code.INTERNAL, message=error_message, details=details or [])
+    def rollout_unauthenticated_error(
+        cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None
+    ) -> "Status":
+        """Create a status indicating the rollout failed with an unauthenticated error."""
+        return cls.unauthenticated_error(error_message, cls._build_details_with_extra_info(extra_info))
+
+    @classmethod
+    def unauthenticated_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an unauthenticated error occurred."""
+        return cls(code=cls.Code.UNAUTHENTICATED, message=error_message, details=details or [])
 
     @classmethod
     def score_invalid(

{eval_protocol-0.2.70 → eval_protocol-0.2.71}/eval_protocol/pytest/default_single_turn_rollout_process.py

@@ -21,6 +21,16 @@ logger = logging.getLogger(__name__)
 class SingleTurnRolloutProcessor(RolloutProcessor):
     """Single turn rollout processor for direct LLM calls."""
 
+    def __init__(self, *, drop_trailing_assistant_messages: bool = True) -> None:
+        """
+        Args:
+            drop_trailing_assistant_messages: When True (default), strip any trailing
+                assistant messages from the input conversation before calling the model.
+                This helps when datasets include previous assistant turns and you want
+                the model to answer the latest user query.
+        """
+        self.drop_trailing_assistant_messages = drop_trailing_assistant_messages
+
     def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) -> List[asyncio.Task[EvaluationRow]]:
         """Generate single turn rollout tasks and return them for external handling."""
         # Do not modify global LiteLLM cache. Disable caching per-request instead.
@@ -32,7 +42,13 @@ class SingleTurnRolloutProcessor(RolloutProcessor):
             if len(row.messages) == 0:
                 raise ValueError("Messages is empty. Please provide a non-empty dataset")
 
-            messages_payload = [message.model_dump() for message in row.messages]
+            # Optionally drop trailing assistant messages for single-turn prompts
+            messages_for_request: List[Message] = list(row.messages)
+            if self.drop_trailing_assistant_messages:
+                while messages_for_request and messages_for_request[-1].role == "assistant":
+                    messages_for_request.pop()
+
+            messages_payload = [message.model_dump() for message in messages_for_request]
 
             request_params = {"messages": messages_payload, **config.completion_params}
             # Ensure caching is disabled only for this request (review feedback)
@@ -114,7 +130,7 @@ class SingleTurnRolloutProcessor(RolloutProcessor):
                         except Exception:
                             pass
 
-            messages = list(row.messages) + [
+            messages = list(messages_for_request) + [
                 Message(
                     role="assistant",
                     content=assistant_content,

{eval_protocol-0.2.70 → eval_protocol-0.2.71}/eval_protocol/pytest/exception_config.py

@@ -11,7 +11,9 @@ import backoff
 import litellm
 import requests
 import httpx
-import openai
+
+import eval_protocol.exceptions
+
 
 # Default exceptions that should be retried with backoff
 DEFAULT_RETRYABLE_EXCEPTIONS: Set[Type[Exception]] = {
@@ -29,14 +31,22 @@ DEFAULT_RETRYABLE_EXCEPTIONS: Set[Type[Exception]] = {
     httpx.TimeoutException,
     httpx.NetworkError,
     httpx.RemoteProtocolError,
+    # LiteLLM library exceptions
     litellm.exceptions.RateLimitError,
     litellm.exceptions.InternalServerError,
     litellm.exceptions.Timeout,
     litellm.exceptions.NotFoundError,
-    litellm.exceptions.BadRequestError,  # remove this once we have a long term solution
+    # litellm.exceptions.BadRequestError,  # remove this once we have a long term solution
     litellm.exceptions.ServiceUnavailableError,
-    openai.NotFoundError,
-    openai.BadRequestError,  # remove this once we have a long term solution
+    litellm.exceptions.APIError,
+    # Eval Protocol exceptions
+    eval_protocol.exceptions.UnknownError,
+    eval_protocol.exceptions.DeadlineExceededError,
+    eval_protocol.exceptions.NotFoundError,
+    eval_protocol.exceptions.PermissionDeniedError,
+    eval_protocol.exceptions.UnavailableError,
+    eval_protocol.exceptions.UnauthenticatedError,
+    eval_protocol.exceptions.ResourceExhaustedError,
 }