eval-protocol 0.2.71__tar.gz → 0.2.72__tar.gz

{eval_protocol-0.2.71/eval_protocol.egg-info → eval_protocol-0.2.72}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.71
+Version: 0.2.72
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.71 → eval_protocol-0.2.72}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-29T17:44:43-0700",
+ "date": "2025-10-30T03:43:20-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "4c855e76a696143f7d7b40b40d336f3f7a76ffcf",
- "version": "0.2.71"
+ "full-revisionid": "a71074ec111c9321e5cb2e8366dbb56504f2fc3a",
+ "version": "0.2.72"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.71 → eval_protocol-0.2.72}/eval_protocol/exceptions.py

@@ -160,12 +160,13 @@ STATUS_CODE_TO_EXCEPTION = {
 }
 
 
-def exception_for_status_code(code: int) -> Optional[EvalProtocolError]:
+def exception_for_status_code(code: int, message: str = "") -> Optional[EvalProtocolError]:
     """
     Create an exception instance for a given status code.
 
     Args:
         code: Status code from Status.Code enum
+        message: Optional error message to include in the exception
 
     Returns:
         Exception instance or None if code is OK (0)
@@ -173,4 +174,4 @@ def exception_for_status_code(code: int) -> Optional[EvalProtocolError]:
     exception_class = STATUS_CODE_TO_EXCEPTION.get(code)
     if exception_class is None:
         return None
-    return exception_class()
+    return exception_class(message) if message else exception_class()

{eval_protocol-0.2.71 → eval_protocol-0.2.72}/eval_protocol/models.py

@@ -21,9 +21,6 @@ from eval_protocol.human_id import generate_id
 from eval_protocol.types import TerminationReason
 
 
-logger = logging.getLogger(__name__)
-
-
 class ErrorInfo(BaseModel):
     """
     AIP-193 ErrorInfo model for structured error details.
@@ -312,6 +309,11 @@ class Status(BaseModel):
         """Create a status indicating the rollout failed with an internal error."""
         return cls.internal_error(error_message, cls._build_details_with_extra_info(extra_info))
 
+    @classmethod
+    def internal_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an internal error occurred."""
+        return cls(code=cls.Code.INTERNAL, message=error_message, details=details or [])
+
     # For backwards compatibility
     @classmethod
     def rollout_error(cls, error_message: str, extra_info: Optional[Dict[str, Any]] = None) -> "Status":
@@ -319,8 +321,8 @@ class Status(BaseModel):
         return cls.internal_error(error_message, cls._build_details_with_extra_info(extra_info))
 
     @classmethod
-    def internal_error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
-        """Create a status indicating an internal error occurred."""
+    def error(cls, error_message: str, details: Optional[List[Dict[str, Any]]] = None) -> "Status":
+        """Create a status indicating an error occurred."""
         return cls(code=cls.Code.INTERNAL, message=error_message, details=details or [])
 
     # UNAVAILABLE = 14

{eval_protocol-0.2.71 → eval_protocol-0.2.72}/eval_protocol/pytest/evaluation_test_utils.py

@@ -398,7 +398,7 @@ async def rollout_processor_with_retry(
                 else:
                     # Non-retryable exception - fail immediately
                     logging.error(f"❌ Rollout failed (non-retryable error encountered): {repr(e)}")
-                    row.rollout_status = Status.rollout_error(repr(e))
+                    row.rollout_status = Status.rollout_error(str(e))
                     return row
 
         async def execute_row_with_backoff_and_log(

{eval_protocol-0.2.71 → eval_protocol-0.2.72}/eval_protocol/pytest/exception_config.py

@@ -36,7 +36,6 @@ DEFAULT_RETRYABLE_EXCEPTIONS: Set[Type[Exception]] = {
     litellm.exceptions.InternalServerError,
     litellm.exceptions.Timeout,
     litellm.exceptions.NotFoundError,
-    # litellm.exceptions.BadRequestError,  # remove this once we have a long term solution
     litellm.exceptions.ServiceUnavailableError,
     litellm.exceptions.APIError,
     # Eval Protocol exceptions

{eval_protocol-0.2.71 → eval_protocol-0.2.72}/eval_protocol/pytest/remote_rollout_processor.py

@@ -94,7 +94,7 @@ class RemoteRolloutProcessor(RolloutProcessor):
             def _post_init() -> None:
                 url = f"{remote_base_url}/init"
                 try:
-                    r = requests.post(url, json=init_payload.model_dump(), timeout=30)
+                    r = requests.post(url, json=init_payload.model_dump(), timeout=300)
                     r.raise_for_status()
                 except requests.exceptions.Timeout:
                     raise TimeoutError(
@@ -133,9 +133,9 @@ class RemoteRolloutProcessor(RolloutProcessor):
                     # For all other exceptions, raise them
                     raise
 
-                # Search Fireworks tracing logs for completion
-                completed_logs = self._tracing_adapter.search_logs(
-                    tags=[f"rollout_id:{row.execution_metadata.rollout_id}"]
+                # Search Fireworks tracing logs for completion (run in thread to avoid blocking event loop)
+                completed_logs = await asyncio.to_thread(
+                    self._tracing_adapter.search_logs, tags=[f"rollout_id:{row.execution_metadata.rollout_id}"]
                 )
                 # Filter for logs that actually have status information
                 status_logs = []
@@ -161,8 +161,8 @@ class RemoteRolloutProcessor(RolloutProcessor):
                         f"Found Fireworks log for rollout {row.execution_metadata.rollout_id} with status code {status_code}"
                     )
 
-                    # Create and raise exception if appropriate
-                    exception = exception_for_status_code(status_code)
+                    # Create and raise exception if appropriate, preserving original message
+                    exception = exception_for_status_code(status_code, status_message)
                     if exception is not None:
                         raise exception
 

{eval_protocol-0.2.71 → eval_protocol-0.2.72}/eval_protocol/quickstart/svg_agent/vercel_svg_server/api/init.py

@@ -12,6 +12,7 @@ import sys
 import asyncio
 from flask import Flask, request, jsonify
 from openai import OpenAI
+import openai
 from dotenv import load_dotenv
 
 from eval_protocol import Status, InitRequest, FireworksTracingHttpHandler, RolloutIdFilter
@@ -49,56 +50,80 @@ root_logger.addHandler(FireworksTracingHttpHandler())
 app = Flask(__name__)
 
 
-async def execute_rollout_background(req, api_key):
+async def execute_rollout_background(req: InitRequest, api_key: str):
     """Execute the OpenAI completion in background and log results"""
     # Attach rollout_id filter to logger
     logger = logging.getLogger(f"{__name__}.{req.metadata.rollout_id}")
     logger.addFilter(RolloutIdFilter(req.metadata.rollout_id))
 
-    try:
-        model = req.completion_params.get("model")
-        # Uncomment if you need to strip fireworks_ai/ prefix
-        # if model and isinstance(model, str) and model.startswith("fireworks_ai/"):
-        #     model = model[len("fireworks_ai/"):]
-
-        # Prepare completion arguments
-        completion_kwargs = {
-            "messages": req.messages,
-            # "messages": [{"role": "user", "content": "Hello, how are you?"}],
-            "model": model,
-            "temperature": req.completion_params.get("temperature"),
-            "max_tokens": req.completion_params.get("max_tokens"),
-        }
+    model = req.completion_params.get("model")
+    # Uncomment if you need to strip fireworks_ai/ prefix
+    # if model and isinstance(model, str) and model.startswith("fireworks_ai/"):
+    #     model = model[len("fireworks_ai/"):]
+
+    # Prepare completion arguments
+    completion_kwargs = {
+        "messages": req.messages,
+        # "messages": [{"role": "user", "content": "Hello, how are you?"}],
+        "model": model,
+        "temperature": req.completion_params.get("temperature"),
+        "max_tokens": req.completion_params.get("max_tokens"),
+    }
+
+    # Add tools if present
+    if req.tools:
+        completion_kwargs["tools"] = req.tools
+
+    logger.info(
+        f"DEBUG: {req.model_base_url}, COMPLETION_KWARGS: {completion_kwargs}, API_KEY: {api_key}, MODEL: {model}"
+    )
 
-        # Add tools if present
-        if req.tools:
-            completion_kwargs["tools"] = req.tools
+    # Create AsyncOpenAI client
+    # client = AsyncOpenAI(base_url=req.model_base_url, api_key=api_key)
+    client = OpenAI(base_url=req.model_base_url, api_key=api_key)
 
-        logger.info(
-            f"DEBUG: {req.model_base_url}, COMPLETION_KWARGS: {completion_kwargs}, API_KEY: {api_key}, MODEL: {model}"
-        )
+    logger.info(f"Sending completion request to model {model}")
 
-        # Create AsyncOpenAI client
-        # client = AsyncOpenAI(base_url=req.model_base_url, api_key=api_key)
-        client = OpenAI(base_url=req.model_base_url, api_key=api_key)
+    # Make the async model call with timeout
+    import time
 
-        logger.info(f"Sending completion request to model {model}")
+    logger.info(f"timing start: {time.time()}")
 
-        # Make the async model call with timeout
-        import time
-
-        logger.info(f"timing start: {time.time()}")
+    try:
         completion = client.chat.completions.create(**completion_kwargs)
-        logger.info(f"Completed response: {completion}")
-        logger.info(f"timing end: {time.time()}")
-        # Log successful completion - THIS IS WHAT RemoteRolloutProcessor POLLS FOR
-        logger.info(f"Rollout {req.metadata.rollout_id} completed", extra={"status": Status.rollout_finished()})
-
+    except (
+        openai.AuthenticationError,
+        openai.PermissionDeniedError,
+    ) as e:
+        # These errors should be logged and will be retried by RemoteRolloutProcessor
+        logger.error(
+            f"Rollout {req.metadata.rollout_id} failed: {e}",
+            extra={"status": Status.rollout_permission_denied_error(str(e))},
+        )
+        return
+    except openai.NotFoundError as e:
+        logger.error(
+            f"Rollout {req.metadata.rollout_id} failed: {e}", extra={"status": Status.rollout_not_found_error(str(e))}
+        )
+        return
+    except openai.RateLimitError as e:
+        logger.error(
+            f"Rollout {req.metadata.rollout_id} failed: {e}",
+            extra={"status": Status.rollout_resource_exhausted_error(str(e))},
+        )
+        return
     except Exception as e:
-        # Log error with structured status - THIS IS WHAT RemoteRolloutProcessor POLLS FOR
+        # Non-OpenAI errors (shouldn't normally happen but catch anyway)
         logger.error(
-            f"Rollout {req.metadata.rollout_id} failed: {e}", extra={"status": Status.rollout_error_from_exception(e)}
+            f"Rollout {req.metadata.rollout_id} failed with unexpected error: {e}",
+            extra={"status": Status.rollout_internal_error(str(e))},
         )
+        return
+
+    logger.info(f"Completed response: {completion}")
+    logger.info(f"timing end: {time.time()}")
+    # Log successful completion - THIS IS WHAT RemoteRolloutProcessor POLLS FOR
+    logger.info(f"Rollout {req.metadata.rollout_id} completed", extra={"status": Status.rollout_finished()})
 
 
 @app.route("/init", methods=["POST"])
@@ -114,7 +139,7 @@ async def init():
         # Validate required fields
         if not req.messages:
             error_msg = "messages is required"
-            logger.error(error_msg, extra={"status": Status.rollout_error(error_msg)})
+            logger.error(error_msg, extra={"status": Status.rollout_internal_error(error_msg)})
             return jsonify({"error": error_msg}), 400
 
         # Get API key (prefer request api_key, fallback to environment)
@@ -126,7 +151,7 @@ async def init():
             api_key = os.environ.get("FIREWORKS_API_KEY")
         else:
             error_msg = "API key not provided in request or environment variable"
-            logger.error(error_msg, extra={"status": Status.rollout_error(error_msg)})
+            logger.error(error_msg, extra={"status": Status.rollout_internal_error(error_msg)})
             return jsonify({"error": error_msg}), 401
 
         # 🔥 FIRE: Return immediately with acceptance (within 30s requirement)
@@ -137,7 +162,7 @@ async def init():
         }
 
         # Fire and forget: Execute rollout asynchronously
-        asyncio.create_task(execute_rollout_background(req, api_key))
+        asyncio.create_task(execute_rollout_background(req, api_key or ""))
 
         return jsonify(response_data), 200
 

{eval_protocol-0.2.71 → eval_protocol-0.2.72/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.71
+Version: 0.2.72
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.71 → eval_protocol-0.2.72}/tests/test_exceptions.py

@@ -348,3 +348,24 @@ def test_integration_with_retry_logic():
         assert exception_class in DEFAULT_RETRYABLE_EXCEPTIONS, (
             f"{exception_class.__name__} should be in DEFAULT_RETRYABLE_EXCEPTIONS for retry support"
         )
+
+
+def test_exception_message_preservation():
+    """Test that error messages are properly preserved in exceptions."""
+    test_cases = [
+        (13, "test error", InternalError),
+        (5, "Model xyz not found", NotFoundError),
+        (7, "Invalid API key", PermissionDeniedError),
+    ]
+
+    for status_code, message, expected_exception_class in test_cases:
+        # Test with message
+        exception = exception_for_status_code(status_code, message)
+        assert exception is not None
+        assert isinstance(exception, expected_exception_class)
+        assert str(exception) == message, f"Exception should preserve message '{message}'"
+
+        # Test without message (should still work)
+        exception_no_msg = exception_for_status_code(status_code)
+        assert exception_no_msg is not None
+        assert isinstance(exception_no_msg, expected_exception_class)