eval-protocol 0.2.94.dev2__tar.gz → 0.2.94.dev3__tar.gz

{eval_protocol-0.2.94.dev2/eval_protocol.egg-info → eval_protocol-0.2.94.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.94.dev2
+Version: 0.2.94.dev3
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.94.dev2 → eval_protocol-0.2.94.dev3}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-12-02T16:49:02-0800",
+ "date": "2025-12-02T21:46:38-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "1d630d46fdc3517d5ac6a0ce2debb4568587b26c",
- "version": "0.2.94-dev.2"
+ "full-revisionid": "01bc8e998a3a0370fc0631d39b4fbd8b4b5c7941",
+ "version": "0.2.94-dev.3"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.94.dev2 → eval_protocol-0.2.94.dev3}/eval_protocol/benchmarks/test_aime25.py

@@ -5,6 +5,7 @@ from eval_protocol.models import (
     EvaluationRow,
     Message,
     MetricResult,
+    ChatCompletionContentPartParam,
     ChatCompletionContentPartTextParam,
 )
 from eval_protocol.pytest.default_single_turn_rollout_process import (
@@ -18,10 +19,12 @@ SYSTEM_PROMPT = (
 
 
 def _coerce_content_to_str(
-    content: str | list[ChatCompletionContentPartTextParam] | None,
+    content: str | list[ChatCompletionContentPartParam] | None,
 ) -> str:
     if isinstance(content, list):
-        return "".join([getattr(p, "text", str(p)) for p in content])
+        return "".join(
+            getattr(p, "text", str(p)) if isinstance(p, ChatCompletionContentPartTextParam) else "" for p in content
+        )
     return str(content or "")
 
 

{eval_protocol-0.2.94.dev2 → eval_protocol-0.2.94.dev3}/eval_protocol/benchmarks/test_gpqa.py

@@ -10,6 +10,7 @@ from eval_protocol.models import (
     EvaluationRow,
     Message,
     MetricResult,
+    ChatCompletionContentPartParam,
     ChatCompletionContentPartTextParam,
 )
 from eval_protocol.pytest.default_single_turn_rollout_process import (
@@ -54,10 +55,12 @@ def _load_gpqa_messages_from_csv() -> list[list[list[Message]]]:
 
 
 def _coerce_content_to_str(
-    content: str | list[ChatCompletionContentPartTextParam] | None,
+    content: str | list[ChatCompletionContentPartParam] | None,
 ) -> str:
     if isinstance(content, list):
-        return "".join([getattr(p, "text", str(p)) for p in content])
+        return "".join(
+            getattr(p, "text", str(p)) if isinstance(p, ChatCompletionContentPartTextParam) else "" for p in content
+        )
     return str(content or "")
 
 

{eval_protocol-0.2.94.dev2 → eval_protocol-0.2.94.dev3}/eval_protocol/benchmarks/test_livebench_data_analysis.py

@@ -8,6 +8,7 @@ from eval_protocol.models import (
     EvaluationRow,
     Message,
     MetricResult,
+    ChatCompletionContentPartParam,
     ChatCompletionContentPartTextParam,
 )
 from eval_protocol.pytest.default_single_turn_rollout_process import (
@@ -37,9 +38,11 @@ def _extract_last_boxed_segment(text: str) -> Optional[str]:
     return matches[-1]
 
 
-def _coerce_content_to_str(content: str | list[ChatCompletionContentPartTextParam] | None) -> str:
+def _coerce_content_to_str(content: str | list[ChatCompletionContentPartParam] | None) -> str:
     if isinstance(content, list):
-        return "".join([getattr(p, "text", str(p)) for p in content])
+        return "".join(
+            getattr(p, "text", str(p)) if isinstance(p, ChatCompletionContentPartTextParam) else "" for p in content
+        )
     return str(content or "")
 
 

{eval_protocol-0.2.94.dev2 → eval_protocol-0.2.94.dev3}/eval_protocol/mcp/execution/base_policy.py

@@ -199,10 +199,6 @@ class LLMBasePolicy(PlaybackPolicyBase, ABC):
         if message.get("tool_calls"):
             assistant_message_for_history["tool_calls"] = message["tool_calls"]
 
-        rd = message.get("reasoning_details", None)
-        if rd is not None:
-            assistant_message_for_history["reasoning_details"] = rd
-
         # Add to actual conversation history
         conversation_history.append(assistant_message_for_history)
 

{eval_protocol-0.2.94.dev2 → eval_protocol-0.2.94.dev3}/eval_protocol/mcp/execution/policy.py

@@ -146,7 +146,7 @@ class LiteLLMPolicy(LLMBasePolicy):
             Clean messages with only OpenAI API compatible fields
         """
         # Standard OpenAI message fields
-        allowed_fields = {"role", "content", "tool_calls", "tool_call_id", "name", "reasoning_details"}
+        allowed_fields = {"role", "content", "tool_calls", "tool_call_id", "name"}
 
         clean_messages = []
         for msg in messages:
@@ -217,36 +217,31 @@ class LiteLLMPolicy(LLMBasePolicy):
                 logger.debug(f"🔄 API call for model: {self.model_id}")
 
             # LiteLLM already returns OpenAI-compatible format
-            message_obj = getattr(response.choices[0], "message", object())
-
-            message_dict: Dict[str, Any] = {
-                "role": getattr(message_obj, "role", "assistant"),
-                "content": getattr(message_obj, "content", None),
-                "tool_calls": (
-                    [
-                        {
-                            "id": getattr(tc, "id", None),
-                            "type": getattr(tc, "type", "function"),
-                            "function": {
-                                "name": getattr(getattr(tc, "function", None), "name", "tool"),
-                                "arguments": getattr(getattr(tc, "function", None), "arguments", "{}"),
-                            },
-                        }
-                        for tc in (getattr(message_obj, "tool_calls", []) or [])
-                    ]
-                    if getattr(message_obj, "tool_calls", None)
-                    else []
-                ),
-            }
-
-            rd = getattr(message_obj, "reasoning_details", None)
-            if rd is not None:
-                message_dict["reasoning_details"] = rd
-
             return {
                 "choices": [
                     {
-                        "message": message_dict,
+                        "message": {
+                            "role": getattr(getattr(response.choices[0], "message", object()), "role", "assistant"),
+                            "content": getattr(getattr(response.choices[0], "message", object()), "content", None),
+                            "tool_calls": (
+                                [
+                                    {
+                                        "id": getattr(tc, "id", None),
+                                        "type": getattr(tc, "type", "function"),
+                                        "function": {
+                                            "name": getattr(getattr(tc, "function", None), "name", "tool"),
+                                            "arguments": getattr(getattr(tc, "function", None), "arguments", "{}"),
+                                        },
+                                    }
+                                    for tc in (
+                                        getattr(getattr(response.choices[0], "message", object()), "tool_calls", [])
+                                        or []
+                                    )
+                                ]
+                                if getattr(getattr(response.choices[0], "message", object()), "tool_calls", None)
+                                else []
+                            ),
+                        },
                         "finish_reason": getattr(response.choices[0], "finish_reason", None),
                     }
                 ],

{eval_protocol-0.2.94.dev2 → eval_protocol-0.2.94.dev3}/eval_protocol/models.py

@@ -466,11 +466,46 @@ class ChatCompletionContentPartTextParam(BaseModel):
         return iter(["text", "type"])
 
 
+class ChatCompletionContentPartImageParam(BaseModel):
+    type: Literal["image_url"] = Field("image_url", description="The type of the content part.")
+    image_url: Dict[str, Any] = Field(
+        ..., description="Image descriptor (e.g., {'url': 'data:image/png;base64,...', 'detail': 'high'})."
+    )
+
+    def __getitem__(self, key: str) -> Any:
+        if key == "image_url":
+            return self.image_url
+        if key == "type":
+            return self.type
+        raise KeyError(key)
+
+    def get(self, key: str, default: Any = None) -> Any:
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def keys(self):
+        return (k for k in ("image_url", "type"))
+
+    def values(self):
+        return (self.image_url, self.type)
+
+    def items(self):
+        return [("image_url", self.image_url), ("type", self.type)]
+
+    def __iter__(self):
+        return iter(["image_url", "type"])
+
+
+ChatCompletionContentPartParam = Union[ChatCompletionContentPartTextParam, ChatCompletionContentPartImageParam]
+
+
 class Message(BaseModel):
     """Chat message model with trajectory evaluation support."""
 
     role: str  # assistant, user, system, tool
-    content: Optional[Union[str, List[ChatCompletionContentPartTextParam]]] = Field(
+    content: Optional[Union[str, List[ChatCompletionContentPartParam]]] = Field(
         default="", description="The content of the message."
     )
     reasoning_content: Optional[str] = Field(

{eval_protocol-0.2.94.dev2 → eval_protocol-0.2.94.dev3}/eval_protocol/pytest/default_agent_rollout_processor.py

@@ -13,7 +13,12 @@ from openai.types.chat.chat_completion_message_param import ChatCompletionMessag
 from eval_protocol.dataset_logger.dataset_logger import DatasetLogger
 from eval_protocol.mcp.execution.policy import LiteLLMPolicy
 from eval_protocol.mcp.mcp_multi_client import MCPMultiClient
-from eval_protocol.models import EvaluationRow, Message, ChatCompletionContentPartTextParam
+from eval_protocol.models import (
+    EvaluationRow,
+    Message,
+    ChatCompletionContentPartParam,
+    ChatCompletionContentPartTextParam,
+)
 from openai.types import CompletionUsage
 from eval_protocol.pytest.rollout_processor import RolloutProcessor
 from eval_protocol.pytest.types import Dataset, RolloutProcessorConfig
@@ -98,7 +103,7 @@ class Agent:
         self.messages.append(message)
         self.logger.log(self.evaluation_row)
 
-    async def call_agent(self) -> Optional[Union[str, List[ChatCompletionContentPartTextParam]]]:
+    async def call_agent(self) -> Optional[Union[str, List[ChatCompletionContentPartParam]]]:
         """
         Call the assistant with the user query.
         """
@@ -222,7 +227,7 @@ class Agent:
 
     def _format_tool_message_content(
         self, content: List[TextContent]
-    ) -> Union[str, List[ChatCompletionContentPartTextParam]]:
+    ) -> Union[str, List[ChatCompletionContentPartParam]]:
         """Format tool result content for inclusion in a tool message.
 
         - If a single text item, return plain string per OpenAI semantics.

{eval_protocol-0.2.94.dev2 → eval_protocol-0.2.94.dev3}/eval_protocol/pytest/default_single_turn_rollout_process.py

@@ -166,13 +166,17 @@ class SingleTurnRolloutProcessor(RolloutProcessor):
             row.execution_metadata.tool_call_count = (
                 len(converted_tool_calls) if converted_tool_calls is not None else 0
             )
-            row.execution_metadata.usage = (
-                CompletionUsage(  # Note: LiteLLM sets usage dynamically via setattr(), not as a typed field
-                    prompt_tokens=response.usage.prompt_tokens,  # pyright: ignore[reportAttributeAccessIssue]
-                    completion_tokens=response.usage.completion_tokens,  # pyright: ignore[reportAttributeAccessIssue]
-                    total_tokens=response.usage.total_tokens,  # pyright: ignore[reportAttributeAccessIssue]
+            usage = getattr(response, "usage", None)
+            if usage:
+                row.execution_metadata.usage = (
+                    CompletionUsage(  # Note: LiteLLM sets usage dynamically via setattr(), not as a typed field
+                        prompt_tokens=getattr(usage, "prompt_tokens", 0),
+                        completion_tokens=getattr(usage, "completion_tokens", 0),
+                        total_tokens=getattr(usage, "total_tokens", 0),
+                    )
                 )
-            )
+            else:
+                row.execution_metadata.usage = None
 
             row.messages = messages
 

{eval_protocol-0.2.94.dev2 → eval_protocol-0.2.94.dev3}/eval_protocol/rewards/accuracy.py

@@ -10,10 +10,16 @@ like normalization and LaTeX parsing.
 import re
 from typing import Any, Callable, Dict, List, Optional, Union, cast
 
-from ..models import EvaluateResult, Message, MetricResult, ChatCompletionContentPartTextParam
+from ..models import (
+    EvaluateResult,
+    Message,
+    MetricResult,
+    ChatCompletionContentPartParam,
+    ChatCompletionContentPartTextParam,
+)
 
 
-def _to_text(content: Optional[Union[str, List[ChatCompletionContentPartTextParam]]]) -> str:
+def _to_text(content: Optional[Union[str, List[ChatCompletionContentPartParam]]]) -> str:
     """Coerce Message.content into a plain string for regex and comparisons."""
     if content is None:
         return ""
@@ -21,7 +27,11 @@ def _to_text(content: Optional[Union[str, List[ChatCompletionContentPartTextPara
         return content
     # List[ChatCompletionContentPartTextParam]
     try:
-        return "\n".join(part.text for part in content)
+        texts: List[str] = []
+        for part in content:
+            if isinstance(part, ChatCompletionContentPartTextParam):
+                texts.append(part.text)
+        return "\n".join(texts)
     except Exception:
         return ""
 

{eval_protocol-0.2.94.dev2 → eval_protocol-0.2.94.dev3}/eval_protocol/rewards/json_schema.py

@@ -2,7 +2,13 @@ import json
 import re
 from typing import Any, Dict, List, Optional, Union
 
-from ..models import EvaluateResult, Message, MetricResult, ChatCompletionContentPartTextParam
+from ..models import (
+    EvaluateResult,
+    Message,
+    MetricResult,
+    ChatCompletionContentPartParam,
+    ChatCompletionContentPartTextParam,
+)
 from ..typed_interface import reward_function
 from .function_calling import (
     calculate_jaccard_similarity,
@@ -59,8 +65,10 @@ def json_schema_reward(
                     content_text = last_message.content
                 else:
                     try:
-                        parts: List[ChatCompletionContentPartTextParam] = last_message.content  # type: ignore[assignment]
-                        content_text = "\n".join(getattr(p, "text", "") for p in parts)
+                        parts: List[ChatCompletionContentPartParam] = last_message.content  # type: ignore[assignment]
+                        content_text = "\n".join(
+                            getattr(p, "text", "") for p in parts if isinstance(p, ChatCompletionContentPartTextParam)
+                        )
                     except Exception:
                         content_text = ""
             else:

{eval_protocol-0.2.94.dev2 → eval_protocol-0.2.94.dev3}/eval_protocol/rewards/language_consistency.py

@@ -9,7 +9,13 @@ are in the expected language.
 import re
 from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
-from ..models import EvaluateResult, Message, MetricResult, ChatCompletionContentPartTextParam
+from ..models import (
+    EvaluateResult,
+    Message,
+    MetricResult,
+    ChatCompletionContentPartParam,
+    ChatCompletionContentPartTextParam,
+)
 from ..typed_interface import reward_function
 
 # Dictionary mapping language codes to common words/patterns in that language
@@ -573,13 +579,17 @@ def language_consistency_reward(
             },
         )
 
-    def _to_text(content: Union[str, List[ChatCompletionContentPartTextParam], None]) -> str:
+    def _to_text(content: Union[str, List[ChatCompletionContentPartParam], None]) -> str:
         if content is None:
             return ""
         if isinstance(content, str):
             return content
         try:
-            return "\n".join(part.text for part in content)
+            texts: List[str] = []
+            for part in content:
+                if isinstance(part, ChatCompletionContentPartTextParam):
+                    texts.append(part.text)
+            return "\n".join(texts)
         except Exception:
             return ""
 

{eval_protocol-0.2.94.dev2 → eval_protocol-0.2.94.dev3}/eval_protocol/rewards/repetition.py

@@ -8,16 +8,26 @@ encouraging more diverse and information-rich outputs.
 import re
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
 
-from ..models import EvaluateResult, Message, MetricResult, ChatCompletionContentPartTextParam
+from ..models import (
+    EvaluateResult,
+    Message,
+    MetricResult,
+    ChatCompletionContentPartParam,
+    ChatCompletionContentPartTextParam,
+)
 
 
-def _to_text(content: Optional[Union[str, List[ChatCompletionContentPartTextParam]]]) -> str:
+def _to_text(content: Optional[Union[str, List[ChatCompletionContentPartParam]]]) -> str:
     if content is None:
         return ""
     if isinstance(content, str):
         return content
     try:
-        return "\n".join(part.text for part in content)
+        texts: List[str] = []
+        for part in content:
+            if isinstance(part, ChatCompletionContentPartTextParam):
+                texts.append(part.text)
+        return "\n".join(texts)
     except Exception:
         return ""
 

{eval_protocol-0.2.94.dev2 → eval_protocol-0.2.94.dev3}/eval_protocol/rewards/tag_count.py

@@ -8,16 +8,26 @@ specified XML/HTML-like tags in correct quantities.
 import re
 from typing import Any, Dict, List, Set, Union
 
-from ..models import EvaluateResult, Message, MetricResult, ChatCompletionContentPartTextParam
+from ..models import (
+    EvaluateResult,
+    Message,
+    MetricResult,
+    ChatCompletionContentPartParam,
+    ChatCompletionContentPartTextParam,
+)
 
 
-def _to_text(content: Union[str, List[ChatCompletionContentPartTextParam], None]) -> str:
+def _to_text(content: Union[str, List[ChatCompletionContentPartParam], None]) -> str:
     if content is None:
         return ""
     if isinstance(content, str):
         return content
     try:
-        return "\n".join(part.text for part in content)
+        texts: List[str] = []
+        for part in content:
+            if isinstance(part, ChatCompletionContentPartTextParam):
+                texts.append(part.text)
+        return "\n".join(texts)
     except Exception:
         return ""
 

{eval_protocol-0.2.94.dev2 → eval_protocol-0.2.94.dev3/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.94.dev2
+Version: 0.2.94.dev3
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.94.dev2 → eval_protocol-0.2.94.dev3}/eval_protocol.egg-info/SOURCES.txt

@@ -42,9 +42,9 @@ eval_protocol.egg-info/requires.txt
 eval_protocol.egg-info/top_level.txt
 eval_protocol/../vite-app/dist/index.html
 eval_protocol/../vite-app/dist/assets/favicon-BkAAWQga.png
-eval_protocol/../vite-app/dist/assets/index-BIhepl19.css
-eval_protocol/../vite-app/dist/assets/index-DaovgarD.js
-eval_protocol/../vite-app/dist/assets/index-DaovgarD.js.map
+eval_protocol/../vite-app/dist/assets/index-CuQbfdPD.js
+eval_protocol/../vite-app/dist/assets/index-CuQbfdPD.js.map
+eval_protocol/../vite-app/dist/assets/index-iZp_HgyW.css
 eval_protocol/../vite-app/dist/assets/logo-light-BprIBJQW.png
 eval_protocol/adapters/__init__.py
 eval_protocol/adapters/base.py