eval-protocol 0.2.95.dev1__tar.gz → 0.2.96__tar.gz

{eval_protocol-0.2.95.dev1/eval_protocol.egg-info → eval_protocol-0.2.96}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.95.dev1
+Version: 0.2.96
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT
@@ -34,7 +34,7 @@ Requires-Dist: toml>=0.10.0
 Requires-Dist: loguru>=0.6.0
 Requires-Dist: docstring-parser>=0.15
 Requires-Dist: rich>=12.0.0
-Requires-Dist: psutil>=5.8.0
+Requires-Dist: psutil>=6.0.0
 Requires-Dist: addict>=2.4.0
 Requires-Dist: deepdiff>=6.0.0
 Requires-Dist: websockets>=15.0.1

{eval_protocol-0.2.95.dev1 → eval_protocol-0.2.96}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-12-02T21:46:38-0800",
+ "date": "2025-12-08T17:48:51-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "01bc8e998a3a0370fc0631d39b4fbd8b4b5c7941",
- "version": "0.2.95-dev.1"
+ "full-revisionid": "0902602fdca3f7d80e259fd29bd293758a79c44a",
+ "version": "0.2.96"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.95.dev1 → eval_protocol-0.2.96}/eval_protocol/cli_commands/create_rft.py

@@ -702,8 +702,13 @@ def _create_rft_job(
     print(f"Prepared RFT job for evaluator '{evaluator_id}' using dataset '{dataset_id}'")
     if getattr(args, "evaluation_dataset", None):
         body["evaluationDataset"] = args.evaluation_dataset
-    if getattr(args, "output_model", None):
-        body.setdefault("trainingConfig", {})["outputModel"] = f"accounts/{account_id}/models/{args.output_model}"
+    
+    output_model_arg = getattr(args, "output_model", None)
+    if output_model_arg:
+        if len(output_model_arg) > 63:
+            print(f"Error: Output model name '{output_model_arg}' exceeds 63 characters.")
+            return 1
+        body.setdefault("trainingConfig", {})["outputModel"] = f"accounts/{account_id}/models/{output_model_arg}"
     else:
         # Auto-generate output model name if not provided
         auto_output_model = build_default_output_model(evaluator_id)

{eval_protocol-0.2.95.dev1 → eval_protocol-0.2.96}/eval_protocol/fireworks_rft.py

@@ -6,6 +6,7 @@ import sys
 import tempfile
 import time
 import uuid
+import hashlib
 from pathlib import Path
 from typing import Any, Callable, Dict, Iterable, Optional, Tuple
 from urllib.parse import urlencode
@@ -216,6 +217,24 @@ def build_default_dataset_id(evaluator_id: str) -> str:
 def build_default_output_model(evaluator_id: str) -> str:
     base = evaluator_id.lower().replace("_", "-")
     uuid_suffix = str(uuid.uuid4())[:4]
+    
+    # suffix is "-rft-{4chars}" -> 9 chars
+    suffix_len = 9
+    max_len = 63
+    
+    # Check if we need to truncate
+    if len(base) + suffix_len > max_len:
+        # Calculate hash of the full base to preserve uniqueness
+        hash_digest = hashlib.sha256(base.encode("utf-8")).hexdigest()[:6]
+        # New structure: {truncated_base}-{hash}-{uuid_suffix}
+        # Space needed for "-{hash}" is 1 + 6 = 7
+        hash_part_len = 7
+        
+        allowed_base_len = max_len - suffix_len - hash_part_len
+        truncated_base = base[:allowed_base_len].strip("-")
+        
+        return f"{truncated_base}-{hash_digest}-rft-{uuid_suffix}"
+
     return f"{base}-rft-{uuid_suffix}"
 
 

{eval_protocol-0.2.95.dev1 → eval_protocol-0.2.96}/eval_protocol/mcp/execution/base_policy.py

@@ -199,6 +199,13 @@ class LLMBasePolicy(PlaybackPolicyBase, ABC):
         if message.get("tool_calls"):
             assistant_message_for_history["tool_calls"] = message["tool_calls"]
 
+        # Preserve specific fields from provider_specific_fields if present
+        if message.get("provider_specific_fields"):
+            if message["provider_specific_fields"].get("reasoning_details"):
+                assistant_message_for_history["reasoning_details"] = message["provider_specific_fields"][
+                    "reasoning_details"
+                ]
+
         # Add to actual conversation history
         conversation_history.append(assistant_message_for_history)
 

{eval_protocol-0.2.95.dev1 → eval_protocol-0.2.96}/eval_protocol/mcp/execution/policy.py

@@ -146,7 +146,7 @@ class LiteLLMPolicy(LLMBasePolicy):
             Clean messages with only OpenAI API compatible fields
         """
         # Standard OpenAI message fields
-        allowed_fields = {"role", "content", "tool_calls", "tool_call_id", "name"}
+        allowed_fields = {"role", "content", "tool_calls", "tool_call_id", "name", "reasoning_details"}
 
         clean_messages = []
         for msg in messages:
@@ -217,12 +217,15 @@ class LiteLLMPolicy(LLMBasePolicy):
                 logger.debug(f"🔄 API call for model: {self.model_id}")
 
             # LiteLLM already returns OpenAI-compatible format
+            message_obj = getattr(response.choices[0], "message", object())
+
             return {
                 "choices": [
                     {
                         "message": {
-                            "role": getattr(getattr(response.choices[0], "message", object()), "role", "assistant"),
-                            "content": getattr(getattr(response.choices[0], "message", object()), "content", None),
+                            "role": getattr(message_obj, "role", "assistant"),
+                            "content": getattr(message_obj, "content", None),
+                            "provider_specific_fields": getattr(message_obj, "provider_specific_fields", None),
                             "tool_calls": (
                                 [
                                     {
@@ -233,12 +236,9 @@ class LiteLLMPolicy(LLMBasePolicy):
                                             "arguments": getattr(getattr(tc, "function", None), "arguments", "{}"),
                                         },
                                     }
-                                    for tc in (
-                                        getattr(getattr(response.choices[0], "message", object()), "tool_calls", [])
-                                        or []
-                                    )
+                                    for tc in (getattr(message_obj, "tool_calls", []) or [])
                                 ]
-                                if getattr(getattr(response.choices[0], "message", object()), "tool_calls", None)
+                                if getattr(message_obj, "tool_calls", None)
                                 else []
                             ),
                         },

{eval_protocol-0.2.95.dev1 → eval_protocol-0.2.96}/eval_protocol/models.py

@@ -550,7 +550,7 @@ class MetricResult(BaseModel):
     """
 
     is_score_valid: bool = True
-    score: float = Field(..., ge=0.0, le=1.0)
+    score: float = Field(..., description="The score for this metric.")
     reason: str
     data: Dict[str, Any] = Field(default_factory=dict, description="Optional extra metric data for debugging.")
 

{eval_protocol-0.2.95.dev1 → eval_protocol-0.2.96/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.95.dev1
+Version: 0.2.96
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT
@@ -34,7 +34,7 @@ Requires-Dist: toml>=0.10.0
 Requires-Dist: loguru>=0.6.0
 Requires-Dist: docstring-parser>=0.15
 Requires-Dist: rich>=12.0.0
-Requires-Dist: psutil>=5.8.0
+Requires-Dist: psutil>=6.0.0
 Requires-Dist: addict>=2.4.0
 Requires-Dist: deepdiff>=6.0.0
 Requires-Dist: websockets>=15.0.1

{eval_protocol-0.2.95.dev1 → eval_protocol-0.2.96}/eval_protocol.egg-info/SOURCES.txt

@@ -326,6 +326,7 @@ tests/test_lean_prover.py
 tests/test_lean_prover_runner.py
 tests/test_length.py
 tests/test_list_comparison_math_reward.py
+tests/test_litellm_policy_provider_fields.py
 tests/test_logs_server.py
 tests/test_logs_server_simple.py
 tests/test_math.py

{eval_protocol-0.2.95.dev1 → eval_protocol-0.2.96}/eval_protocol.egg-info/requires.txt

@@ -22,7 +22,7 @@ toml>=0.10.0
 loguru>=0.6.0
 docstring-parser>=0.15
 rich>=12.0.0
-psutil>=5.8.0
+psutil>=6.0.0
 addict>=2.4.0
 deepdiff>=6.0.0
 websockets>=15.0.1

{eval_protocol-0.2.95.dev1 → eval_protocol-0.2.96}/pyproject.toml

@@ -42,7 +42,7 @@ dependencies = [
     "loguru>=0.6.0",
     "docstring-parser>=0.15",
     "rich>=12.0.0",
-    "psutil>=5.8.0",
+    "psutil>=6.0.0",
     "addict>=2.4.0",
     "deepdiff>=6.0.0",
     "websockets>=15.0.1",

eval_protocol-0.2.96/tests/test_litellm_policy_provider_fields.py

@@ -0,0 +1,95 @@
+import types
+
+import pytest
+
+import eval_protocol.mcp.execution.policy as policy_mod
+from eval_protocol.mcp.execution.policy import LiteLLMPolicy
+
+
+@pytest.mark.asyncio
+async def test_litellm_policy_surfaces_provider_specific_reasoning_details(monkeypatch):
+    """
+    Ensure that provider_specific_fields from the LiteLLM message object are
+    preserved on the returned message dict from LiteLLMPolicy._make_llm_call.
+    """
+
+    # Define a fake ModelResponse base class and patch the module's ModelResponse
+    class FakeModelResponseBase: ...
+
+    policy_mod.ModelResponse = FakeModelResponseBase
+
+    async def fake_acompletion(*args, **kwargs):
+        # This mimics the LiteLLM Message object shape we rely on in policy._make_llm_call
+        message_obj = types.SimpleNamespace(
+            role="assistant",
+            content="",
+            tool_calls=[
+                types.SimpleNamespace(
+                    id="tool_get_reservation_details_123",
+                    type="function",
+                    function=types.SimpleNamespace(
+                        name="get_reservation_details",
+                        arguments='{"reservation_id":"EHGLP3"}',
+                    ),
+                )
+            ],
+            provider_specific_fields={
+                "reasoning_details": [{"id": "tool_get_reservation_details_123", "type": "reasoning.encrypted"}],
+                "custom_field": "keep_me",
+            },
+        )
+
+        class FakeModelResponse(FakeModelResponseBase):
+            def __init__(self) -> None:
+                self.choices = [
+                    types.SimpleNamespace(
+                        finish_reason="tool_calls",
+                        index=0,
+                        message=message_obj,
+                    )
+                ]
+                self.usage = types.SimpleNamespace(
+                    prompt_tokens=10,
+                    completion_tokens=5,
+                    total_tokens=15,
+                )
+
+        return FakeModelResponse()
+
+    # Patch acompletion so we don't hit the network
+    monkeypatch.setattr(policy_mod, "acompletion", fake_acompletion)
+
+    # Use a concrete policy instance; base_url/model_id values don't matter for this unit test
+    policy = LiteLLMPolicy(model_id="openrouter/google/gemini-3-pro-preview", use_caching=False)
+
+    messages = [
+        {
+            "role": "assistant",
+            "content": "",
+            "tool_calls": [
+                {
+                    "id": "tool_get_reservation_details_123",
+                    "type": "function",
+                    "function": {"name": "get_reservation_details", "arguments": '{"reservation_id":"EHGLP3"}'},
+                }
+            ],
+        }
+    ]
+
+    # No tools are needed for this test – we only care about the returned message shape
+    result = await policy._make_llm_call(messages, tools=[])
+
+    assert "choices" in result
+    assert len(result["choices"]) == 1
+    msg = result["choices"][0]["message"]
+
+    # Core fields should be present
+    assert msg["role"] == "assistant"
+    assert isinstance(msg.get("tool_calls"), list)
+
+    # provider_specific_fields should be preserved on the message
+    ps = msg.get("provider_specific_fields")
+    assert isinstance(ps, dict)
+    assert ps["reasoning_details"] == [{"id": "tool_get_reservation_details_123", "type": "reasoning.encrypted"}]
+    # Non-core provider_specific_fields should also be preserved
+    assert ps.get("custom_field") == "keep_me"