eval-protocol 0.3.27__tar.gz → 0.3.28__tar.gz

{eval_protocol-0.3.27/eval_protocol.egg-info → eval_protocol-0.3.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.27
+Version: 0.3.28
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.3.27 → eval_protocol-0.3.28}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2026-04-06T16:29:39-0700",
+ "date": "2026-04-28T20:08:36-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "f77b26fd89b097ac0b09b56ccdd1b11d1feeeb72",
- "version": "0.3.27"
+ "full-revisionid": "fd4358c26f9cbe57bb9c0091f28ae6642194941b",
+ "version": "0.3.28"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.3.27 → eval_protocol-0.3.28}/eval_protocol/adapters/fireworks_tracing.py

@@ -5,7 +5,10 @@ to pull data from Langfuse deployments with simplified retry logic handling.
 """
 
 from __future__ import annotations
+import asyncio
+import json
 import logging
+import aiohttp
 import requests
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Protocol
@@ -14,6 +17,7 @@ import os
 from eval_protocol.models import EvaluationRow, InputMetadata, ExecutionMetadata, Message
 from .base import BaseAdapter
 from .utils import extract_messages_from_data
+from ..common_utils import get_user_agent
 
 logger = logging.getLogger(__name__)
 
@@ -280,8 +284,6 @@ class FireworksTracingAdapter(BaseAdapter):
         if not tags:
             raise ValueError("At least one tag is required to fetch logs")
 
-        from ..common_utils import get_user_agent
-
         headers = {
             "Authorization": f"Bearer {self._get_api_key()}",
             "User-Agent": get_user_agent(),
@@ -327,6 +329,83 @@ class FireworksTracingAdapter(BaseAdapter):
             )
         return results
 
+    async def async_search_logs(
+        self, session: aiohttp.ClientSession, tags: List[str], limit: int = 100, hours_back: int = 24
+    ) -> List[Dict[str, Any]]:
+        """Async version of search_logs, reuses a caller-provided aiohttp session."""
+        if not tags:
+            raise ValueError("At least one tag is required to fetch logs")
+
+        params: Dict[str, Any] = {"tags": tags, "limit": limit, "hours_back": hours_back, "program": "eval_protocol"}
+        headers = {"Authorization": f"Bearer {self._get_api_key()}", "User-Agent": get_user_agent()}
+        timeout = aiohttp.ClientTimeout(total=self.timeout)
+
+        urls_to_try = [f"{self.base_url}/logs", f"{self.base_url}/v1/logs"]
+        data: Dict[str, Any] = {}
+        last_error: Optional[str] = None
+        for url in urls_to_try:
+            try:
+                async with session.get(url, params=params, headers=headers, timeout=timeout) as resp:
+                    if resp.status == 404:
+                        last_error = f"404 for {url}"
+                        continue
+                    resp.raise_for_status()
+                    data = (await resp.json(content_type=None)) or {}
+                    break
+            except (aiohttp.ClientError, asyncio.TimeoutError, json.JSONDecodeError) as e:
+                last_error = str(e)
+                continue
+        else:
+            if last_error:
+                logger.error("Failed to fetch logs from Fireworks (tried %s): %s", urls_to_try, last_error)
+            return []
+
+        entries: List[Dict[str, Any]] = data.get("entries", []) or []
+        results: List[Dict[str, Any]] = []
+        for e in entries:
+            results.append(
+                {
+                    "timestamp": e.get("timestamp"),
+                    "message": e.get("message"),
+                    "severity": e.get("severity", "INFO"),
+                    "tags": e.get("tags", []),
+                    "status": e.get("status"),
+                    "extras": e.get("extras"),
+                }
+            )
+        return results
+
+    async def async_get_status(self, session: aiohttp.ClientSession, rollout_id: str) -> Optional[Dict[str, Any]]:
+        """Fetch rollout status from the lightweight /status endpoint.
+
+        Returns the parsed JSON response or None if the status is not yet available.
+        Response shape: {"rollout_id": "...", "status": {"code": ...} | null, "extras": {...} | null}
+        """
+        headers = {
+            "Authorization": f"Bearer {self._get_api_key()}",
+            "User-Agent": get_user_agent(),
+        }
+        params: Dict[str, Any] = {"rollout_id": rollout_id}
+        timeout = aiohttp.ClientTimeout(total=self.timeout)
+
+        urls_to_try = [f"{self.base_url}/v1/status", f"{self.base_url}/status"]
+        last_error: Optional[str] = None
+        for url in urls_to_try:
+            try:
+                async with session.get(url, params=params, headers=headers, timeout=timeout) as resp:
+                    if resp.status == 404:
+                        last_error = f"404 for {url}"
+                        continue
+                    resp.raise_for_status()
+                    return (await resp.json(content_type=None)) or {}
+            except (aiohttp.ClientError, asyncio.TimeoutError, json.JSONDecodeError) as e:
+                last_error = str(e)
+                continue
+
+        if last_error:
+            logger.error("Failed to fetch status from Fireworks (tried %s): %s", urls_to_try, last_error)
+        return None
+
     def get_evaluation_rows(
         self,
         tags: List[str],
@@ -411,8 +490,6 @@ class FireworksTracingAdapter(BaseAdapter):
         else:
             url = f"{self.base_url}/v1/traces/pointwise"
 
-        from ..common_utils import get_user_agent
-
         headers = {
             "Authorization": f"Bearer {self._get_api_key()}",
             "User-Agent": get_user_agent(),

{eval_protocol-0.3.27 → eval_protocol-0.3.28}/eval_protocol/benchmarks/test_frozen_lake.py

@@ -42,7 +42,7 @@ def frozen_lake_to_evaluation_row(data: List[Dict[str, Any]]) -> List[Evaluation
         {
             "temperature": 0.0,
             "max_tokens": 4096,
-            "model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct-0905",
+            "model": "fireworks_ai/accounts/fireworks/models/kimi-k2p5",
         }
     ],
     rollout_processor=MCPGymRolloutProcessor(),

{eval_protocol-0.3.27 → eval_protocol-0.3.28}/eval_protocol/pytest/remote_rollout_processor.py

@@ -52,7 +52,8 @@ class RemoteRolloutProcessor(RolloutProcessor):
 
     def _get_or_create_session(self) -> aiohttp.ClientSession:
         if self._session is None or self._session.closed:
-            self._session = aiohttp.ClientSession()
+            connector = aiohttp.TCPConnector(limit=0)  # no total connection limit
+            self._session = aiohttp.ClientSession(connector=connector)
         return self._session
 
     def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) -> List[asyncio.Task[EvaluationRow]]:
@@ -99,7 +100,7 @@ class RemoteRolloutProcessor(RolloutProcessor):
             # Fire-and-poll
             init_url = f"{remote_base_url}/init"
 
-            timeout_init = aiohttp.ClientTimeout(total=300)
+            timeout_init = aiohttp.ClientTimeout(total=600)
 
             try:
                 session = self._get_or_create_session()
@@ -114,52 +115,33 @@ class RemoteRolloutProcessor(RolloutProcessor):
                     await resp.read()  # Drain the response body and release the connection back to the pool
             except asyncio.TimeoutError:
                 raise TimeoutError(
-                    f"The /init endpoint tried {init_url} with {init_payload.model_dump()} but timed out after 300 seconds."
+                    f"The /init endpoint tried {init_url} with {init_payload.model_dump()} but timed out after 600 seconds."
                 )
 
             deadline = time.time() + timeout_seconds
 
             while time.time() < deadline:
-                # Search Fireworks tracing logs for completion (run in thread to avoid blocking event loop)
-                completed_logs = await asyncio.to_thread(
-                    self._tracing_adapter.search_logs, tags=[f"rollout_id:{row.execution_metadata.rollout_id}"]
+                session = self._get_or_create_session()
+                status_result = await self._tracing_adapter.async_get_status(
+                    session,
+                    rollout_id=row.execution_metadata.rollout_id,
                 )
-                # Filter for logs that actually have status information
-                status_logs = []
-                for log in completed_logs:
-                    status_dict = log.get("status")
-                    if status_dict and isinstance(status_dict, dict) and "code" in status_dict:
-                        status_logs.append(log)
-
-                if status_logs:
-                    if len(status_logs) > 1:
-                        logger.warning(
-                            "Found %s status logs for rollout %s; expected at most 1. Using the first one: %s",
-                            len(status_logs),
-                            row.execution_metadata.rollout_id,
-                            status_logs[0],
-                        )
-                    # Use the first log with status information
-                    status_log = status_logs[0]
-                    status_dict = status_log.get("status")
-                    raw_extras = status_log.get("extras") or {}
-                    status_extras = {
-                        k: v for k, v in raw_extras.items() if k not in ("logger_name", "level", "timestamp")
-                    }
+                status = (status_result or {}).get("status")
+                if isinstance(status, dict) and "code" in status:
+                    status_code = status["code"]
+                    if status_code == Status.Code.RUNNING:
+                        await asyncio.sleep(poll_interval)
+                        continue
 
                     logger.info(
-                        f"Found status log for rollout {row.execution_metadata.rollout_id}: {status_log.get('message', '')}"
+                        "Found status for rollout %s with code %s",
+                        row.execution_metadata.rollout_id,
+                        status_code,
                     )
 
-                    status_code = status_dict.get("code")
-                    status_message = status_dict.get("message", "")
-                    status_details = status_dict.get("details", [])
-
-                    logger.info(
-                        f"Found Fireworks log for rollout {row.execution_metadata.rollout_id} with status code {status_code}"
-                    )
+                    status_message = status.get("message", "") or ""
+                    status_details = status.get("details", []) or []
 
-                    # Create and raise exception if appropriate, preserving original message
                     exception = exception_for_status_code(status_code, status_message)
                     if exception is not None:
                         raise exception
@@ -170,10 +152,12 @@ class RemoteRolloutProcessor(RolloutProcessor):
                         details=status_details,
                     )
 
-                    if row.execution_metadata.extra:
-                        row.execution_metadata.extra.update(status_extras)
-                    else:
-                        row.execution_metadata.extra = status_extras
+                    status_extras = (status_result or {}).get("extras")
+                    if isinstance(status_extras, dict):
+                        if row.execution_metadata.extra:
+                            row.execution_metadata.extra.update(status_extras)
+                        else:
+                            row.execution_metadata.extra = status_extras
 
                     logger.info("Stopping polling for rollout %s", row.execution_metadata.rollout_id)
                     break

{eval_protocol-0.3.27 → eval_protocol-0.3.28}/eval_protocol/quickstart/aha_judge/llm_judge.py

@@ -17,7 +17,7 @@ from openai import AsyncOpenAI
 
 
 async def aha_judge(
-    row: EvaluationRow, judge_name: str = "kimi-k2-instruct-0905", adapter: Optional[BaseAdapter] = None
+    row: EvaluationRow, judge_name: str = "kimi-k2p5", adapter: Optional[BaseAdapter] = None
 ) -> EvaluationRow:
     """
     LLM Judge evaluation using Arena-Hard-Auto style pairwise comparisons for a single row.

{eval_protocol-0.3.27 → eval_protocol-0.3.28}/eval_protocol/quickstart/aha_judge/llm_judge_openai_responses.py

@@ -50,7 +50,7 @@ def openai_responses_data_generator():
             "model": "fireworks_ai/accounts/fireworks/models/deepseek-v3p1",
         },
         {
-            "model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct-0905",
+            "model": "fireworks_ai/accounts/fireworks/models/kimi-k2p5",
         },
     ],
 )

{eval_protocol-0.3.27 → eval_protocol-0.3.28}/eval_protocol/quickstart/aha_judge/utils.py

@@ -49,8 +49,8 @@ JUDGE_CONFIGS = {
         "api_key": os.getenv("GEMINI_API_KEY"),
         "base_url": "https://generativelanguage.googleapis.com/v1beta/openai/",
     },
-    "kimi-k2-instruct-0905": {
-        "model": "accounts/fireworks/models/kimi-k2-instruct-0905",
+    "kimi-k2p5": {
+        "model": "accounts/fireworks/models/kimi-k2p5",
         "temperature": 0.6,  # Kimi recommended temperature
         "max_tokens": 131000,
         "api_key": os.getenv("FIREWORKS_API_KEY"),

{eval_protocol-0.3.27 → eval_protocol-0.3.28}/eval_protocol/quickstart/llm_judge.py

@@ -17,7 +17,7 @@ from openai import AsyncOpenAI
 
 
 async def aha_judge(
-    row: EvaluationRow, judge_name: str = "kimi-k2-instruct-0905", adapter: Optional[BaseAdapter] = None
+    row: EvaluationRow, judge_name: str = "kimi-k2p5", adapter: Optional[BaseAdapter] = None
 ) -> EvaluationRow:
     """
     LLM Judge evaluation using Arena-Hard-Auto style pairwise comparisons for a single row.

{eval_protocol-0.3.27 → eval_protocol-0.3.28}/eval_protocol/quickstart/utils.py

@@ -51,8 +51,8 @@ JUDGE_CONFIGS = {
         "api_key": os.getenv("GEMINI_API_KEY"),
         "base_url": "https://generativelanguage.googleapis.com/v1beta/openai/",
     },
-    "kimi-k2-instruct-0905": {
-        "model": "accounts/fireworks/models/kimi-k2-instruct-0905",
+    "kimi-k2p5": {
+        "model": "accounts/fireworks/models/kimi-k2p5",
         "temperature": 0.6,  # Kimi recommended temperature
         "max_tokens": 131000,
         "api_key": os.getenv("FIREWORKS_API_KEY"),

{eval_protocol-0.3.27 → eval_protocol-0.3.28}/eval_protocol/training/gepa_utils.py

@@ -40,7 +40,7 @@ REFLECTION_LM_CONFIGS = {
     },
     # Fireworks models
     "kimi-k2": {
-        "model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct-0905",
+        "model": "fireworks_ai/accounts/fireworks/models/kimi-k2p5",
         "temperature": 0.6,
         "max_tokens": 131000,
     },
@@ -68,7 +68,7 @@ def build_reflection_lm(reflection_lm_name: str) -> LM:
 
     Args:
         reflection_lm_name: One of the predefined configs ("gpt-5", "gpt-4o",
-                           "claude-sonnet", "kimi-k2-instruct-0905")
+                           "claude-sonnet", "kimi-k2p5")
                            OR a raw LiteLLM model string (e.g., "openai/gpt-4o")
 
     Returns:

{eval_protocol-0.3.27 → eval_protocol-0.3.28/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.27
+Version: 0.3.28
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT