eval-protocol 0.2.26__tar.gz → 0.2.28__tar.gz

{eval_protocol-0.2.26/eval_protocol.egg-info → eval_protocol-0.2.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.26
+Version: 0.2.28
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT
@@ -119,133 +119,114 @@ Dynamic: license-file
 # Eval Protocol (EP)
 
 [![PyPI - Version](https://img.shields.io/pypi/v/eval-protocol)](https://pypi.org/project/eval-protocol/)
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/eval-protocol/python-sdk)
 
-**The open-source toolkit for building your internal model leaderboard.**
+**Stop guessing which AI model to use. Build a data-driven model leaderboard.**
 
-When you have multiple AI models to choose from—different versions, providers,
-or configurations—how do you know which one is best for your use case?
+With hundreds of models and configs, you need objective data to choose the right one for your use case. EP helps you evaluate real traces, compare models, and visualize results locally.
 
 ## 🚀 Features
 
-- **Custom Evaluations**: Write evaluations tailored to your specific business needs
-- **Auto-Evaluation**: Stack-rank models using LLMs as judges with just model traces using out-of-the-box evaluators
-- **RL Environments via MCP**: Build reinforcement learning environments using the Model Control Protocol (MCP) to simulate user interactions and advanced evaluation scenarios
-- **Consistent Testing**: Test across various models and configurations with a unified framework
-- **Resilient Runtime**: Automatic retries for unstable LLM APIs and concurrent execution for long-running evaluations
-- **Rich Visualizations**: Built-in pivot tables and visualizations for result analysis
-- **Data-Driven Decisions**: Make informed model deployment decisions based on comprehensive evaluation results
+- **Pytest authoring**: `@evaluation_test` decorator to configure evaluations
+- **Robust rollouts**: Handles flaky LLM APIs and parallel execution
+- **Integrations**: Works with Langfuse, LangSmith, Braintrust, Responses API
+- **Agent support**: LangGraph and Pydantic AI
+- **MCP RL envs**: Build reinforcement learning environments with MCP
+- **Built-in benchmarks**: AIME, tau-bench
+- **LLM judge**: Stack-rank models using pairwise Arena-Hard-Auto
+- **Local UI**: Pivot/table views for real-time analysis
 
-## Quick Examples
+## ⚡ Quickstart (no labels needed)
 
-### Basic Model Comparison
+Install with your tracing platform extras and set API keys:
 
-Compare models on a simple formatting task:
-
-```python test_bold_format.py
-from eval_protocol.models import EvaluateResult, EvaluationRow, Message
-from eval_protocol.pytest import default_single_turn_rollout_processor, evaluation_test
+```bash
+pip install 'eval-protocol[langfuse]'
 
-@evaluation_test(
-    input_messages=[
-        [
-            Message(role="system", content="Use bold text to highlight important information."),
-            Message(role="user", content="Explain why evaluations matter for AI agents. Make it dramatic!"),
-        ],
-    ],
-    completion_params=[
-        {"model": "fireworks/accounts/fireworks/models/llama-v3p1-8b-instruct"},
-        {"model": "openai/gpt-4"},
-        {"model": "anthropic/claude-3-sonnet"}
-    ],
-    rollout_processor=default_single_turn_rollout_processor,
-    mode="pointwise",
-)
-def test_bold_format(row: EvaluationRow) -> EvaluationRow:
-    """Check if the model's response contains bold text."""
-    assistant_response = row.messages[-1].content
+# Model API keys (set what you need)
+export OPENAI_API_KEY=...
+export FIREWORKS_API_KEY=...
+export GEMINI_API_KEY=...
 
-    if assistant_response is None:
-        row.evaluation_result = EvaluateResult(score=0.0, reason="No response")
-        return row
+# Platform keys
+export LANGFUSE_PUBLIC_KEY=...
+export LANGFUSE_SECRET_KEY=...
+export LANGFUSE_HOST=https://your-deployment.com  # optional
+```
 
-    has_bold = "**" in str(assistant_response)
-    score = 1.0 if has_bold else 0.0
-    reason = "Contains bold text" if has_bold else "No bold text found"
+Minimal evaluation using the built-in AHA judge:
 
-    row.evaluation_result = EvaluateResult(score=score, reason=reason)
-    return row
-```
+```python
+from datetime import datetime
+import pytest
+
+from eval_protocol import (
+    evaluation_test,
+    aha_judge,
+    EvaluationRow,
+    SingleTurnRolloutProcessor,
+    DynamicDataLoader,
+    create_langfuse_adapter,
+)
 
-### Using Datasets
 
-Evaluate models on existing datasets:
+def langfuse_data_generator() -> list[EvaluationRow]:
+    adapter = create_langfuse_adapter()
+    return adapter.get_evaluation_rows(
+        to_timestamp=datetime.utcnow(),
+        limit=20,
+        sample_size=5,
+    )
 
-```python
-from eval_protocol.pytest import evaluation_test
-from eval_protocol.adapters.huggingface import create_gsm8k_adapter
 
-@evaluation_test(
-    input_dataset=["development/gsm8k_sample.jsonl"],  # Local JSONL file
-    dataset_adapter=create_gsm8k_adapter(),  # Adapter to convert data
-    completion_params=[
-        {"model": "openai/gpt-4"},
-        {"model": "anthropic/claude-3-sonnet"}
+@pytest.mark.parametrize(
+    "completion_params",
+    [
+        {"model": "openai/gpt-4.1"},
+        {"model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b"},
     ],
-    mode="pointwise"
 )
-def test_math_reasoning(row: EvaluationRow) -> EvaluationRow:
-    # Your evaluation logic here
-    return row
+@evaluation_test(
+    data_loaders=DynamicDataLoader(generators=[langfuse_data_generator]),
+    rollout_processor=SingleTurnRolloutProcessor(),
+)
+async def test_llm_judge(row: EvaluationRow) -> EvaluationRow:
+    return await aha_judge(row)
 ```
 
+Run it:
 
-## 📚 Resources
+```bash
+pytest -q -s
+```
 
-- **[Documentation](https://evalprotocol.io)** - Complete guides and API reference
-- **[Discord](https://discord.com/channels/1137072072808472616/1400975572405850155)** - Community discussions
-- **[GitHub](https://github.com/eval-protocol/python-sdk)** - Source code and examples
+The pytest output includes local links for a leaderboard and row-level traces (pivot/table) at `http://localhost:8000`.
 
 ## Installation
 
-**This library requires Python >= 3.10.**
+This library requires Python >= 3.10.
 
-### Basic Installation
-
-Install with pip:
+### pip
 
 ```bash
 pip install eval-protocol
 ```
 
-### Recommended Installation with uv
-
-For better dependency management and faster installs, we recommend using [uv](https://docs.astral.sh/uv/):
+### uv (recommended)
 
 ```bash
-# Install uv if you haven't already
+# Install uv (if needed)
 curl -LsSf https://astral.sh/uv/install.sh | sh
 
-# Install eval-protocol
+# Add to your project
 uv add eval-protocol
 ```
 
-### Optional Dependencies
-
-Install with additional features:
-
-```bash
-# For Langfuse integration
-pip install 'eval-protocol[langfuse]'
-
-# For HuggingFace datasets
-pip install 'eval-protocol[huggingface]'
-
-# For all adapters
-pip install 'eval-protocol[adapters]'
+## 📚 Resources
 
-# For development
-pip install 'eval-protocol[dev]'
-```
+- **[Documentation](https://evalprotocol.io)** – Guides and API reference
+- **[Discord](https://discord.com/channels/1137072072808472616/1400975572405850155)** – Community
+- **[GitHub](https://github.com/eval-protocol/python-sdk)** – Source and examples
 
 ## License
 

eval_protocol-0.2.28/README.md

@@ -0,0 +1,115 @@
+# Eval Protocol (EP)
+
+[![PyPI - Version](https://img.shields.io/pypi/v/eval-protocol)](https://pypi.org/project/eval-protocol/)
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/eval-protocol/python-sdk)
+
+**Stop guessing which AI model to use. Build a data-driven model leaderboard.**
+
+With hundreds of models and configs, you need objective data to choose the right one for your use case. EP helps you evaluate real traces, compare models, and visualize results locally.
+
+## 🚀 Features
+
+- **Pytest authoring**: `@evaluation_test` decorator to configure evaluations
+- **Robust rollouts**: Handles flaky LLM APIs and parallel execution
+- **Integrations**: Works with Langfuse, LangSmith, Braintrust, Responses API
+- **Agent support**: LangGraph and Pydantic AI
+- **MCP RL envs**: Build reinforcement learning environments with MCP
+- **Built-in benchmarks**: AIME, tau-bench
+- **LLM judge**: Stack-rank models using pairwise Arena-Hard-Auto
+- **Local UI**: Pivot/table views for real-time analysis
+
+## ⚡ Quickstart (no labels needed)
+
+Install with your tracing platform extras and set API keys:
+
+```bash
+pip install 'eval-protocol[langfuse]'
+
+# Model API keys (set what you need)
+export OPENAI_API_KEY=...
+export FIREWORKS_API_KEY=...
+export GEMINI_API_KEY=...
+
+# Platform keys
+export LANGFUSE_PUBLIC_KEY=...
+export LANGFUSE_SECRET_KEY=...
+export LANGFUSE_HOST=https://your-deployment.com  # optional
+```
+
+Minimal evaluation using the built-in AHA judge:
+
+```python
+from datetime import datetime
+import pytest
+
+from eval_protocol import (
+    evaluation_test,
+    aha_judge,
+    EvaluationRow,
+    SingleTurnRolloutProcessor,
+    DynamicDataLoader,
+    create_langfuse_adapter,
+)
+
+
+def langfuse_data_generator() -> list[EvaluationRow]:
+    adapter = create_langfuse_adapter()
+    return adapter.get_evaluation_rows(
+        to_timestamp=datetime.utcnow(),
+        limit=20,
+        sample_size=5,
+    )
+
+
+@pytest.mark.parametrize(
+    "completion_params",
+    [
+        {"model": "openai/gpt-4.1"},
+        {"model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b"},
+    ],
+)
+@evaluation_test(
+    data_loaders=DynamicDataLoader(generators=[langfuse_data_generator]),
+    rollout_processor=SingleTurnRolloutProcessor(),
+)
+async def test_llm_judge(row: EvaluationRow) -> EvaluationRow:
+    return await aha_judge(row)
+```
+
+Run it:
+
+```bash
+pytest -q -s
+```
+
+The pytest output includes local links for a leaderboard and row-level traces (pivot/table) at `http://localhost:8000`.
+
+## Installation
+
+This library requires Python >= 3.10.
+
+### pip
+
+```bash
+pip install eval-protocol
+```
+
+### uv (recommended)
+
+```bash
+# Install uv (if needed)
+curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Add to your project
+uv add eval-protocol
+```
+
+## 📚 Resources
+
+- **[Documentation](https://evalprotocol.io)** – Guides and API reference
+- **[Discord](https://discord.com/channels/1137072072808472616/1400975572405850155)** – Community
+- **[GitHub](https://github.com/eval-protocol/python-sdk)** – Source and examples
+
+## License
+
+[MIT](LICENSE)

{eval_protocol-0.2.26 → eval_protocol-0.2.28}/eval_protocol/__init__.py

@@ -22,6 +22,7 @@ from .mcp_env import (
     rollout,
     test_mcp,
 )
+from .data_loader import DynamicDataLoader, InlineDataLoader
 
 # Try to import FireworksPolicy if available
 try:
@@ -32,13 +33,13 @@ except (ImportError, AttributeError):
     _FIREWORKS_AVAILABLE = False
 # Import submodules to make them available via eval_protocol.rewards, etc.
 from . import mcp, rewards
-from .models import EvaluateResult, Message, MetricResult, EvaluationRow
+from .models import EvaluateResult, Message, MetricResult, EvaluationRow, InputMetadata
 from .playback_policy import PlaybackPolicyBase
 from .resources import create_llm_resource
 from .reward_function import RewardFunction
 from .typed_interface import reward_function
 from .quickstart import aha_judge, multi_turn_assistant_to_ground_truth, assistant_to_ground_truth
-from .pytest import evaluation_test, SingleTurnRolloutProcessor
+from .pytest import evaluation_test, SingleTurnRolloutProcessor, RemoteRolloutProcessor
 from .pytest.parameterize import DefaultParameterIdGenerator
 
 try:
@@ -61,11 +62,23 @@ try:
 except ImportError:
     LangSmithAdapter = None
 
+# Remote server types
+from .types.remote_rollout_processor import (
+    InitRequest,
+    RolloutMetadata,
+    StatusResponse,
+    create_langfuse_config_tags,
+)
 
 warnings.filterwarnings("default", category=DeprecationWarning, module="eval_protocol")
 
 __all__ = [
+    "RemoteRolloutProcessor",
+    "InputMetadata",
+    "EvaluationRow",
     "DefaultParameterIdGenerator",
+    "DynamicDataLoader",
+    "InlineDataLoader",
     "aha_judge",
     "multi_turn_assistant_to_ground_truth",
     "assistant_to_ground_truth",
@@ -107,6 +120,11 @@ __all__ = [
     # Submodules
     "rewards",
     "mcp",
+    # Remote server types
+    "InitRequest",
+    "RolloutMetadata",
+    "StatusResponse",
+    "create_langfuse_config_tags",
 ]
 
 from . import _version

{eval_protocol-0.2.26 → eval_protocol-0.2.28}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-09-23T16:59:02-0700",
+ "date": "2025-09-26T11:40:07-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "bcc2d7c22085ccddf46d952e3481012250245d90",
- "version": "0.2.26"
+ "full-revisionid": "f9c6f1be71edb167cdbb8fc1d57e71ec3df4fe18",
+ "version": "0.2.28"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.26 → eval_protocol-0.2.28}/eval_protocol/adapters/langfuse.py

@@ -12,7 +12,7 @@ from datetime import datetime, timedelta
 from typing import Any, Dict, List, Optional, Protocol, TYPE_CHECKING, cast
 
 from langfuse.api.resources.commons.types.observations_view import ObservationsView
-from eval_protocol.models import EvaluationRow, InputMetadata, Message
+from eval_protocol.models import EvaluationRow, InputMetadata, ExecutionMetadata, Message
 from .base import BaseAdapter
 from .utils import extract_messages_from_data
 
@@ -82,14 +82,41 @@ def convert_trace_to_evaluation_row(
         if not messages:
             return None
 
+        execution_metadata = ExecutionMetadata()
+        row_id = None
+
+        if trace.tags:
+            for tag in trace.tags:
+                if tag.startswith("invocation_id:"):
+                    execution_metadata.invocation_id = tag.split(":", 1)[1]
+                elif tag.startswith("experiment_id:"):
+                    execution_metadata.experiment_id = tag.split(":", 1)[1]
+                elif tag.startswith("rollout_id:"):
+                    execution_metadata.rollout_id = tag.split(":", 1)[1]
+                elif tag.startswith("run_id:"):
+                    execution_metadata.run_id = tag.split(":", 1)[1]
+                elif tag.startswith("row_id:"):
+                    row_id = tag.split(":", 1)[1]
+
+                if (
+                    execution_metadata.invocation_id
+                    and execution_metadata.experiment_id
+                    and execution_metadata.rollout_id
+                    and execution_metadata.run_id
+                    and row_id
+                ):
+                    break  # Break early if we've found all the metadata we need
+
         return EvaluationRow(
             messages=messages,
             tools=tools,
             input_metadata=InputMetadata(
+                row_id=row_id,
                 session_data={
                     "langfuse_trace_id": trace.id,  # Store the trace ID here
-                }
+                },
             ),
+            execution_metadata=execution_metadata,
         )
 
     except (AttributeError, ValueError, KeyError) as e:
@@ -259,9 +286,6 @@ class LangfuseAdapter(BaseAdapter):
         max_retries: int = 3,
         span_name: Optional[str] = None,
         converter: Optional[TraceConverter] = None,
-        metadata: Optional[Dict[str, Any]] = None,
-        requester_metadata: Optional[Dict[str, Any]] = None,
-        requester_metadata_contains: Optional[str] = None,
     ) -> List[EvaluationRow]:
         """Pull traces from Langfuse and convert to EvaluationRow format.
 
@@ -296,10 +320,6 @@ class LangfuseAdapter(BaseAdapter):
             to_timestamp = datetime.now()
             from_timestamp = to_timestamp - timedelta(hours=hours_back)
 
-        # If filtering by metadata/requester_metadata, prefer fetching metadata fields
-        if (metadata is not None or requester_metadata is not None or requester_metadata_contains) and not fields:
-            fields = "core,metadata,observations"
-
         # Collect trace summaries via pagination (up to limit)
         all_traces = []
         page = 1
@@ -332,16 +352,18 @@ class LangfuseAdapter(BaseAdapter):
                         to_timestamp=to_timestamp,
                         order_by="timestamp.desc",
                     )
+
+                    # If no results, possible due to indexing delay--remote rollout processor just finished pushing rows to Langfuse
+                    if traces and traces.meta and traces.meta.total_items == 0 and page == 1:
+                        raise Exception("Empty results - indexing delay")
+
                     break
                 except Exception as e:
                     list_retries += 1
-                    if "429" in str(e) and list_retries < max_retries:
+                    if list_retries < max_retries and ("429" in str(e) or "Empty results" in str(e)):
                         sleep_time = 2**list_retries  # Exponential backoff
                         logger.warning(
-                            "Rate limit hit on trace.list(), retrying in %ds (attempt %d/%d)",
-                            sleep_time,
-                            list_retries,
-                            max_retries,
+                            "Retrying in %ds (attempt %d/%d): %s", sleep_time, list_retries, max_retries, str(e)
                         )
                         time.sleep(sleep_time)
                     else:
@@ -379,74 +401,6 @@ class LangfuseAdapter(BaseAdapter):
             selected_traces = all_traces
             logger.debug("Processing all %d collected traces (no sampling)", len(all_traces))
 
-        # Helper to check if a trace matches provided metadata filters. We look in multiple places
-        # to account for Langfuse moving fields (e.g., metadata vs requester_metadata) and SDK shape.
-        def _trace_matches_metadata_filters(trace_obj: Any) -> bool:
-            if metadata is None and requester_metadata is None:
-                return True
-
-            def _as_dict(val: Any) -> Dict[str, Any]:
-                if val is None:
-                    return {}
-                if isinstance(val, dict):
-                    return val
-                # Some SDK objects expose .model_dump() or behave like pydantic models
-                dump = getattr(val, "model_dump", None)
-                if callable(dump):
-                    try:
-                        return dump()  # type: ignore[no-any-return]
-                    except Exception:
-                        return {}
-                return {}
-
-            # Try common locations for metadata on full trace
-            trace_meta = _as_dict(getattr(trace_obj, "metadata", None))
-            trace_req_meta = _as_dict(getattr(trace_obj, "requester_metadata", None))
-            # Some Langfuse deployments nest requester_metadata inside metadata
-            nested_req_meta = {}
-            try:
-                if isinstance(trace_meta, dict) and isinstance(trace_meta.get("requester_metadata"), dict):
-                    nested_req_meta = _as_dict(trace_meta.get("requester_metadata"))
-            except Exception:
-                nested_req_meta = {}
-
-            # Fallbacks: sometimes metadata is embedded in input
-            input_meta = {}
-            try:
-                inp = getattr(trace_obj, "input", None)
-                if isinstance(inp, dict):
-                    input_meta = _as_dict(inp.get("metadata"))
-            except Exception:
-                input_meta = {}
-
-            # Combine for matching convenience (later keys override earlier for equality check only)
-            combined_meta = {**trace_meta, **input_meta}
-            combined_req_meta = {**trace_req_meta}
-
-            # Also merge nested requester metadata when present
-            if nested_req_meta:
-                combined_req_meta = {**combined_req_meta, **nested_req_meta}
-
-            def _is_subset(needle: Dict[str, Any], haystack: Dict[str, Any]) -> bool:
-                for k, v in needle.items():
-                    if haystack.get(k) != v:
-                        return False
-                return True
-
-            ok_meta = True
-            ok_req_meta = True
-
-            if metadata is not None:
-                # Accept match if found either in metadata or requester_metadata buckets
-                ok_meta = _is_subset(metadata, combined_meta) or _is_subset(metadata, combined_req_meta)
-
-            if requester_metadata is not None:
-                ok_req_meta = _is_subset(requester_metadata, combined_req_meta) or _is_subset(
-                    requester_metadata, combined_meta
-                )
-
-            return ok_meta and ok_req_meta
-
         # Process each selected trace with sleep and retry logic
         for trace_info in selected_traces:
             # Sleep between gets to avoid rate limits
@@ -483,39 +437,6 @@ class LangfuseAdapter(BaseAdapter):
                         break  # Skip this trace
 
             if trace_full:
-                # If metadata filters are provided, skip non-matching traces early
-                try:
-                    if not _trace_matches_metadata_filters(trace_full):
-                        continue
-                except Exception:
-                    # Be permissive on filter errors; treat as non-match
-                    continue
-
-                # If observations carry requester_metadata, allow substring filtering
-                if requester_metadata_contains:
-                    contains_val = requester_metadata_contains
-                    found_match = False
-                    try:
-                        for obs in getattr(trace_full, "observations", []) or []:
-                            obs_rmd = getattr(obs, "requester_metadata", None)
-                            if isinstance(obs_rmd, dict) and any(
-                                (isinstance(v, str) and contains_val in v) for v in obs_rmd.values()
-                            ):
-                                found_match = True
-                                break
-                            obs_md = getattr(obs, "metadata", None)
-                            if isinstance(obs_md, dict):
-                                nested = obs_md.get("requester_metadata")
-                                if isinstance(nested, dict) and any(
-                                    (isinstance(v, str) and contains_val in v) for v in nested.values()
-                                ):
-                                    found_match = True
-                                    break
-                    except Exception:
-                        found_match = False
-                    if not found_match:
-                        continue
-
                 try:
                     if converter:
                         eval_row = converter(trace_full, include_tool_calls, span_name)

eval_protocol-0.2.28/eval_protocol/data_loader/__init__.py

@@ -0,0 +1,4 @@
+from .dynamic_data_loader import DynamicDataLoader
+from .inline_data_loader import InlineDataLoader
+
+__all__ = ["DynamicDataLoader", "InlineDataLoader"]

eval_protocol-0.2.28/eval_protocol/data_loader/dynamic_data_loader.py

@@ -0,0 +1,38 @@
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+
+from eval_protocol.data_loader.models import (
+    DataLoaderResult,
+    DataLoaderVariant,
+    EvaluationDataLoader,
+)
+from eval_protocol.models import EvaluationRow
+
+
+@dataclass(kw_only=True)
+class DynamicDataLoader(EvaluationDataLoader):
+    """Data loader for dynamic data generation."""
+
+    generators: Sequence[Callable[[], list[EvaluationRow]]]
+    """Dynamic data generation functions. These callables are invoked each time data
+    needs to be loaded, allowing for dynamic data generation, lazy loading, or data that
+    changes between evaluation runs. Each function should return a list of EvaluationRow
+    objects. This is useful for scenarios like generating test data on-the-fly, loading
+    data from external sources, or creating data with randomized elements for robust testing."""
+
+    def variants(self) -> Sequence[DataLoaderVariant]:
+        variants: Sequence[DataLoaderVariant] = []
+        for generator in self.generators:
+
+            def _load() -> DataLoaderResult:
+                resolved_rows = generator()
+                return DataLoaderResult(
+                    rows=resolved_rows,
+                    type=self.__class__.__name__,
+                    variant_id=generator.__name__,
+                    variant_description=generator.__doc__,
+                )
+
+            variants.append(_load)
+
+        return variants