judgeval 0.9.3__py3-none-any.whl → 0.10.0__py3-none-any.whl

judgeval/data/judgment_types.py

@@ -1,10 +1,11 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-08-29T04:49:38+00:00
+#   timestamp: 2025-09-10T17:42:11+00:00
 
 from __future__ import annotations
 from typing import Annotated, Any, Dict, List, Optional, Union
 from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, RootModel
+from enum import Enum
 
 
 class TraceAndSpanId(RootModel[List]):
@@ -17,42 +18,10 @@ class EvalResultsFetch(BaseModel):
 
 
 class DatasetFetch(BaseModel):
-    dataset_alias: Annotated[str, Field(title="Dataset Alias")]
+    dataset_name: Annotated[str, Field(title="Dataset Name")]
     project_name: Annotated[str, Field(title="Project Name")]
 
 
-class TraceSave(BaseModel):
-    project_name: Annotated[str, Field(title="Project Name")]
-    trace_id: Annotated[str, Field(title="Trace Id")]
-    name: Annotated[str, Field(title="Name")]
-    created_at: Annotated[str, Field(title="Created At")]
-    duration: Annotated[float, Field(title="Duration")]
-    offline_mode: Annotated[Optional[bool], Field(title="Offline Mode")] = False
-    has_notification: Annotated[Optional[bool], Field(title="Has Notification")] = False
-    customer_id: Annotated[Optional[str], Field(title="Customer Id")] = None
-    tags: Annotated[Optional[List[str]], Field(title="Tags")] = None
-    metadata: Annotated[Optional[Dict[str, Any]], Field(title="Metadata")] = None
-    update_id: Annotated[Optional[int], Field(title="Update Id")] = 1
-
-
-class TraceFetch(BaseModel):
-    trace_id: Annotated[str, Field(title="Trace Id")]
-
-
-class TraceAddToDataset(BaseModel):
-    trace_id: Annotated[str, Field(title="Trace Id")]
-    trace_span_id: Annotated[str, Field(title="Trace Span Id")]
-    dataset_alias: Annotated[str, Field(title="Dataset Alias")]
-    project_name: Annotated[str, Field(title="Project Name")]
-
-
-class EvaluationRunsBatchRequest(BaseModel):
-    organization_id: Annotated[str, Field(title="Organization Id")]
-    evaluation_entries: Annotated[
-        List[Dict[str, Any]], Field(title="Evaluation Entries")
-    ]
-
-
 class ProjectAdd(BaseModel):
     project_name: Annotated[str, Field(title="Project Name")]
 
@@ -82,7 +51,7 @@ class SavePromptScorerRequest(BaseModel):
     prompt: Annotated[str, Field(title="Prompt")]
     threshold: Annotated[float, Field(title="Threshold")]
     options: Annotated[Optional[Dict[str, float]], Field(title="Options")] = None
-    is_trace: Annotated[Optional[bool], Field(title="Is Trace")] = False
+    is_trace: Annotated[Optional[bool], Field(title="Is Trace")] = None
 
 
 class SavePromptScorerResponse(BaseModel):
@@ -161,8 +130,8 @@ class Example(BaseModel):
     model_config = ConfigDict(
         extra="allow",
     )
-    example_id: Annotated[str, Field(title="Example Id")]
-    created_at: Annotated[str, Field(title="Created At")]
+    example_id: Annotated[Optional[str], Field(title="Example Id")] = None
+    created_at: Annotated[Optional[str], Field(title="Created At")] = None
     name: Annotated[Optional[str], Field(title="Name")] = None
 
 
@@ -172,34 +141,9 @@ class ValidationError(BaseModel):
     type: Annotated[str, Field(title="Error Type")]
 
 
-class SpanBatchItem(BaseModel):
-    span_id: Annotated[str, Field(title="Span Id")]
-    trace_id: Annotated[str, Field(title="Trace Id")]
-    function: Annotated[str, Field(title="Function")]
-    created_at: Annotated[Any, Field(title="Created At")] = None
-    parent_span_id: Annotated[Optional[str], Field(title="Parent Span Id")] = None
-    span_type: Annotated[Optional[str], Field(title="Span Type")] = "span"
-    inputs: Annotated[Optional[Dict[str, Any]], Field(title="Inputs")] = None
-    output: Annotated[Any, Field(title="Output")] = None
-    error: Annotated[Optional[Dict[str, Any]], Field(title="Error")] = None
-    usage: Annotated[Optional[Dict[str, Any]], Field(title="Usage")] = None
-    duration: Annotated[Optional[float], Field(title="Duration")] = None
-    expected_tools: Annotated[
-        Optional[List[Dict[str, Any]]], Field(title="Expected Tools")
-    ] = None
-    additional_metadata: Annotated[
-        Optional[Dict[str, Any]], Field(title="Additional Metadata")
-    ] = None
-    has_evaluation: Annotated[Optional[bool], Field(title="Has Evaluation")] = False
-    agent_name: Annotated[Optional[str], Field(title="Agent Name")] = None
-    class_name: Annotated[Optional[str], Field(title="Class Name")] = None
-    state_before: Annotated[Optional[Dict[str, Any]], Field(title="State Before")] = (
-        None
-    )
-    state_after: Annotated[Optional[Dict[str, Any]], Field(title="State After")] = None
-    span_state: Annotated[str, Field(title="Span State")]
-    update_id: Annotated[Optional[int], Field(title="Update Id")] = 1
-    queued_at: Annotated[float, Field(title="Queued At")]
+class DatasetKind(Enum):
+    trace = "trace"
+    example = "example"
 
 
 class PromptScorer(BaseModel):
@@ -213,52 +157,55 @@ class PromptScorer(BaseModel):
 
 
 class ScorerData(BaseModel):
+    id: Annotated[Optional[str], Field(title="Id")] = None
     name: Annotated[str, Field(title="Name")]
     threshold: Annotated[float, Field(title="Threshold")]
     success: Annotated[bool, Field(title="Success")]
     score: Annotated[Optional[float], Field(title="Score")] = None
     reason: Annotated[Optional[str], Field(title="Reason")] = None
     strict_mode: Annotated[Optional[bool], Field(title="Strict Mode")] = None
-    evaluation_model: Annotated[
-        Optional[Union[List[str], str]], Field(title="Evaluation Model")
-    ] = None
+    evaluation_model: Annotated[Optional[str], Field(title="Evaluation Model")] = None
     error: Annotated[Optional[str], Field(title="Error")] = None
     additional_metadata: Annotated[
         Optional[Dict[str, Any]], Field(title="Additional Metadata")
     ] = None
 
 
-class TraceUsage(BaseModel):
-    prompt_tokens: Annotated[Optional[int], Field(title="Prompt Tokens")] = None
-    completion_tokens: Annotated[Optional[int], Field(title="Completion Tokens")] = None
-    cache_creation_input_tokens: Annotated[
-        Optional[int], Field(title="Cache Creation Input Tokens")
-    ] = None
-    cache_read_input_tokens: Annotated[
-        Optional[int], Field(title="Cache Read Input Tokens")
-    ] = None
-    total_tokens: Annotated[Optional[int], Field(title="Total Tokens")] = None
-    prompt_tokens_cost_usd: Annotated[
-        Optional[float], Field(title="Prompt Tokens Cost Usd")
+class OtelTraceSpan(BaseModel):
+    organization_id: Annotated[str, Field(title="Organization Id")]
+    project_id: Annotated[Optional[str], Field(title="Project Id")] = None
+    user_id: Annotated[str, Field(title="User Id")]
+    timestamp: Annotated[str, Field(title="Timestamp")]
+    trace_id: Annotated[str, Field(title="Trace Id")]
+    span_id: Annotated[str, Field(title="Span Id")]
+    parent_span_id: Annotated[Optional[str], Field(title="Parent Span Id")] = None
+    trace_state: Annotated[Optional[str], Field(title="Trace State")] = None
+    span_name: Annotated[Optional[str], Field(title="Span Name")] = None
+    span_kind: Annotated[Optional[str], Field(title="Span Kind")] = None
+    service_name: Annotated[Optional[str], Field(title="Service Name")] = None
+    resource_attributes: Annotated[
+        Optional[Dict[str, Any]], Field(title="Resource Attributes")
     ] = None
-    completion_tokens_cost_usd: Annotated[
-        Optional[float], Field(title="Completion Tokens Cost Usd")
+    span_attributes: Annotated[
+        Optional[Dict[str, Any]], Field(title="Span Attributes")
     ] = None
-    total_cost_usd: Annotated[Optional[float], Field(title="Total Cost Usd")] = None
-    model_name: Annotated[Optional[str], Field(title="Model Name")] = None
-
-
-class Tool(BaseModel):
-    tool_name: Annotated[str, Field(title="Tool Name")]
-    parameters: Annotated[Optional[Dict[str, Any]], Field(title="Parameters")] = None
-    agent_name: Annotated[Optional[str], Field(title="Agent Name")] = None
-    result_dependencies: Annotated[
-        Optional[List[Dict[str, Any]]], Field(title="Result Dependencies")
-    ] = None
-    action_dependencies: Annotated[
-        Optional[List[Dict[str, Any]]], Field(title="Action Dependencies")
+    duration: Annotated[Optional[int], Field(title="Duration")] = None
+    status_code: Annotated[Optional[str], Field(title="Status Code")] = None
+    status_message: Annotated[Optional[str], Field(title="Status Message")] = None
+    events: Annotated[Optional[List[Dict[str, Any]]], Field(title="Events")] = None
+    links: Annotated[Optional[List[Dict[str, Any]]], Field(title="Links")] = None
+    legacy_span_id: Annotated[Optional[str], Field(title="Legacy Span Id")] = None
+    inputs: Annotated[Optional[Dict[str, Any]], Field(title="Inputs")] = None
+    output: Annotated[Any, Field(title="Output")]
+    error: Annotated[Optional[Dict[str, Any]], Field(title="Error")] = None
+    agent_id: Annotated[Optional[str], Field(title="Agent Id")] = None
+    cumulative_llm_cost: Annotated[
+        Optional[float], Field(title="Cumulative Llm Cost")
     ] = None
-    require_all: Annotated[Optional[bool], Field(title="Require All")] = None
+    state_after: Annotated[Optional[Dict[str, Any]], Field(title="State After")] = None
+    state_before: Annotated[Optional[Dict[str, Any]], Field(title="State Before")] = (
+        None
+    )
 
 
 class ExampleEvaluationRun(BaseModel):
@@ -301,61 +248,27 @@ class TraceEvaluationRun(BaseModel):
 
 
 class DatasetInsertExamples(BaseModel):
-    dataset_alias: Annotated[str, Field(title="Dataset Alias")]
+    dataset_name: Annotated[str, Field(title="Dataset Name")]
     examples: Annotated[List[Example], Field(title="Examples")]
     project_name: Annotated[str, Field(title="Project Name")]
 
 
-class SpansBatchRequest(BaseModel):
-    spans: Annotated[List[SpanBatchItem], Field(title="Spans")]
-    organization_id: Annotated[str, Field(title="Organization Id")]
-
-
-class FetchPromptScorerResponse(BaseModel):
-    scorer: PromptScorer
+class DatasetReturn(BaseModel):
+    name: Annotated[str, Field(title="Name")]
+    project_name: Annotated[str, Field(title="Project Name")]
+    examples: Annotated[Optional[List[Example]], Field(title="Examples")] = None
 
 
-class TraceSpan(BaseModel):
-    span_id: Annotated[str, Field(title="Span Id")]
-    trace_id: Annotated[str, Field(title="Trace Id")]
-    function: Annotated[str, Field(title="Function")]
-    created_at: Annotated[Any, Field(title="Created At")] = None
-    parent_span_id: Annotated[Optional[str], Field(title="Parent Span Id")] = None
-    span_type: Annotated[Optional[str], Field(title="Span Type")] = "span"
-    inputs: Annotated[Optional[Dict[str, Any]], Field(title="Inputs")] = None
-    error: Annotated[Optional[Dict[str, Any]], Field(title="Error")] = None
-    output: Annotated[Any, Field(title="Output")] = None
-    usage: Optional[TraceUsage] = None
-    duration: Annotated[Optional[float], Field(title="Duration")] = None
-    expected_tools: Annotated[Optional[List[Tool]], Field(title="Expected Tools")] = (
-        None
-    )
-    additional_metadata: Annotated[
-        Optional[Dict[str, Any]], Field(title="Additional Metadata")
-    ] = None
-    has_evaluation: Annotated[Optional[bool], Field(title="Has Evaluation")] = False
-    agent_name: Annotated[Optional[str], Field(title="Agent Name")] = None
-    class_name: Annotated[Optional[str], Field(title="Class Name")] = None
-    state_before: Annotated[Optional[Dict[str, Any]], Field(title="State Before")] = (
-        None
-    )
-    state_after: Annotated[Optional[Dict[str, Any]], Field(title="State After")] = None
-    update_id: Annotated[Optional[int], Field(title="Update Id")] = 1
+class DatasetCreate(BaseModel):
+    name: Annotated[str, Field(title="Name")]
+    dataset_kind: DatasetKind
+    project_name: Annotated[str, Field(title="Project Name")]
+    examples: Annotated[Optional[List[Example]], Field(title="Examples")] = None
+    overwrite: Annotated[Optional[bool], Field(title="Overwrite")] = False
 
 
-class Trace(BaseModel):
-    trace_id: Annotated[str, Field(title="Trace Id")]
-    name: Annotated[str, Field(title="Name")]
-    created_at: Annotated[str, Field(title="Created At")]
-    duration: Annotated[float, Field(title="Duration")]
-    trace_spans: Annotated[List[TraceSpan], Field(title="Trace Spans")]
-    offline_mode: Annotated[Optional[bool], Field(title="Offline Mode")] = False
-    rules: Annotated[Optional[Dict[str, Any]], Field(title="Rules")] = {}
-    has_notification: Annotated[Optional[bool], Field(title="Has Notification")] = False
-    customer_id: Annotated[Optional[str], Field(title="Customer Id")] = None
-    tags: Annotated[Optional[List[str]], Field(title="Tags")] = []
-    metadata: Annotated[Optional[Dict[str, Any]], Field(title="Metadata")] = {}
-    update_id: Annotated[Optional[int], Field(title="Update Id")] = 1
+class FetchPromptScorerResponse(BaseModel):
+    scorer: PromptScorer
 
 
 class ScoringResult(BaseModel):
@@ -365,34 +278,13 @@ class ScoringResult(BaseModel):
     )
     name: Annotated[Optional[str], Field(title="Name")] = None
     data_object: Annotated[
-        Optional[Union[TraceSpan, Example]], Field(title="Data Object")
+        Optional[Union[OtelTraceSpan, Example]], Field(title="Data Object")
     ] = None
     trace_id: Annotated[Optional[str], Field(title="Trace Id")] = None
     run_duration: Annotated[Optional[float], Field(title="Run Duration")] = None
     evaluation_cost: Annotated[Optional[float], Field(title="Evaluation Cost")] = None
 
 
-class TraceRun(BaseModel):
-    project_name: Annotated[Optional[str], Field(title="Project Name")] = None
-    eval_name: Annotated[Optional[str], Field(title="Eval Name")] = None
-    traces: Annotated[List[Trace], Field(title="Traces")]
-    scorers: Annotated[List[ScorerConfig], Field(title="Scorers")]
-    model: Annotated[str, Field(title="Model")]
-    trace_span_id: Annotated[Optional[str], Field(title="Trace Span Id")] = None
-    tools: Annotated[Optional[List[Dict[str, Any]]], Field(title="Tools")] = None
-
-
 class EvalResults(BaseModel):
     results: Annotated[List[ScoringResult], Field(title="Results")]
     run: Annotated[Union[ExampleEvaluationRun, TraceEvaluationRun], Field(title="Run")]
-
-
-class DatasetPush(BaseModel):
-    dataset_alias: Annotated[str, Field(title="Dataset Alias")]
-    comments: Annotated[Optional[str], Field(title="Comments")] = None
-    source_file: Annotated[Optional[str], Field(title="Source File")] = None
-    examples: Annotated[Optional[List[Example]], Field(title="Examples")] = None
-    traces: Annotated[Optional[List[Trace]], Field(title="Traces")] = None
-    is_trace: Annotated[Optional[bool], Field(title="Is Trace")] = False
-    project_name: Annotated[str, Field(title="Project Name")]
-    overwrite: Annotated[Optional[bool], Field(title="Overwrite")] = False

judgeval/data/result.py

@@ -1,6 +1,5 @@
 from typing import List, Union
 from judgeval.data import ScorerData, Example
-from judgeval.data.trace import TraceSpan
 from judgeval.data.judgment_types import ScoringResult as JudgmentScoringResult
 
 
@@ -34,7 +33,7 @@ class ScoringResult(JudgmentScoringResult):
 
 
 def generate_scoring_result(
-    data_object: Union[Example, TraceSpan],
+    data_object: Union[Example],
     scorers_data: List[ScorerData],
     run_duration: float,
     success: bool,

judgeval/data/trace.py

@@ -1,40 +1,14 @@
-from datetime import datetime, timezone
-from judgeval.data.judgment_types import (
-    TraceUsage as JudgmentTraceUsage,
-    TraceSpan as JudgmentTraceSpan,
-    Trace as JudgmentTrace,
-)
-from judgeval.utils.serialize import json_encoder
-
-
-class TraceUsage(JudgmentTraceUsage):
-    pass
-
-
-class TraceSpan(JudgmentTraceSpan):
-    def model_dump(self, **kwargs):
-        return {
-            "span_id": self.span_id,
-            "trace_id": self.trace_id,
-            "created_at": datetime.fromtimestamp(
-                self.created_at, tz=timezone.utc
-            ).isoformat(),
-            "inputs": json_encoder(self.inputs),
-            "output": json_encoder(self.output),
-            "error": json_encoder(self.error),
-            "parent_span_id": self.parent_span_id,
-            "function": self.function,
-            "duration": self.duration,
-            "span_type": self.span_type,
-            "usage": self.usage.model_dump() if self.usage else None,
-            "has_evaluation": self.has_evaluation,
-            "agent_name": self.agent_name,
-            "state_before": self.state_before,
-            "state_after": self.state_after,
-            "additional_metadata": json_encoder(self.additional_metadata),
-            "update_id": self.update_id,
-        }
-
-
-class Trace(JudgmentTrace):
-    pass
+from typing import Optional
+from pydantic import BaseModel
+
+
+class TraceUsage(BaseModel):
+    prompt_tokens: Optional[int] = None
+    completion_tokens: Optional[int] = None
+    cache_creation_input_tokens: Optional[int] = None
+    cache_read_input_tokens: Optional[int] = None
+    total_tokens: Optional[int] = None
+    prompt_tokens_cost_usd: Optional[float] = None
+    completion_tokens_cost_usd: Optional[float] = None
+    total_cost_usd: Optional[float] = None
+    model_name: Optional[str] = None

judgeval/dataset/__init__.py

@@ -5,7 +5,7 @@ import yaml
 from dataclasses import dataclass
 from typing import List, Literal, Optional
 
-from judgeval.data import Example, Trace
+from judgeval.data import Example
 from judgeval.utils.file_utils import get_examples_from_yaml, get_examples_from_json
 from judgeval.api import JudgmentSyncClient
 from judgeval.logger import judgeval_logger
@@ -15,7 +15,6 @@ from judgeval.env import JUDGMENT_API_KEY, JUDGMENT_ORG_ID
 @dataclass
 class Dataset:
     examples: List[Example]
-    traces: List[Trace]
     name: str
     project_name: str
     judgment_api_key: str = JUDGMENT_API_KEY or ""
@@ -30,7 +29,7 @@ class Dataset:
         client = JudgmentSyncClient(cls.judgment_api_key, cls.organization_id)
         dataset = client.datasets_pull_for_judgeval(
             {
-                "dataset_alias": name,
+                "dataset_name": name,
                 "project_name": project_name,
             },
         )
@@ -40,12 +39,14 @@ class Dataset:
         for e in examples:
             if isinstance(e, dict) and isinstance(e.get("data"), dict):
                 e.update(e.pop("data"))
+                e.pop(
+                    "example_id"
+                )  # TODO: remove once scorer data migraiton is complete
         judgeval_logger.info(f"Succesfully retrieved dataset {name}!")
         return cls(
             name=name,
             project_name=project_name,
             examples=[Example(**e) for e in examples],
-            traces=[Trace(**t) for t in dataset.get("traces", [])],
         )
 
     @classmethod
@@ -54,25 +55,18 @@ class Dataset:
         name: str,
         project_name: str,
         examples: Optional[List[Example]] = None,
-        traces: Optional[List[Trace]] = None,
         overwrite: bool = False,
     ):
-        if examples and traces:
-            raise ValueError("Only one of examples or traces must be provided")
-
         if not examples:
             examples = []
 
-        if not traces:
-            traces = []
-
         client = JudgmentSyncClient(cls.judgment_api_key, cls.organization_id)
-        client.datasets_push(
+        client.datasets_create_for_judgeval(
             {
-                "dataset_alias": name,
+                "name": name,
                 "project_name": project_name,
-                "examples": [e.model_dump() for e in examples],  # type: ignore
-                "traces": [t.model_dump() for t in traces],  # type: ignore
+                "examples": [e.model_dump() for e in examples],
+                "dataset_kind": "example",
                 "overwrite": overwrite,
             }
         )
@@ -82,7 +76,6 @@ class Dataset:
             name=name,
             project_name=project_name,
             examples=examples,
-            traces=traces,
         )
 
     def add_from_json(self, file_path: str) -> None:
@@ -123,29 +116,15 @@ class Dataset:
         self.add_examples(examples)
 
     def add_examples(self, examples: List[Example]) -> None:
-        client = JudgmentSyncClient(self.judgment_api_key, self.organization_id)
-        client.datasets_insert_examples(
-            {
-                "dataset_alias": self.name,
-                "project_name": self.project_name,
-                "examples": [
-                    {
-                        "name": e.name,
-                        "created_at": e.created_at,
-                        "example_id": e.example_id,
-                    }
-                    for e in examples
-                ],
-            }
-        )
+        if not isinstance(examples, list):
+            raise TypeError("examples must be a list")
 
-    def add_traces(self, traces: List[Trace]) -> None:
         client = JudgmentSyncClient(self.judgment_api_key, self.organization_id)
-        client.traces_add_to_dataset(
+        client.datasets_insert_examples_for_judgeval(
             {
-                "dataset_alias": self.name,
+                "dataset_name": self.name,
                 "project_name": self.project_name,
-                "traces": [t.model_dump() for t in traces],  # type: ignore
+                "examples": [e.model_dump() for e in examples],
             }
         )
 
@@ -200,10 +179,4 @@ class Dataset:
         return len(self.examples)
 
     def __str__(self):
-        return (
-            f"{self.__class__.__name__}("
-            f"examples={self.examples}, "
-            f"traces={self.traces}, "
-            f"name={self.name}"
-            f")"
-        )
+        return f"{self.__class__.__name__}(examples={self.examples}, name={self.name})"

judgeval/evaluation/__init__.py

@@ -10,7 +10,7 @@ from typing import List, Dict, Union, Tuple, TYPE_CHECKING
 from rich import print as rprint
 
 from judgeval.data import ScorerData, ScoringResult, Example
-from judgeval.scorers import BaseScorer, APIScorerConfig
+from judgeval.scorers import BaseScorer, ExampleAPIScorerConfig
 from judgeval.scorers.score import a_execute_scoring
 from judgeval.api import JudgmentSyncClient
 from judgeval.env import (
@@ -86,7 +86,7 @@ def log_evaluation_results(
 
 
 def check_examples(
-    examples: List[Example], scorers: List[Union[APIScorerConfig, BaseScorer]]
+    examples: List[Example], scorers: List[Union[ExampleAPIScorerConfig, BaseScorer]]
 ) -> None:
     """
     Checks if the example contains the necessary parameters for the scorer.
@@ -118,10 +118,8 @@ def check_examples(
 
 
 def _poll_evaluation_until_complete(
-    experiment_run_id: str,
-    project_name: str,
+    evaluation_run: EvaluationRun,
     judgment_api_key: str,
-    organization_id: str,
     expected_scorer_data_count: int,
     poll_interval_seconds: float = 5,
     max_failures: int = 5,
@@ -142,6 +140,10 @@ def _poll_evaluation_until_complete(
     Returns:
         List[ScoringResult]: The evaluation results
     """
+    organization_id = evaluation_run.organization_id
+    project_name = evaluation_run.project_name
+    experiment_run_id = evaluation_run.id
+
     poll_count = 0
     exception_count = 0
     api_client = JudgmentSyncClient(judgment_api_key, organization_id)
@@ -157,6 +159,11 @@ def _poll_evaluation_until_complete(
                 time.sleep(poll_interval_seconds)
                 continue
 
+            example_scorer_pairings = status_response.get("results", [])
+            if len(example_scorer_pairings) != expected_scorer_data_count:
+                time.sleep(poll_interval_seconds)
+                continue
+
             results_response = api_client.fetch_experiment_run(
                 {
                     "experiment_run_id": experiment_run_id,
@@ -165,36 +172,20 @@ def _poll_evaluation_until_complete(
             )
             url = results_response.get("ui_results_url")
 
-            if results_response.get("examples") is None:
-                time.sleep(poll_interval_seconds)
-                continue
-
-            examples_data = results_response.get("examples", [])
-            scoring_results = []
-            scorer_data_count = 0
-
-            for example_data in examples_data:
-                scorer_data_list = []
-                for raw_scorer_data in example_data.get("scorer_data", []):
-                    scorer_data = ScorerData(**raw_scorer_data)
-                    scorer_data_list.append(scorer_data)
-                    scorer_data_count += 1
-
-                example = Example(**example_data)
-
-                success = all(scorer_data.success for scorer_data in scorer_data_list)
+            scoring_result_list = []
+            for res in results_response.get("results", []):
+                example = res.get("data", {}).copy()
+                example["example_id"] = res.get("example_id")
                 scoring_result = ScoringResult(
-                    success=success,
-                    scorers_data=scorer_data_list,
+                    scorers_data=res.get("scorers", []),
+                    success=all(
+                        t.get("success", False) for t in res.get("scorers", [])
+                    ),
                     data_object=example,
                 )
-                scoring_results.append(scoring_result)
-
-            if scorer_data_count != expected_scorer_data_count:
-                time.sleep(poll_interval_seconds)
-                continue
+                scoring_result_list.append(scoring_result)
 
-            return scoring_results, url
+            return scoring_result_list, url
         except Exception as e:
             exception_count += 1
             if isinstance(e, JudgmentAPIError):
@@ -294,10 +285,8 @@ def run_eval(
                 else sum(1 for cs in evaluation_run.custom_scorers if cs.server_hosted)
             )
             results, url = _poll_evaluation_until_complete(
-                experiment_run_id=evaluation_run.id,
-                project_name=evaluation_run.project_name,
+                evaluation_run=evaluation_run,
                 judgment_api_key=judgment_api_key,
-                organization_id=evaluation_run.organization_id,
                 expected_scorer_data_count=(num_scorers * len(evaluation_run.examples)),
             )
         finally:

judgeval/scorers/__init__.py

@@ -1,25 +1,27 @@
-from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.scorers.api_scorer import (
+    APIScorerConfig,
+    ExampleAPIScorerConfig,
+    TraceAPIScorerConfig,
+)
 from judgeval.scorers.base_scorer import BaseScorer
 from judgeval.scorers.judgeval_scorers.api_scorers import (
     FaithfulnessScorer,
     AnswerRelevancyScorer,
     AnswerCorrectnessScorer,
     InstructionAdherenceScorer,
-    DerailmentScorer,
-    ToolOrderScorer,
+    TracePromptScorer,
     PromptScorer,
-    ToolDependencyScorer,
 )
 
 __all__ = [
     "APIScorerConfig",
+    "ExampleAPIScorerConfig",
+    "TraceAPIScorerConfig",
     "BaseScorer",
+    "TracePromptScorer",
     "PromptScorer",
     "FaithfulnessScorer",
     "AnswerRelevancyScorer",
     "AnswerCorrectnessScorer",
     "InstructionAdherenceScorer",
-    "DerailmentScorer",
-    "ToolOrderScorer",
-    "ToolDependencyScorer",
 ]

judgeval/scorers/api_scorer.py

@@ -63,3 +63,11 @@ class APIScorerConfig(BaseModel):
 
     def __str__(self):
         return f"JudgmentScorer(score_type={self.score_type.value}, threshold={self.threshold})"
+
+
+class ExampleAPIScorerConfig(APIScorerConfig):
+    pass
+
+
+class TraceAPIScorerConfig(APIScorerConfig):
+    pass