judgeval 0.0.52__py3-none-any.whl → 0.0.54__py3-none-any.whl

judgeval/data/example.py

@@ -2,15 +2,12 @@
 Classes for representing examples in a dataset.
 """
 
-from typing import Optional, Any, Dict, List, Union
-from uuid import uuid4
-from pydantic import BaseModel, Field, field_validator
 from enum import Enum
 from datetime import datetime
-from judgeval.data.tool import Tool
+from judgeval.data.judgment_types import ExampleJudgmentType
 
 
-class ExampleParams(Enum):
+class ExampleParams(str, Enum):
     INPUT = "input"
     ACTUAL_OUTPUT = "actual_output"
     EXPECTED_OUTPUT = "expected_output"
@@ -22,144 +19,14 @@ class ExampleParams(Enum):
     ADDITIONAL_METADATA = "additional_metadata"
 
 
-class Example(BaseModel):
-    input: Optional[Union[str, Dict[str, Any]]] = None
-    actual_output: Optional[Union[str, List[str]]] = None
-    expected_output: Optional[Union[str, List[str]]] = None
-    context: Optional[List[str]] = None
-    retrieval_context: Optional[List[str]] = None
-    additional_metadata: Optional[Dict[str, Any]] = None
-    tools_called: Optional[List[str]] = None
-    expected_tools: Optional[List[Tool]] = None
-    name: Optional[str] = None
-    example_id: str = Field(default_factory=lambda: str(uuid4()))
-    example_index: Optional[int] = None
-    created_at: Optional[str] = None
-    trace_id: Optional[str] = None
+class Example(ExampleJudgmentType):
+    example_id: str = ""
 
     def __init__(self, **data):
-        if "example_id" not in data:
-            data["example_id"] = str(uuid4())
-        # Set timestamp if not provided
         if "created_at" not in data:
             data["created_at"] = datetime.now().isoformat()
         super().__init__(**data)
-
-    @field_validator("input", mode="before")
-    @classmethod
-    def validate_input(cls, v):
-        if v is not None:
-            if not isinstance(v, (str, dict)):
-                raise ValueError(
-                    f"Input must be a string or dictionary but got {v} of type {type(v)}"
-                )
-
-            # If it's a string, check that it's not empty
-            if isinstance(v, str) and not v:
-                raise ValueError(f"Input string must be non-empty but got '{v}'")
-
-            # If it's a dictionary, check that it's not empty
-            if isinstance(v, dict) and not v:
-                raise ValueError(f"Input dictionary must be non-empty but got {v}")
-
-        return v
-
-    @field_validator("actual_output", mode="before")
-    @classmethod
-    def validate_actual_output(cls, v):
-        if v is not None:
-            if not isinstance(v, (str, list)):
-                raise ValueError(
-                    f"Actual output must be a string or a list of strings but got {v} of type {type(v)}"
-                )
-            if isinstance(v, list) and not all(isinstance(item, str) for item in v):
-                raise ValueError(
-                    f"All items in actual_output must be strings but got {v}"
-                )
-        return v
-
-    @field_validator("expected_output", mode="before")
-    @classmethod
-    def validate_expected_output(cls, v):
-        if v is not None and not isinstance(v, (str, list)):
-            raise ValueError(
-                f"Expected output must be a string, a list of strings, or None but got {v} of type {type(v)}"
-            )
-        if isinstance(v, list) and not all(isinstance(item, str) for item in v):
-            raise ValueError(
-                f"All items in expected_output must be strings but got {v}"
-            )
-        return v
-
-    @field_validator("expected_tools")
-    @classmethod
-    def validate_expected_tools(cls, v):
-        if v is not None:
-            if not isinstance(v, list):
-                raise ValueError(
-                    f"Expected tools must be a list of Tools or None but got {v} of type {type(v)}"
-                )
-
-            # Check that each item in the list is a Tool
-            for i, item in enumerate(v):
-                if not isinstance(item, Tool):
-                    raise ValueError(
-                        f"Expected tools must be a list of Tools, but item at index {i} is {item} of type {type(item)}"
-                    )
-
-        return v
-
-    @field_validator("context", "retrieval_context", "tools_called", mode="before")
-    @classmethod
-    def validate_string_lists(cls, v, info):
-        field_name = info.field_name
-        if v is not None:
-            if not isinstance(v, list):
-                raise ValueError(
-                    f"{field_name} must be a list of strings or None but got {v} of type {type(v)}"
-                )
-            for i, item in enumerate(v):
-                if not isinstance(item, str):
-                    raise ValueError(
-                        f"All items in {field_name} must be strings but item at index {i} is {item} of type {type(item)}"
-                    )
-        return v
-
-    @field_validator("additional_metadata", mode="before")
-    @classmethod
-    def validate_additional_metadata(cls, v):
-        if v is not None and not isinstance(v, dict):
-            raise ValueError(
-                f"Additional metadata must be a dictionary or None but got {v} of type {type(v)}"
-            )
-        return v
-
-    @field_validator("example_index", mode="before")
-    @classmethod
-    def validate_example_index(cls, v):
-        if v is not None and not isinstance(v, int):
-            raise ValueError(
-                f"Example index must be an integer or None but got {v} of type {type(v)}"
-            )
-        return v
-
-    @field_validator("created_at", mode="before")
-    @classmethod
-    def validate_created_at(cls, v):
-        if v is not None and not isinstance(v, str):
-            raise ValueError(
-                f"Timestamp must be a string or None but got {v} of type {type(v)}"
-            )
-        return v
-
-    @field_validator("trace_id", mode="before")
-    @classmethod
-    def validate_trace_id(cls, v):
-        if v is not None and not isinstance(v, str):
-            raise ValueError(
-                f"Trace ID must be a string or None but got {v} of type {type(v)}"
-            )
-        return v
+        self.example_id = None
 
     def to_dict(self):
         return {

judgeval/data/judgment_types.py

@@ -0,0 +1,214 @@
+# generated by datamodel-codegen:
+#   filename:  openapi_new.json
+#   timestamp: 2025-07-12T17:11:33+00:00
+
+from __future__ import annotations
+
+from typing import Annotated, Any, Dict, List, Optional, Union
+
+from pydantic import BaseModel, Field
+
+
+class ValidationErrorJudgmentType(BaseModel):
+    loc: Annotated[List[Union[str, int]], Field(title="Location")]
+    msg: Annotated[str, Field(title="Message")]
+    type: Annotated[str, Field(title="Error Type")]
+
+
+class ScorerDataJudgmentType(BaseModel):
+    name: Annotated[str, Field(title="Name")]
+    threshold: Annotated[float, Field(title="Threshold")]
+    success: Annotated[bool, Field(title="Success")]
+    score: Annotated[Optional[float], Field(title="Score")] = None
+    reason: Annotated[Optional[str], Field(title="Reason")] = None
+    strict_mode: Annotated[Optional[bool], Field(title="Strict Mode")] = None
+    evaluation_model: Annotated[
+        Optional[Union[List[str], str]], Field(title="Evaluation Model")
+    ] = None
+    error: Annotated[Optional[str], Field(title="Error")] = None
+    additional_metadata: Annotated[
+        Optional[Dict[str, Any]], Field(title="Additional Metadata")
+    ] = None
+
+
+class ScorerConfigJudgmentType(BaseModel):
+    score_type: Annotated[str, Field(title="Score Type")]
+    name: Annotated[Optional[str], Field(title="Name")] = None
+    threshold: Annotated[Optional[float], Field(title="Threshold")] = 0.5
+    strict_mode: Annotated[Optional[bool], Field(title="Strict Mode")] = False
+    required_params: Annotated[Optional[List[str]], Field(title="Required Params")] = (
+        Field(default_factory=list)
+    )
+    kwargs: Annotated[Optional[Dict[str, Any]], Field(title="Kwargs")] = None
+
+
+class TraceUsageJudgmentType(BaseModel):
+    prompt_tokens: Annotated[Optional[int], Field(title="Prompt Tokens")] = None
+    completion_tokens: Annotated[Optional[int], Field(title="Completion Tokens")] = None
+    cache_creation_input_tokens: Annotated[
+        Optional[int], Field(title="Cache Creation Input Tokens")
+    ] = None
+    cache_read_input_tokens: Annotated[
+        Optional[int], Field(title="Cache Read Input Tokens")
+    ] = None
+    total_tokens: Annotated[Optional[int], Field(title="Total Tokens")] = None
+    prompt_tokens_cost_usd: Annotated[
+        Optional[float], Field(title="Prompt Tokens Cost Usd")
+    ] = None
+    completion_tokens_cost_usd: Annotated[
+        Optional[float], Field(title="Completion Tokens Cost Usd")
+    ] = None
+    total_cost_usd: Annotated[Optional[float], Field(title="Total Cost Usd")] = None
+    model_name: Annotated[Optional[str], Field(title="Model Name")] = None
+
+
+class ToolJudgmentType(BaseModel):
+    tool_name: Annotated[str, Field(title="Tool Name")]
+    parameters: Annotated[Optional[Dict[str, Any]], Field(title="Parameters")] = None
+    agent_name: Annotated[Optional[str], Field(title="Agent Name")] = None
+    result_dependencies: Annotated[
+        Optional[List[Dict[str, Any]]], Field(title="Result Dependencies")
+    ] = None
+    action_dependencies: Annotated[
+        Optional[List[Dict[str, Any]]], Field(title="Action Dependencies")
+    ] = None
+    require_all: Annotated[Optional[bool], Field(title="Require All")] = None
+
+
+class HTTPValidationErrorJudgmentType(BaseModel):
+    detail: Annotated[
+        Optional[List[ValidationErrorJudgmentType]], Field(title="Detail")
+    ] = None
+
+
+class TraceSpanJudgmentType(BaseModel):
+    span_id: Annotated[str, Field(title="Span Id")]
+    trace_id: Annotated[str, Field(title="Trace Id")]
+    function: Annotated[str, Field(title="Function")]
+    depth: Annotated[int, Field(title="Depth")]
+    created_at: Annotated[Any, Field(title="Created At")] = None
+    parent_span_id: Annotated[Optional[str], Field(title="Parent Span Id")] = None
+    span_type: Annotated[Optional[str], Field(title="Span Type")] = "span"
+    inputs: Annotated[Optional[Dict[str, Any]], Field(title="Inputs")] = None
+    error: Annotated[Optional[Dict[str, Any]], Field(title="Error")] = None
+    output: Annotated[Any, Field(title="Output")] = None
+    usage: Optional[TraceUsageJudgmentType] = None
+    duration: Annotated[Optional[float], Field(title="Duration")] = None
+    annotation: Annotated[Optional[List[Dict[str, Any]]], Field(title="Annotation")] = (
+        None
+    )
+    expected_tools: Annotated[
+        Optional[List[ToolJudgmentType]], Field(title="Expected Tools")
+    ] = None
+    additional_metadata: Annotated[
+        Optional[Dict[str, Any]], Field(title="Additional Metadata")
+    ] = None
+    has_evaluation: Annotated[Optional[bool], Field(title="Has Evaluation")] = False
+    agent_name: Annotated[Optional[str], Field(title="Agent Name")] = None
+    state_before: Annotated[Optional[Dict[str, Any]], Field(title="State Before")] = (
+        None
+    )
+    state_after: Annotated[Optional[Dict[str, Any]], Field(title="State After")] = None
+    update_id: Annotated[Optional[int], Field(title="Update Id")] = 1
+
+
+class ExampleJudgmentType(BaseModel):
+    input: Annotated[Optional[Union[str, Dict[str, Any]]], Field(title="Input")] = None
+    actual_output: Annotated[
+        Optional[Union[str, List[str]]], Field(title="Actual Output")
+    ] = None
+    expected_output: Annotated[
+        Optional[Union[str, List[str]]], Field(title="Expected Output")
+    ] = None
+    context: Annotated[Optional[List[str]], Field(title="Context")] = None
+    retrieval_context: Annotated[
+        Optional[List[str]], Field(title="Retrieval Context")
+    ] = None
+    additional_metadata: Annotated[
+        Optional[Dict[str, Any]], Field(title="Additional Metadata")
+    ] = None
+    tools_called: Annotated[Optional[List[str]], Field(title="Tools Called")] = Field(
+        default_factory=list
+    )
+    expected_tools: Annotated[
+        Optional[List[ToolJudgmentType]], Field(title="Expected Tools")
+    ] = Field(default_factory=list)
+    name: Annotated[Optional[str], Field(title="Name")] = None
+    example_id: Annotated[str, Field(title="Example Id")]
+    example_index: Annotated[Optional[int], Field(title="Example Index")] = None
+    created_at: Annotated[Optional[str], Field(title="Created At")] = None
+    trace_id: Annotated[Optional[str], Field(title="Trace Id")] = None
+    trace_span_id: Annotated[Optional[str], Field(title="Trace Span Id")] = None
+    dataset_id: Annotated[Optional[str], Field(title="Dataset Id")] = None
+
+
+class TraceJudgmentType(BaseModel):
+    trace_id: Annotated[str, Field(title="Trace Id")]
+    name: Annotated[str, Field(title="Name")]
+    created_at: Annotated[str, Field(title="Created At")]
+    duration: Annotated[float, Field(title="Duration")]
+    trace_spans: Annotated[List[TraceSpanJudgmentType], Field(title="Trace Spans")]
+    overwrite: Annotated[Optional[bool], Field(title="Overwrite")] = False
+    offline_mode: Annotated[Optional[bool], Field(title="Offline Mode")] = False
+    rules: Annotated[Optional[Dict[str, Any]], Field(title="Rules")] = Field(
+        default_factory=dict
+    )
+    has_notification: Annotated[Optional[bool], Field(title="Has Notification")] = False
+    customer_id: Annotated[Optional[str], Field(title="Customer Id")] = None
+    tags: Annotated[Optional[List[str]], Field(title="Tags")] = Field(
+        default_factory=list
+    )
+    metadata: Annotated[Optional[Dict[str, Any]], Field(title="Metadata")] = Field(
+        default_factory=dict
+    )
+    update_id: Annotated[Optional[int], Field(title="Update Id")] = 1
+
+
+class ScoringResultJudgmentType(BaseModel):
+    success: Annotated[bool, Field(title="Success")]
+    scorers_data: Annotated[
+        Optional[List[ScorerDataJudgmentType]], Field(title="Scorers Data")
+    ]
+    name: Annotated[Optional[str], Field(title="Name")] = None
+    data_object: Annotated[
+        Optional[Union[TraceSpanJudgmentType, ExampleJudgmentType]],
+        Field(title="Data Object"),
+    ] = None
+    trace_id: Annotated[Optional[str], Field(title="Trace Id")] = None
+    run_duration: Annotated[Optional[float], Field(title="Run Duration")] = None
+
+
+class TraceRunJudgmentType(BaseModel):
+    project_name: Annotated[Optional[str], Field(title="Project Name")] = None
+    eval_name: Annotated[Optional[str], Field(title="Eval Name")] = None
+    traces: Annotated[List[TraceJudgmentType], Field(title="Traces")]
+    scorers: Annotated[List[ScorerConfigJudgmentType], Field(title="Scorers")]
+    model: Annotated[str, Field(title="Model")]
+    judgment_api_key: Annotated[Optional[str], Field(title="Judgment Api Key")] = None
+    append: Annotated[Optional[bool], Field(title="Append")] = False
+    override_existing_eval_run_name: Annotated[
+        Optional[bool], Field(title="Override Existing Eval Run Name")
+    ] = False
+    trace_span_id: Annotated[Optional[str], Field(title="Trace Span Id")] = None
+    tools: Annotated[Optional[List[Dict[str, Any]]], Field(title="Tools")] = None
+
+
+class JudgmentEvalJudgmentType(BaseModel):
+    project_name: Annotated[Optional[str], Field(title="Project Name")] = None
+    eval_name: Annotated[Optional[str], Field(title="Eval Name")] = None
+    examples: Annotated[List[ExampleJudgmentType], Field(title="Examples")]
+    scorers: Annotated[List[ScorerConfigJudgmentType], Field(title="Scorers")]
+    model: Annotated[str, Field(title="Model")]
+    judgment_api_key: Annotated[Optional[str], Field(title="Judgment Api Key")] = None
+    append: Annotated[Optional[bool], Field(title="Append")] = False
+    override_existing_eval_run_name: Annotated[
+        Optional[bool], Field(title="Override Existing Eval Run Name")
+    ] = False
+    trace_span_id: Annotated[Optional[str], Field(title="Trace Span Id")] = None
+
+
+class EvalResultsJudgmentType(BaseModel):
+    results: Annotated[List[ScoringResultJudgmentType], Field(title="Results")]
+    run: Annotated[
+        Union[TraceRunJudgmentType, JudgmentEvalJudgmentType], Field(title="Run")
+    ]

judgeval/data/result.py

@@ -1,11 +1,10 @@
-from typing import List, Optional, Union
-from judgeval.common.logger import debug
-from pydantic import BaseModel
-from judgeval.data import ScorerData, Example, CustomExample
+from typing import List, Union
+from judgeval.data import ScorerData, Example
 from judgeval.data.trace import TraceSpan
+from judgeval.data.judgment_types import ScoringResultJudgmentType
 
 
-class ScoringResult(BaseModel):
+class ScoringResult(ScoringResultJudgmentType):
     """
     A ScoringResult contains the output of one or more scorers applied to a single example.
     Ie: One input, one actual_output, one expected_output, etc..., and 1+ scorer (Faithfulness, Hallucination, Summarization, etc...)
@@ -14,23 +13,10 @@ class ScoringResult(BaseModel):
         success (bool): Whether the evaluation was successful.
                         This means that all scorers applied to this example returned a success.
         scorer_data (List[ScorerData]): The scorers data for the evaluated example
-        data_object (Optional[Example]): The original example object that was used to create the ScoringResult, can be Example, CustomExample (future), WorkflowRun (future)
+        data_object (Optional[Example]): The original example object that was used to create the ScoringResult, can be Example, WorkflowRun (future)
 
     """
 
-    # Fields for scoring outputs
-    success: bool  # used for unit testing
-    scorers_data: Union[List[ScorerData], None]
-    name: Optional[str] = None
-
-    # The original example object that was used to create the ScoringResult
-    data_object: Optional[Union[TraceSpan, CustomExample, Example]] = None
-    trace_id: Optional[str] = None
-
-    # Additional fields for internal use
-    run_duration: Optional[float] = None
-    evaluation_cost: Optional[float] = None
-
     def to_dict(self) -> dict:
         """Convert the ScoringResult instance to a dictionary, properly serializing scorer_data."""
         return {
@@ -46,8 +32,7 @@ class ScoringResult(BaseModel):
             success={self.success}, \
             scorer_data={self.scorers_data}, \
             data_object={self.data_object}, \
-            run_duration={self.run_duration}, \
-            evaluation_cost={self.evaluation_cost})"
+            run_duration={self.run_duration})"
 
 
 def generate_scoring_result(
@@ -62,18 +47,15 @@ def generate_scoring_result(
     When an LLMTestCase is executed, it turns into an LLMApiTestCase and the progress of the evaluation run is tracked.
     At the end of the evaluation run, we create a TestResult object out of the completed LLMApiTestCase.
     """
-    if data_object.name is not None:
+    if hasattr(data_object, "name") and data_object.name is not None:
         name = data_object.name
     else:
         name = "Test Case Placeholder"
-        debug(f"No name provided for example, using default name: {name}")
-    debug(f"Creating ScoringResult for: {name}")
     scoring_result = ScoringResult(
         name=name,
         data_object=data_object,
         success=success,
         scorers_data=scorers_data,
         run_duration=run_duration,
-        evaluation_cost=None,
     )
     return scoring_result

judgeval/data/scorer_data.py

@@ -4,13 +4,12 @@ Implementation of the ScorerData class.
 ScorerData holds the information related to a single, completed Scorer evaluation run.
 """
 
-from typing import List, Union, Optional, Dict
-from pydantic import BaseModel
+from judgeval.data.judgment_types import ScorerDataJudgmentType
+from judgeval.scorers import BaseScorer
+from typing import List
 
-from judgeval.scorers import JudgevalScorer
 
-
-class ScorerData(BaseModel):
+class ScorerData(ScorerDataJudgmentType):
     """
     ScorerData holds the information related to a single, completed Scorer evaluation run.
 
@@ -20,18 +19,6 @@ class ScorerData(BaseModel):
     judge model(s).
     """
 
-    name: str
-    threshold: float
-    success: bool
-    score: Optional[float] = None
-    reason: Optional[str] = None
-    strict_mode: Optional[bool] = None
-    evaluation_model: Union[List[str], str] | None = None
-    error: Optional[str] = None
-    evaluation_cost: Union[float, None] = None
-    verbose_logs: Optional[str] = None
-    additional_metadata: Optional[Dict] = None
-
     def to_dict(self) -> dict:
         """Convert the ScorerData instance to a JSON-serializable dictionary."""
         return {
@@ -43,13 +30,11 @@ class ScorerData(BaseModel):
             "strict_mode": self.strict_mode,
             "evaluation_model": self.evaluation_model,
             "error": self.error,
-            "evaluation_cost": self.evaluation_cost,
-            "verbose_logs": self.verbose_logs,
             "additional_metadata": self.additional_metadata,
         }
 
 
-def create_scorer_data(scorer: JudgevalScorer) -> ScorerData:
+def create_scorer_data(scorer: BaseScorer) -> List[ScorerData]:
     """
     After a `scorer` is run, it contains information about the example that was evaluated
     using the scorer. For example, after computing Faithfulness, the `scorer` object will contain
@@ -59,30 +44,33 @@ def create_scorer_data(scorer: JudgevalScorer) -> ScorerData:
     contains the output of the scorer run that can be exported to be logged as a part of
     the ScorerResult.
     """
-    if scorer.error is not None:  # error occurred during eval run
-        return ScorerData(
-            name=scorer.__name__,
+    scorers_result = list()
+
+    scorers_result.append(
+        ScorerData(
+            name=scorer.name,
             threshold=scorer.threshold,
-            score=None,
-            reason=None,
-            success=False,
-            strict_mode=scorer.strict_mode,
-            evaluation_model=scorer.evaluation_model,
-            error=scorer.error,
-            evaluation_cost=scorer.evaluation_cost,
-            verbose_logs=scorer.verbose_logs,
-        )
-    else:  # standard execution, no error
-        return ScorerData(
-            name=scorer.__name__,
             score=scorer.score,
-            threshold=scorer.threshold,
             reason=scorer.reason,
-            success=scorer._success_check(),
+            success=scorer.success,
             strict_mode=scorer.strict_mode,
             evaluation_model=scorer.evaluation_model,
-            error=None,
-            evaluation_cost=scorer.evaluation_cost,
-            verbose_logs=scorer.verbose_logs,
+            error=scorer.error,
             additional_metadata=scorer.additional_metadata,
         )
+    )
+    if hasattr(scorer, "internal_scorer") and scorer.internal_scorer is not None:
+        scorers_result.append(
+            ScorerData(
+                name=scorer.internal_scorer.name,
+                score=scorer.internal_scorer.score,
+                threshold=scorer.internal_scorer.threshold,
+                reason=scorer.internal_scorer.reason,
+                success=scorer.internal_scorer.success,
+                strict_mode=scorer.internal_scorer.strict_mode,
+                evaluation_model=scorer.internal_scorer.evaluation_model,
+                error=scorer.internal_scorer.error,
+                additional_metadata=scorer.internal_scorer.additional_metadata,
+            )
+        )
+    return scorers_result

judgeval/data/scripts/fix_default_factory.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+"""
+Post-process generated Pydantic models with default_factory defaults.
+"""
+
+import sys
+
+
+def fix_mutable_defaults(file_path: str) -> None:
+    """Fix mutable defaults in generated Pydantic models."""
+
+    with open(file_path, "r") as f:
+        content = f.read()
+
+    content = content.replace(" = {}", " = Field(default_factory=dict)")
+    content = content.replace(" = []", " = Field(default_factory=list)")
+    with open(file_path, "w") as f:
+        f.write(content)
+
+
+if __name__ == "__main__":
+    file_path = sys.argv[1]
+    fix_mutable_defaults(file_path)