judgeval 0.0.11__py3-none-any.whl → 0.22.2__py3-none-any.whl

judgeval/data/result.py

@@ -1,81 +1,43 @@
-from dataclasses import dataclass
-from typing import List, Union, Optional, Dict, Any
+from typing import List, Union
+from judgeval.data import ScorerData, Example
+from judgeval.data.judgment_types import ScoringResult as JudgmentScoringResult
 
-from judgeval.data import ScorerData, ProcessExample
 
-@dataclass
-class ScoringResult:
+class ScoringResult(JudgmentScoringResult):
     """
     A ScoringResult contains the output of one or more scorers applied to a single example.
     Ie: One input, one actual_output, one expected_output, etc..., and 1+ scorer (Faithfulness, Hallucination, Summarization, etc...)
 
     Args:
-        success (bool): Whether the evaluation was successful. 
+        success (bool): Whether the evaluation was successful.
                         This means that all scorers applied to this example returned a success.
         scorer_data (List[ScorerData]): The scorers data for the evaluated example
-        input (Optional[str]): The input to the example
-        actual_output (Optional[str]): The actual output of the example
-        expected_output (Optional[str]): The expected output of the example
-        context (Optional[List[str]]): The context of the example
-        retrieval_context (Optional[List[str]]): The retrieval context of the example
-        additional_metadata (Optional[Dict[str, Any]]): The additional metadata of the example
-        tools_called (Optional[List[str]]): The tools called by the example
-        expected_tools (Optional[List[str]]): The expected tools of the example
-        trace_id (Optional[str]): The trace id of the example
-        
+        data_object (Optional[Example]): The original example object that was used to create the ScoringResult, can be Example, WorkflowRun (future)
+
     """
-    # Fields for scoring outputs 
-    success: bool  # used for unit testing
-    scorers_data: Union[List[ScorerData], None]
 
-    # Inputs from the original example
-    input: Optional[str] = None
-    actual_output: Optional[str] = None
-    expected_output: Optional[str] = None
-    context: Optional[List[str]] = None
-    retrieval_context: Optional[List[str]] = None
-    additional_metadata: Optional[Dict[str, Any]] = None
-    tools_called: Optional[List[str]] = None
-    expected_tools: Optional[List[str]] = None
-    trace_id: Optional[str] = None
-    
-    example_id: Optional[str] = None
-    eval_run_name: Optional[str] = None
-    
-    def to_dict(self) -> dict:
-        """Convert the ScoringResult instance to a dictionary, properly serializing scorer_data."""
-        return {
-            "success": self.success,
-            "scorers_data": [scorer_data.to_dict() for scorer_data in self.scorers_data] if self.scorers_data else None,
-            "input": self.input,
-            "actual_output": self.actual_output,
-            "expected_output": self.expected_output,
-            "context": self.context,
-            "retrieval_context": self.retrieval_context,
-            "additional_metadata": self.additional_metadata,
-            "tools_called": self.tools_called,
-            "expected_tools": self.expected_tools,
-            "trace_id": self.trace_id,
-            "example_id": self.example_id
-        }
-    
+    # Need to override this so that it uses this repo's Example class
+    data_object: Example
+    scorers_data: List[ScorerData]
+
+    def model_dump(self, **kwargs):
+        data = super().model_dump(**kwargs)
+        data["data_object"] = self.data_object.model_dump()
+        return data
+
     def __str__(self) -> str:
         return f"ScoringResult(\
             success={self.success}, \
-            scorer_data={self.scorers_data}, \
-            input={self.input}, \
-            actual_output={self.actual_output}, \
-            expected_output={self.expected_output}, \
-            context={self.context}, \
-            retrieval_context={self.retrieval_context}, \
-            additional_metadata={self.additional_metadata}, \
-            tools_called={self.tools_called}, \
-            expected_tools={self.expected_tools}, \
-            trace_id={self.trace_id})"
+            scorers_data={self.scorers_data}, \
+            data_object={self.data_object}, \
+            run_duration={self.run_duration})"
 
 
 def generate_scoring_result(
-    process_example: ProcessExample,
+    data_object: Union[Example],
+    scorers_data: List[ScorerData],
+    run_duration: float,
+    success: bool,
 ) -> ScoringResult:
     """
     Creates a final ScoringResult object for an evaluation run based on the results from a completed LLMApiTestCase.
@@ -83,16 +45,10 @@ def generate_scoring_result(
     When an LLMTestCase is executed, it turns into an LLMApiTestCase and the progress of the evaluation run is tracked.
     At the end of the evaluation run, we create a TestResult object out of the completed LLMApiTestCase.
     """
-    return ScoringResult(
-        success=process_example.success,
-        scorers_data=process_example.scorers_data,
-        input=process_example.input,
-        actual_output=process_example.actual_output,
-        expected_output=process_example.expected_output,
-        context=process_example.context,
-        retrieval_context=process_example.retrieval_context,
-        additional_metadata=process_example.additional_metadata,
-        tools_called=process_example.tools_called,
-        expected_tools=process_example.expected_tools,
-        trace_id=process_example.trace_id
+    scoring_result = ScoringResult(
+        data_object=data_object,
+        success=success,
+        scorers_data=scorers_data,
+        run_duration=run_duration,
     )
+    return scoring_result

judgeval/data/scorer_data.py

@@ -4,50 +4,14 @@ Implementation of the ScorerData class.
 ScorerData holds the information related to a single, completed Scorer evaluation run.
 """
 
-from typing import List, Union, Optional, Dict
-from pydantic import BaseModel, Field
+from __future__ import annotations
 
-from judgeval.scorers import JudgevalScorer
+from judgeval.data.judgment_types import ScorerData
+from judgeval.scorers import BaseScorer
+from typing import List
 
-class ScorerData(BaseModel):
-    """
-    ScorerData holds the information related to a single, completed Scorer evaluation run.
-
-    For example, if running the Judgment Faithfulness scorer on an example, the ScorerData
-    object will contain whether the example passed its threshold expectation, as well as more detailed
-    information surrounding the evaluation run such as the claims and verdicts generated by the
-    judge model(s).
-    """
-    name: str
-    threshold: float
-    success: bool
-    score: Optional[float] = None
-    reason: Optional[str] = None
-    strict_mode: Optional[bool] = None
-    evaluation_model: Union[List[str], str] = None
-    error: Optional[str] = None
-    evaluation_cost: Union[float, None] = None
-    verbose_logs: Optional[str] = None
-    additional_metadata: Optional[Dict] = None
 
-    def to_dict(self) -> dict:
-        """Convert the ScorerData instance to a JSON-serializable dictionary."""
-        return {
-            "name": self.name,
-            "threshold": self.threshold,
-            "success": self.success,
-            "score": self.score,
-            "reason": self.reason,
-            "strict_mode": self.strict_mode,
-            "evaluation_model": self.evaluation_model,
-            "error": self.error,
-            "evaluation_cost": self.evaluation_cost,
-            "verbose_logs": self.verbose_logs,
-            "additional_metadata": self.additional_metadata
-        }
-
-
-def create_scorer_data(scorer: JudgevalScorer) -> ScorerData:
+def create_scorer_data(scorer: BaseScorer) -> List[ScorerData]:
     """
     After a `scorer` is run, it contains information about the example that was evaluated
     using the scorer. For example, after computing Faithfulness, the `scorer` object will contain
@@ -57,30 +21,33 @@ def create_scorer_data(scorer: JudgevalScorer) -> ScorerData:
     contains the output of the scorer run that can be exported to be logged as a part of
     the ScorerResult.
     """
-    if scorer.error is not None:  # error occurred during eval run
-        return ScorerData(
-            name=scorer.__name__,
+    scorers_result = list()
+
+    scorers_result.append(
+        ScorerData(
+            name=scorer.name,
             threshold=scorer.threshold,
-            score=None,
-            reason=None,
-            success=False,
-            strict_mode=scorer.strict_mode,
-            evaluation_model=scorer.evaluation_model,
-            error=scorer.error,
-            evaluation_cost=scorer.evaluation_cost,
-            verbose_logs=scorer.verbose_logs,
-        )
-    else:  # standard execution, no error
-        return ScorerData(
-            name=scorer.__name__,
             score=scorer.score,
-            threshold=scorer.threshold,
             reason=scorer.reason,
-            success=scorer._success_check(),
+            success=scorer.success,
             strict_mode=scorer.strict_mode,
-            evaluation_model=scorer.evaluation_model,
-            error=None,
-            evaluation_cost=scorer.evaluation_cost,
-            verbose_logs=scorer.verbose_logs,
+            evaluation_model=scorer.model,
+            error=scorer.error,
             additional_metadata=scorer.additional_metadata,
         )
+    )
+    if hasattr(scorer, "internal_scorer") and scorer.internal_scorer is not None:
+        scorers_result.append(
+            ScorerData(
+                name=scorer.internal_scorer.name,
+                score=scorer.internal_scorer.score,
+                threshold=scorer.internal_scorer.threshold,
+                reason=scorer.internal_scorer.reason,
+                success=scorer.internal_scorer.success,
+                strict_mode=scorer.internal_scorer.strict_mode,
+                evaluation_model=scorer.internal_scorer.model,
+                error=scorer.internal_scorer.error,
+                additional_metadata=scorer.internal_scorer.additional_metadata,
+            )
+        )
+    return scorers_result

judgeval/data/scripts/fix_default_factory.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+"""
+Post-process generated Pydantic models with default_factory defaults.
+"""
+
+import sys
+
+
+def fix_mutable_defaults(file_path: str) -> None:
+    """Fix mutable defaults in generated Pydantic models."""
+
+    with open(file_path, "r") as f:
+        content = f.read()
+
+    content = content.replace(" = {}", " = Field(default_factory=dict)")
+    content = content.replace(" = []", " = Field(default_factory=list)")
+    with open(file_path, "w") as f:
+        f.write(content)
+
+
+if __name__ == "__main__":
+    file_path = sys.argv[1]
+    fix_mutable_defaults(file_path)

judgeval/data/scripts/openapi_transform.py

@@ -0,0 +1,123 @@
+import orjson
+import sys
+from typing import Any, Dict, Generator, List
+import requests
+
+spec_file = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8000/openapi.json"
+
+if spec_file.startswith("http"):
+    r = requests.get(spec_file)
+    r.raise_for_status()
+    SPEC = r.json()
+else:
+    with open(spec_file, "rb") as f:
+        SPEC = orjson.loads(f.read())
+
+JUDGEVAL_PATHS: List[str] = [
+    "/log_eval_results/",
+]
+
+
+def resolve_ref(ref: str) -> str:
+    assert ref.startswith("#/components/schemas/"), (
+        "Reference must start with #/components/schemas/"
+    )
+    return ref.replace("#/components/schemas/", "")
+
+
+def walk(obj: Any) -> Generator[Any, None, None]:
+    yield obj
+    if isinstance(obj, list):
+        for item in obj:
+            yield from walk(item)
+    elif isinstance(obj, dict):
+        for value in obj.values():
+            yield from walk(value)
+
+
+def get_referenced_schemas(obj: Any) -> Generator[str, None, None]:
+    for value in walk(obj):
+        if isinstance(value, dict) and "$ref" in value:
+            ref = value["$ref"]
+            resolved = resolve_ref(ref)
+            assert isinstance(ref, str), "Reference must be a string"
+            # Strip the _JudgmentType suffix if it exists to get the original schema name
+            if resolved.endswith("_JudgmentType"):
+                resolved = resolved[: -len("_JudgmentType")]
+            yield resolved
+
+
+def transform_schema_refs(obj: Any) -> Any:
+    """Transform all $ref values in a schema to use the _JudgmentType suffix"""
+    if isinstance(obj, dict):
+        result = {}
+        for key, value in obj.items():
+            if (
+                key == "$ref"
+                and isinstance(value, str)
+                and value.startswith("#/components/schemas/")
+            ):
+                # Update the reference to use the suffixed name
+                original_name = resolve_ref(value)
+                suffixed_name = f"{original_name}_JudgmentType"
+                result[key] = f"#/components/schemas/{suffixed_name}"
+            else:
+                result[key] = transform_schema_refs(value)
+        return result
+    elif isinstance(obj, list):
+        return [transform_schema_refs(item) for item in obj]
+    else:
+        return obj
+
+
+filtered_paths = {
+    path: spec_data
+    for path, spec_data in SPEC["paths"].items()
+    if path in JUDGEVAL_PATHS
+}
+
+
+def filter_schemas() -> Dict[str, Any]:
+    result: Dict[str, Any] = {}
+    processed_original_names: set[str] = set()
+    schemas_to_scan: Any = filtered_paths
+
+    while True:
+        to_commit: Dict[str, Any] = {}
+        for original_schema_name in get_referenced_schemas(schemas_to_scan):
+            if original_schema_name in processed_original_names:
+                continue
+
+            assert original_schema_name in SPEC["components"]["schemas"], (
+                f"Schema {original_schema_name} not found in components.schemas"
+            )
+            # Transform the schema to update any internal references
+            original_schema = SPEC["components"]["schemas"][original_schema_name]
+            transformed_schema = transform_schema_refs(original_schema)
+            suffixed_name = f"{original_schema_name}_JudgmentType"
+            to_commit[suffixed_name] = transformed_schema
+            processed_original_names.add(original_schema_name)
+
+        if not to_commit:
+            break
+
+        result.update(to_commit)
+        schemas_to_scan = to_commit
+
+    return result
+
+
+# Transform the filtered paths to update schema references
+transformed_paths = transform_schema_refs(filtered_paths)
+
+spec = {
+    "openapi": SPEC["openapi"],
+    "info": SPEC["info"],
+    "paths": transformed_paths,
+    "components": {
+        **SPEC["components"],
+        "schemas": filter_schemas(),
+    },
+}
+
+print(orjson.dumps(spec, option=orjson.OPT_INDENT_2).decode("utf-8"))

judgeval/data/trace.py

@@ -0,0 +1,121 @@
+from typing import Optional, List, Dict, Any
+from pydantic import BaseModel
+from .judgment_types import (
+    OtelSpanDetailScores,
+    OtelSpanDetail,
+    OtelTraceListItem,
+)
+
+
+class TraceUsage(BaseModel):
+    prompt_tokens: Optional[int] = None
+    completion_tokens: Optional[int] = None
+    cache_creation_input_tokens: Optional[int] = None
+    cache_read_input_tokens: Optional[int] = None
+    total_tokens: Optional[int] = None
+    prompt_tokens_cost_usd: Optional[float] = None
+    completion_tokens_cost_usd: Optional[float] = None
+    total_cost_usd: Optional[float] = None
+    model_name: Optional[str] = None
+
+
+class TraceScore(OtelSpanDetailScores):
+    """Score information for a trace or span."""
+
+    pass
+
+
+class TraceRule(BaseModel):
+    """Rule that was triggered for a trace."""
+
+    rule_id: str
+    rule_name: str
+
+
+class TraceSpan(OtelSpanDetail):
+    """Individual span within a trace with complete telemetry data."""
+
+    @classmethod
+    def from_otel_span_detail(cls, span_detail: OtelSpanDetail) -> "TraceSpan":
+        """Create TraceSpan from OtelSpanDetail, converting scores to TraceScore."""
+        data = span_detail.model_dump()
+
+        if "scores" in data and data["scores"]:
+            data["scores"] = [TraceScore(**score) for score in data["scores"]]
+
+        return cls(**data)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert TraceSpan to dictionary."""
+        return self.model_dump(exclude_none=True)
+
+
+class Trace(OtelTraceListItem):
+    """Complete trace with metadata and all associated spans."""
+
+    spans: List[TraceSpan] = []
+    rules: Optional[List[TraceRule]] = []
+
+    @classmethod
+    def from_dataset_trace_with_spans(cls, dataset_trace: Any) -> "Trace":
+        """Create Trace from DatasetTraceWithSpans (handles both API and judgment types)."""
+
+        if hasattr(dataset_trace, "trace_detail"):
+            trace_detail = dataset_trace.trace_detail
+            spans_data = dataset_trace.spans
+        else:
+            trace_detail = dataset_trace.get("trace_detail", {})
+            spans_data = dataset_trace.get("spans", [])
+
+        if hasattr(trace_detail, "model_dump"):
+            trace_data = trace_detail.model_dump()
+        elif isinstance(trace_detail, dict):
+            trace_data = trace_detail.copy()
+        else:
+            trace_data = dict(trace_detail)
+
+        spans = []
+        for span in spans_data:
+            if hasattr(span, "model_dump"):
+                spans.append(TraceSpan.from_otel_span_detail(span))
+            else:
+                # Handle dict spans
+                span_data = dict(span) if not isinstance(span, dict) else span.copy()
+                if "scores" in span_data and span_data["scores"]:
+                    span_data["scores"] = [
+                        TraceScore(**score)
+                        if isinstance(score, dict)
+                        else TraceScore(**score.model_dump())
+                        for score in span_data["scores"]
+                    ]
+                spans.append(TraceSpan(**span_data))
+
+        rules = []
+        if "rule_id" in trace_data and trace_data["rule_id"]:
+            rules = [
+                TraceRule(
+                    rule_id=trace_data["rule_id"],
+                    rule_name=f"Rule {trace_data['rule_id']}",
+                )
+            ]
+
+        trace_data.pop("scores", [])
+        trace_data.pop("rule_id", None)
+        trace = cls(**trace_data)
+
+        trace.spans = spans
+        trace.rules = rules
+
+        return trace
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert Trace to dictionary."""
+        return self.model_dump(exclude_none=True)
+
+    def __len__(self) -> int:
+        """Return the number of spans in the trace."""
+        return len(self.spans)
+
+    def __iter__(self):
+        """Iterate over spans in the trace."""
+        return iter(self.spans)