judgeval 0.1.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

judgeval/data/result.py

@@ -1,10 +1,9 @@
 from typing import List, Union
 from judgeval.data import ScorerData, Example
-from judgeval.data.trace import TraceSpan
-from judgeval.data.judgment_types import ScoringResultJudgmentType
+from judgeval.data.judgment_types import ScoringResult as JudgmentScoringResult
 
 
-class ScoringResult(ScoringResultJudgmentType):
+class ScoringResult(JudgmentScoringResult):
     """
     A ScoringResult contains the output of one or more scorers applied to a single example.
     Ie: One input, one actual_output, one expected_output, etc..., and 1+ scorer (Faithfulness, Hallucination, Summarization, etc...)
@@ -17,15 +16,14 @@ class ScoringResult(ScoringResultJudgmentType):
 
     """
 
-    def to_dict(self) -> dict:
-        """Convert the ScoringResult instance to a dictionary, properly serializing scorer_data."""
-        return {
-            "success": self.success,
-            "scorers_data": [scorer_data.to_dict() for scorer_data in self.scorers_data]
-            if self.scorers_data
-            else None,
-            "data_object": self.data_object.to_dict() if self.data_object else None,
-        }
+    # Need to override this so that it uses this repo's Example class
+    data_object: Example
+    scorers_data: List[ScorerData]
+
+    def model_dump(self, **kwargs):
+        data = super().model_dump(**kwargs)
+        data["data_object"] = self.data_object.model_dump()
+        return data
 
     def __str__(self) -> str:
         return f"ScoringResult(\
@@ -36,7 +34,7 @@ class ScoringResult(ScoringResultJudgmentType):
 
 
 def generate_scoring_result(
-    data_object: Union[Example, TraceSpan],
+    data_object: Union[Example],
     scorers_data: List[ScorerData],
     run_duration: float,
     success: bool,
@@ -47,12 +45,7 @@ def generate_scoring_result(
     When an LLMTestCase is executed, it turns into an LLMApiTestCase and the progress of the evaluation run is tracked.
     At the end of the evaluation run, we create a TestResult object out of the completed LLMApiTestCase.
     """
-    if hasattr(data_object, "name") and data_object.name is not None:
-        name = data_object.name
-    else:
-        name = "Test Case Placeholder"
     scoring_result = ScoringResult(
-        name=name,
         data_object=data_object,
         success=success,
         scorers_data=scorers_data,

judgeval/data/scorer_data.py

@@ -4,36 +4,13 @@ Implementation of the ScorerData class.
 ScorerData holds the information related to a single, completed Scorer evaluation run.
 """
 
-from judgeval.data.judgment_types import ScorerDataJudgmentType
+from __future__ import annotations
+
+from judgeval.data.judgment_types import ScorerData
 from judgeval.scorers import BaseScorer
 from typing import List
 
 
-class ScorerData(ScorerDataJudgmentType):
-    """
-    ScorerData holds the information related to a single, completed Scorer evaluation run.
-
-    For example, if running the Judgment Faithfulness scorer on an example, the ScorerData
-    object will contain whether the example passed its threshold expectation, as well as more detailed
-    information surrounding the evaluation run such as the claims and verdicts generated by the
-    judge model(s).
-    """
-
-    def to_dict(self) -> dict:
-        """Convert the ScorerData instance to a JSON-serializable dictionary."""
-        return {
-            "name": self.name,
-            "threshold": self.threshold,
-            "success": self.success,
-            "score": self.score,
-            "reason": self.reason,
-            "strict_mode": self.strict_mode,
-            "evaluation_model": self.evaluation_model,
-            "error": self.error,
-            "additional_metadata": self.additional_metadata,
-        }
-
-
 def create_scorer_data(scorer: BaseScorer) -> List[ScorerData]:
     """
     After a `scorer` is run, it contains information about the example that was evaluated

judgeval/data/scripts/openapi_transform.py

@@ -1,7 +1,7 @@
-import json
+import orjson
 import sys
 from typing import Any, Dict, Generator, List
-from judgeval.utils.requests import requests
+import requests
 
 spec_file = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8000/openapi.json"
 
@@ -10,8 +10,8 @@ if spec_file.startswith("http"):
     r.raise_for_status()
     SPEC = r.json()
 else:
-    with open(spec_file, "r") as f:
-        SPEC = json.load(f)
+    with open(spec_file, "rb") as f:
+        SPEC = orjson.loads(f.read())
 
 JUDGEVAL_PATHS: List[str] = [
     "/log_eval_results/",
@@ -120,4 +120,4 @@ spec = {
     },
 }
 
-print(json.dumps(spec, indent=4))
+print(orjson.dumps(spec, option=orjson.OPT_INDENT_2).decode("utf-8"))

judgeval/data/trace.py

@@ -1,200 +1,121 @@
-from typing import Any
-import json
-import sys
-import threading
-from datetime import datetime, timezone
-from judgeval.data.judgment_types import (
-    TraceUsageJudgmentType,
-    TraceSpanJudgmentType,
-    TraceJudgmentType,
-)
-from judgeval.constants import SPAN_LIFECYCLE_END_UPDATE_ID
+from typing import Optional, List, Dict, Any
 from pydantic import BaseModel
+from .judgment_types import (
+    OtelSpanDetailScores,
+    OtelSpanDetail,
+    OtelTraceListItem,
+)
 
 
-class TraceUsage(TraceUsageJudgmentType):
-    pass
+class TraceUsage(BaseModel):
+    prompt_tokens: Optional[int] = None
+    completion_tokens: Optional[int] = None
+    cache_creation_input_tokens: Optional[int] = None
+    cache_read_input_tokens: Optional[int] = None
+    total_tokens: Optional[int] = None
+    prompt_tokens_cost_usd: Optional[float] = None
+    completion_tokens_cost_usd: Optional[float] = None
+    total_cost_usd: Optional[float] = None
+    model_name: Optional[str] = None
 
 
-class TraceSpan(TraceSpanJudgmentType):
-    def model_dump(self, **kwargs):
-        return {
-            "span_id": self.span_id,
-            "trace_id": self.trace_id,
-            "depth": self.depth,
-            "created_at": datetime.fromtimestamp(
-                self.created_at, tz=timezone.utc
-            ).isoformat(),
-            "inputs": self._serialize_value(self.inputs),
-            "output": self._serialize_value(self.output),
-            "error": self._serialize_value(self.error),
-            "parent_span_id": self.parent_span_id,
-            "function": self.function,
-            "duration": self.duration,
-            "span_type": self.span_type,
-            "usage": self.usage.model_dump() if self.usage else None,
-            "has_evaluation": self.has_evaluation,
-            "agent_name": self.agent_name,
-            "state_before": self.state_before,
-            "state_after": self.state_after,
-            "additional_metadata": self._serialize_value(self.additional_metadata),
-            "update_id": self.update_id,
-        }
-
-    def __init__(self, **data):
-        super().__init__(**data)
-        # Initialize thread lock for thread-safe update_id increment
-        self._update_id_lock = threading.Lock()
-
-    def increment_update_id(self) -> int:
-        """
-        Thread-safe method to increment the update_id counter.
-        Returns:
-            int: The new update_id value after incrementing
-        """
-        with self._update_id_lock:
-            self.update_id += 1
-            return self.update_id
-
-    def set_update_id_to_ending_number(
-        self, ending_number: int = SPAN_LIFECYCLE_END_UPDATE_ID
-    ) -> int:
-        """
-        Thread-safe method to set the update_id to a predetermined ending number.
-
-        Args:
-            ending_number (int): The number to set update_id to. Defaults to SPAN_LIFECYCLE_END_UPDATE_ID.
-
-        Returns:
-            int: The new update_id value after setting
-        """
-        with self._update_id_lock:
-            self.update_id = ending_number
-            return self.update_id
-
-    def print_span(self):
-        """Print the span with proper formatting and parent relationship information."""
-        indent = "  " * self.depth
-        parent_info = (
-            f" (parent_id: {self.parent_span_id})" if self.parent_span_id else ""
-        )
-        print(f"{indent}→ {self.function} (id: {self.span_id}){parent_info}")
-
-    def _is_json_serializable(self, obj: Any) -> bool:
-        """Helper method to check if an object is JSON serializable."""
-        try:
-            json.dumps(obj)
-            return True
-        except (TypeError, OverflowError, ValueError):
-            return False
-
-    def safe_stringify(self, output, function_name):
-        """
-        Safely converts an object to a JSON-serializable structure, handling common object types intelligently.
-        """
-        # Handle Pydantic models
-        if hasattr(output, "model_dump"):
-            try:
-                return output.model_dump()
-            except Exception:
-                pass
-
-        # Handle LangChain messages and similar objects with content/type
-        if hasattr(output, "content") and hasattr(output, "type"):
-            try:
-                result = {"type": output.type, "content": output.content}
-                # Add additional fields if they exist
-                if hasattr(output, "additional_kwargs"):
-                    result["additional_kwargs"] = output.additional_kwargs
-                if hasattr(output, "response_metadata"):
-                    result["response_metadata"] = output.response_metadata
-                if hasattr(output, "name"):
-                    result["name"] = output.name
-                return result
-            except Exception:
-                pass
-
-        if hasattr(output, "dict"):
-            try:
-                return output.dict()
-            except Exception:
-                pass
-
-        if hasattr(output, "to_dict"):
-            try:
-                return output.to_dict()
-            except Exception:
-                pass
-
-        if hasattr(output, "__dataclass_fields__"):
-            try:
-                import dataclasses
-
-                return dataclasses.asdict(output)
-            except Exception:
-                pass
-
-        if hasattr(output, "__dict__"):
-            try:
-                return output.__dict__
-            except Exception:
-                pass
-
-        try:
-            return str(output)
-        except (TypeError, OverflowError, ValueError):
-            pass
-
-        try:
-            return repr(output)
-        except (TypeError, OverflowError, ValueError):
-            pass
-
-        return None
-
-    def _serialize_value(self, value: Any) -> Any:
-        """Helper method to deep serialize a value safely supporting Pydantic Models / regular PyObjects."""
-        if value is None:
-            return None
-
-        recursion_limit = sys.getrecursionlimit()
-        recursion_limit = int(recursion_limit * 0.75)
-
-        def serialize_value(value, current_depth=0):
-            try:
-                if current_depth > recursion_limit:
-                    return {"error": "max_depth_reached: " + type(value).__name__}
-
-                if isinstance(value, BaseModel):
-                    return value.model_dump()
-                elif isinstance(value, dict):
-                    # Recursively serialize dictionary values
-                    return {
-                        k: serialize_value(v, current_depth + 1)
-                        for k, v in value.items()
-                    }
-                elif isinstance(value, (list, tuple)):
-                    # Recursively serialize list/tuple items
-                    return [serialize_value(item, current_depth + 1) for item in value]
-                else:
-                    # Try direct JSON serialization first
-                    try:
-                        json.dumps(value)
-                        return value
-                    except (TypeError, OverflowError, ValueError):
-                        # Fallback to safe stringification
-                        return self.safe_stringify(value, self.function)
-                    except Exception:
-                        return {"error": "Unable to serialize"}
-            except Exception:
-                return {"error": "Unable to serialize"}
-
-        # Start serialization with the top-level value
-        try:
-            return serialize_value(value, current_depth=0)
-        except Exception:
-            return {"error": "Unable to serialize"}
-
-
-class Trace(TraceJudgmentType):
+class TraceScore(OtelSpanDetailScores):
+    """Score information for a trace or span."""
+
     pass
+
+
+class TraceRule(BaseModel):
+    """Rule that was triggered for a trace."""
+
+    rule_id: str
+    rule_name: str
+
+
+class TraceSpan(OtelSpanDetail):
+    """Individual span within a trace with complete telemetry data."""
+
+    @classmethod
+    def from_otel_span_detail(cls, span_detail: OtelSpanDetail) -> "TraceSpan":
+        """Create TraceSpan from OtelSpanDetail, converting scores to TraceScore."""
+        data = span_detail.model_dump()
+
+        if "scores" in data and data["scores"]:
+            data["scores"] = [TraceScore(**score) for score in data["scores"]]
+
+        return cls(**data)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert TraceSpan to dictionary."""
+        return self.model_dump(exclude_none=True)
+
+
+class Trace(OtelTraceListItem):
+    """Complete trace with metadata and all associated spans."""
+
+    spans: List[TraceSpan] = []
+    rules: Optional[List[TraceRule]] = []
+
+    @classmethod
+    def from_dataset_trace_with_spans(cls, dataset_trace: Any) -> "Trace":
+        """Create Trace from DatasetTraceWithSpans (handles both API and judgment types)."""
+
+        if hasattr(dataset_trace, "trace_detail"):
+            trace_detail = dataset_trace.trace_detail
+            spans_data = dataset_trace.spans
+        else:
+            trace_detail = dataset_trace.get("trace_detail", {})
+            spans_data = dataset_trace.get("spans", [])
+
+        if hasattr(trace_detail, "model_dump"):
+            trace_data = trace_detail.model_dump()
+        elif isinstance(trace_detail, dict):
+            trace_data = trace_detail.copy()
+        else:
+            trace_data = dict(trace_detail)
+
+        spans = []
+        for span in spans_data:
+            if hasattr(span, "model_dump"):
+                spans.append(TraceSpan.from_otel_span_detail(span))
+            else:
+                # Handle dict spans
+                span_data = dict(span) if not isinstance(span, dict) else span.copy()
+                if "scores" in span_data and span_data["scores"]:
+                    span_data["scores"] = [
+                        TraceScore(**score)
+                        if isinstance(score, dict)
+                        else TraceScore(**score.model_dump())
+                        for score in span_data["scores"]
+                    ]
+                spans.append(TraceSpan(**span_data))
+
+        rules = []
+        if "rule_id" in trace_data and trace_data["rule_id"]:
+            rules = [
+                TraceRule(
+                    rule_id=trace_data["rule_id"],
+                    rule_name=f"Rule {trace_data['rule_id']}",
+                )
+            ]
+
+        trace_data.pop("scores", [])
+        trace_data.pop("rule_id", None)
+        trace = cls(**trace_data)
+
+        trace.spans = spans
+        trace.rules = rules
+
+        return trace
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert Trace to dictionary."""
+        return self.model_dump(exclude_none=True)
+
+    def __len__(self) -> int:
+        """Return the number of spans in the trace."""
+        return len(self.spans)
+
+    def __iter__(self):
+        """Iterate over spans in the trace."""
+        return iter(self.spans)