judgeval 0.0.39__py3-none-any.whl → 0.0.41__py3-none-any.whl

judgeval/common/utils.py

@@ -12,9 +12,10 @@ NOTE: any function beginning with 'a', e.g. 'afetch_together_api_response', is a
 import asyncio
 import concurrent.futures
 import os
+from types import TracebackType
 import requests
 import pprint
-from typing import Any, Dict, List, Literal, Mapping, Optional, Union
+from typing import Any, Dict, List, Literal, Mapping, Optional, TypeAlias, Union
 
 # Third-party imports
 import litellm
@@ -782,3 +783,6 @@ if __name__ == "__main__":
             ]
         ]
     ))
+    
+ExcInfo: TypeAlias = tuple[type[BaseException], BaseException, TracebackType]
+OptExcInfo: TypeAlias = ExcInfo | tuple[None, None, None]

judgeval/constants.py

@@ -28,6 +28,8 @@ class APIScorer(str, Enum):
     GROUNDEDNESS = "groundedness"
     DERAILMENT = "derailment"
     TOOL_ORDER = "tool_order"
+    CLASSIFIER = "classifier"
+    TOOL_DEPENDENCY = "tool_dependency"
     @classmethod
     def _missing_(cls, value):
         # Handle case-insensitive lookup

judgeval/data/__init__.py

@@ -2,7 +2,7 @@ from judgeval.data.example import Example, ExampleParams
 from judgeval.data.custom_example import CustomExample
 from judgeval.data.scorer_data import ScorerData, create_scorer_data
 from judgeval.data.result import ScoringResult, generate_scoring_result
-from judgeval.data.trace import Trace, TraceSpan
+from judgeval.data.trace import Trace, TraceSpan, TraceUsage
 
 
 __all__ = [
@@ -15,4 +15,5 @@ __all__ = [
     "generate_scoring_result",
     "Trace",
     "TraceSpan",
+    "TraceUsage"
 ]

judgeval/data/datasets/dataset.py

@@ -5,14 +5,15 @@ import json
 import os
 import yaml
 from dataclasses import dataclass, field
-from typing import List, Union, Literal
+from typing import List, Union, Literal, Optional
 
-from judgeval.data import Example
+from judgeval.data import Example, Trace
 from judgeval.common.logger import debug, error, warning, info
 
 @dataclass
 class EvalDataset:
     examples: List[Example]
+    traces: List[Trace]
     _alias: Union[str, None] = field(default=None)
     _id: Union[str, None] = field(default=None)
     judgment_api_key: str = field(default="")
@@ -20,12 +21,13 @@ class EvalDataset:
     def __init__(self, 
                  judgment_api_key: str = os.getenv("JUDGMENT_API_KEY"),  
                  organization_id: str = os.getenv("JUDGMENT_ORG_ID"),
-                 examples: List[Example] = [],
+                 examples: Optional[List[Example]] = None,
+                 traces: Optional[List[Trace]] = None
                  ):
-        debug(f"Initializing EvalDataset with {len(examples)} examples")
         if not judgment_api_key:
             warning("No judgment_api_key provided")
-        self.examples = examples
+        self.examples = examples or []
+        self.traces = traces or []
         self._alias = None
         self._id = None
         self.judgment_api_key = judgment_api_key
@@ -218,8 +220,11 @@ class EvalDataset:
             self.add_example(e)
 
     def add_example(self, e: Example) -> None:
-        self.examples = self.examples + [e]
+        self.examples.append(e)
         # TODO if we need to add rank, then we need to do it here
+    
+    def add_trace(self, t: Trace) -> None:
+        self.traces.append(t)
 
     def save_as(self, file_type: Literal["json", "csv", "yaml"], dir_path: str, save_name: str = None) -> None:
         """
@@ -307,6 +312,7 @@ class EvalDataset:
         return (
             f"{self.__class__.__name__}("
             f"examples={self.examples}, "
+            f"traces={self.traces}, "
             f"_alias={self._alias}, "
             f"_id={self._id}"
             f")"

judgeval/data/datasets/eval_dataset_client.py

@@ -13,7 +13,7 @@ from judgeval.constants import (
     JUDGMENT_DATASETS_INSERT_API_URL,
     JUDGMENT_DATASETS_EXPORT_JSONL_API_URL
 )
-from judgeval.data import Example
+from judgeval.data import Example, Trace
 from judgeval.data.datasets import EvalDataset
 
 
@@ -58,6 +58,7 @@ class EvalDatasetClient:
                     "dataset_alias": alias,
                     "project_name": project_name,
                     "examples": [e.to_dict() for e in dataset.examples],
+                    "traces": [t.model_dump() for t in dataset.traces],
                     "overwrite": overwrite,
                 }
             try:
@@ -202,6 +203,7 @@ class EvalDatasetClient:
                 info(f"Successfully pulled dataset with alias '{alias}'")
                 payload = response.json()
                 dataset.examples = [Example(**e) for e in payload.get("examples", [])]
+                dataset.traces = [Trace(**t) for t in payload.get("traces", [])]
                 dataset._alias = payload.get("alias")
                 dataset._id = payload.get("id")
                 progress.update(

judgeval/data/example.py

@@ -36,15 +36,15 @@ class Example(BaseModel):
     name: Optional[str] = None
     example_id: str = Field(default_factory=lambda: str(uuid4()))
     example_index: Optional[int] = None
-    timestamp: Optional[str] = None
+    created_at: Optional[str] = None
     trace_id: Optional[str] = None
     
     def __init__(self, **data):
         if 'example_id' not in data:
             data['example_id'] = str(uuid4())
         # Set timestamp if not provided
-        if 'timestamp' not in data:
-            data['timestamp'] = datetime.now().strftime("%Y%m%d_%H%M%S")
+        if 'created_at' not in data:
+            data['created_at'] = datetime.now().isoformat()
         super().__init__(**data)
     
     @field_validator('input', mode='before')
@@ -123,9 +123,9 @@ class Example(BaseModel):
             raise ValueError(f"Example index must be an integer or None but got {v} of type {type(v)}")
         return v
     
-    @field_validator('timestamp', mode='before')
+    @field_validator('created_at', mode='before')
     @classmethod
-    def validate_timestamp(cls, v):
+    def validate_created_at(cls, v):
         if v is not None and not isinstance(v, str):
             raise ValueError(f"Timestamp must be a string or None but got {v} of type {type(v)}")
         return v
@@ -150,7 +150,7 @@ class Example(BaseModel):
             "name": self.name,
             "example_id": self.example_id,
             "example_index": self.example_index,
-            "timestamp": self.timestamp,
+            "created_at": self.created_at,
         }
 
     def __str__(self):
@@ -166,5 +166,5 @@ class Example(BaseModel):
             f"name={self.name}, "
             f"example_id={self.example_id}, "
             f"example_index={self.example_index}, "
-            f"timestamp={self.timestamp}, "
+            f"created_at={self.created_at}, "
         )

judgeval/data/tool.py

@@ -1,10 +1,14 @@
 from pydantic import BaseModel, field_validator
-from typing import Dict, Any, Optional
+from typing import Dict, Any, Optional, List
 import warnings
 
 class Tool(BaseModel):
     tool_name: str
     parameters: Optional[Dict[str, Any]] = None
+    agent_name: Optional[str] = None
+    result_dependencies: Optional[List[Dict[str, Any]]] = None
+    action_dependencies: Optional[List[Dict[str, Any]]] = None
+    require_all: Optional[bool] = None
     
     @field_validator('tool_name')
     def validate_tool_name(cls, v):
@@ -16,4 +20,28 @@ class Tool(BaseModel):
     def validate_parameters(cls, v):
         if v is not None and not isinstance(v, dict):
             warnings.warn(f"Parameters should be a dictionary, got {type(v)}", UserWarning)
+        return v
+    
+    @field_validator('agent_name')
+    def validate_agent_name(cls, v):
+        if v is not None and not isinstance(v, str):
+            warnings.warn(f"Agent name should be a string, got {type(v)}", UserWarning)
+        return v
+    
+    @field_validator('result_dependencies')
+    def validate_result_dependencies(cls, v):
+        if v is not None and not isinstance(v, list):
+            warnings.warn(f"Result dependencies should be a list, got {type(v)}", UserWarning)
+        return v
+    
+    @field_validator('action_dependencies')
+    def validate_action_dependencies(cls, v):
+        if v is not None and not isinstance(v, list):
+            warnings.warn(f"Action dependencies should be a list, got {type(v)}", UserWarning)
+        return v
+
+    @field_validator('require_all')
+    def validate_require_all(cls, v):
+        if v is not None and not isinstance(v, bool):
+            warnings.warn(f"Require all should be a boolean, got {type(v)}", UserWarning)
         return v

judgeval/data/trace.py

@@ -5,36 +5,56 @@ from judgeval.data.tool import Tool
 import json
 from datetime import datetime, timezone
 
+class TraceUsage(BaseModel):
+    prompt_tokens: Optional[int] = None
+    completion_tokens: Optional[int] = None
+    total_tokens: Optional[int] = None
+    prompt_tokens_cost_usd: Optional[float] = None
+    completion_tokens_cost_usd: Optional[float] = None
+    total_cost_usd: Optional[float] = None
+    model_name: Optional[str] = None
+
 class TraceSpan(BaseModel):
     span_id: str
     trace_id: str
-    function: Optional[str] = None
+    function: str
     depth: int
     created_at: Optional[Any] = None
     parent_span_id: Optional[str] = None
     span_type: Optional[str] = "span"
     inputs: Optional[Dict[str, Any]] = None
+    error: Optional[Dict[str, Any]] = None
     output: Optional[Any] = None
+    usage: Optional[TraceUsage] = None
     duration: Optional[float] = None
     annotation: Optional[List[Dict[str, Any]]] = None
     evaluation_runs: Optional[List[EvaluationRun]] = []
     expected_tools: Optional[List[Tool]] = None
     additional_metadata: Optional[Dict[str, Any]] = None
+    has_evaluation: Optional[bool] = False
+    agent_name: Optional[str] = None
+    state_before: Optional[Dict[str, Any]] = None
+    state_after: Optional[Dict[str, Any]] = None
 
     def model_dump(self, **kwargs):
         return {
             "span_id": self.span_id,
             "trace_id": self.trace_id,
             "depth": self.depth,
-#             "created_at": datetime.fromtimestamp(self.created_at).isoformat(),
             "created_at": datetime.fromtimestamp(self.created_at, tz=timezone.utc).isoformat(),
-            "inputs": self._serialize_inputs(),
-            "output": self._serialize_output(),
+            "inputs": self._serialize_value(self.inputs),
+            "output": self._serialize_value(self.output),
+            "error": self._serialize_value(self.error),
             "evaluation_runs": [run.model_dump() for run in self.evaluation_runs] if self.evaluation_runs else [],
             "parent_span_id": self.parent_span_id,
             "function": self.function,
             "duration": self.duration,
-            "span_type": self.span_type
+            "span_type": self.span_type,
+            "usage": self.usage.model_dump() if self.usage else None,
+            "has_evaluation": self.has_evaluation,
+            "agent_name": self.agent_name,
+            "state_before": self.state_before,
+            "state_after": self.state_after 
         }
     
     def print_span(self):
@@ -42,30 +62,6 @@ class TraceSpan(BaseModel):
         indent = "  " * self.depth
         parent_info = f" (parent_id: {self.parent_span_id})" if self.parent_span_id else ""
         print(f"{indent}→ {self.function} (id: {self.span_id}){parent_info}")
-    
-    def _serialize_inputs(self) -> dict:
-        """Helper method to serialize input data safely."""
-        if self.inputs is None:
-            return {}
-            
-        serialized_inputs = {}
-        for key, value in self.inputs.items():
-            if isinstance(value, BaseModel):
-                serialized_inputs[key] = value.model_dump()
-            elif isinstance(value, (list, tuple)):
-                # Handle lists/tuples of arguments
-                serialized_inputs[key] = [
-                    item.model_dump() if isinstance(item, BaseModel)
-                    else None if not self._is_json_serializable(item)
-                    else item
-                    for item in value
-                ]
-            else:
-                if self._is_json_serializable(value):
-                    serialized_inputs[key] = value
-                else:
-                    serialized_inputs[key] = self.safe_stringify(value, self.function)
-        return serialized_inputs
 
     def _is_json_serializable(self, obj: Any) -> bool:
         """Helper method to check if an object is JSON serializable."""
@@ -88,15 +84,11 @@ class TraceSpan(BaseModel):
             return repr(output)
         except (TypeError, OverflowError, ValueError):
             pass
-
-        warnings.warn(
-            f"Output for function {function_name} is not JSON serializable and could not be converted to string. Setting to None."
-        )
         return None
         
-    def _serialize_output(self) -> Any:
-        """Helper method to serialize output data safely."""
-        if self.output is None:
+    def _serialize_value(self, value: Any) -> Any:
+        """Helper method to deep serialize a value safely supporting Pydantic Models / regular PyObjects."""
+        if value is None:
             return None
             
         def serialize_value(value):
@@ -117,15 +109,15 @@ class TraceSpan(BaseModel):
                     # Fallback to safe stringification
                     return self.safe_stringify(value, self.function)
 
-        # Start serialization with the top-level output
-        return serialize_value(self.output)
+        # Start serialization with the top-level value
+        return serialize_value(value)
 
 class Trace(BaseModel):
     trace_id: str
     name: str
     created_at: str
     duration: float
-    entries: List[TraceSpan]
+    trace_spans: List[TraceSpan]
     overwrite: bool = False
     offline_mode: bool = False
     rules: Optional[Dict[str, Any]] = None

judgeval/data/trace_run.py

@@ -1,4 +1,3 @@
-
 from pydantic import BaseModel
 from typing import List, Optional, Dict, Any, Union, Callable
 from judgeval.data import Trace
@@ -22,6 +21,7 @@ class TraceRun(BaseModel):
         judgment_api_key (Optional[str]): The API key for running evaluations on the Judgment API
         rules (Optional[List[Rule]]): Rules to evaluate against scoring results
         append (Optional[bool]): Whether to append to existing evaluation results
+        tools (Optional[List[Dict[str, Any]]]): List of tools to use for evaluation
     """
 
     # The user will specify whether they want log_results when they call run_eval
@@ -40,6 +40,7 @@ class TraceRun(BaseModel):
     judgment_api_key: Optional[str] = ""
     override: Optional[bool] = False
     rules: Optional[List[Rule]] = None
+    tools: Optional[List[Dict[str, Any]]] = None
 
     class Config:
         arbitrary_types_allowed = True

judgeval/evaluation_run.py

@@ -1,5 +1,5 @@
 from typing import List, Optional, Dict, Any, Union
-from pydantic import BaseModel, field_validator
+from pydantic import BaseModel, field_validator, Field
 
 from judgeval.data import Example, CustomExample
 from judgeval.scorers import JudgevalScorer, APIJudgmentScorer
@@ -27,12 +27,12 @@ class EvaluationRun(BaseModel):
     # The user will specify whether they want log_results when they call run_eval
     log_results: bool = False  # NOTE: log_results has to be set first because it is used to validate project_name and eval_name
     organization_id: Optional[str] = None
-    project_name: Optional[str] = None
-    eval_name: Optional[str] = None
+    project_name: Optional[str] = Field(default=None, validate_default=True)
+    eval_name: Optional[str] = Field(default=None, validate_default=True)
     examples: Union[List[Example], List[CustomExample]]
     scorers: List[Union[APIJudgmentScorer, JudgevalScorer]]
     model: Optional[Union[str, List[str], JudgevalJudge]] = "gpt-4.1"
-    aggregator: Optional[str] = None
+    aggregator: Optional[str] = Field(default=None, validate_default=True)
     metadata: Optional[Dict[str, Any]] = None
     trace_span_id: Optional[str] = None
     # API Key will be "" until user calls client.run_eval(), then API Key will be set
@@ -96,9 +96,6 @@ class EvaluationRun(BaseModel):
     def validate_scorers(cls, v):
         if not v:
             raise ValueError("Scorers cannot be empty.")
-        for s in v:
-            if not isinstance(s, APIJudgmentScorer) and not isinstance(s, JudgevalScorer):
-                raise ValueError(f"Invalid type for Scorer: {type(s)}")
         return v
 
     @field_validator('model')

judgeval/judgment_client.py

@@ -5,6 +5,7 @@ import os
 from uuid import uuid4
 from typing import Optional, List, Dict, Any, Union, Callable
 import requests
+import asyncio
 
 from judgeval.constants import ROOT_API
 from judgeval.data.datasets import EvalDataset, EvalDatasetClient
@@ -62,7 +63,15 @@ class SingletonMeta(type):
         return cls._instances[cls]
 
 class JudgmentClient(metaclass=SingletonMeta):
-    def __init__(self, judgment_api_key: str = os.getenv("JUDGMENT_API_KEY"), organization_id: str = os.getenv("JUDGMENT_ORG_ID")):
+    def __init__(self, judgment_api_key: Optional[str] = os.getenv("JUDGMENT_API_KEY"), organization_id: Optional[str] = os.getenv("JUDGMENT_ORG_ID")):
+        # Check if API key is None
+        if judgment_api_key is None:
+            raise ValueError("JUDGMENT_API_KEY cannot be None. Please provide a valid API key or set the JUDGMENT_API_KEY environment variable.")
+        
+        # Check if organization ID is None
+        if organization_id is None:
+            raise ValueError("JUDGMENT_ORG_ID cannot be None. Please provide a valid organization ID or set the JUDGMENT_ORG_ID environment variable.")
+        
         self.judgment_api_key = judgment_api_key
         self.organization_id = organization_id
         self.eval_dataset_client = EvalDatasetClient(judgment_api_key, organization_id)
@@ -121,7 +130,8 @@ class JudgmentClient(metaclass=SingletonMeta):
         ignore_errors: bool = True,
         rules: Optional[List[Rule]] = None,
         function: Optional[Callable] = None,
-        tracer: Optional[Union[Tracer, BaseCallbackHandler]] = None
+        tracer: Optional[Union[Tracer, BaseCallbackHandler]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None
     ) -> List[ScoringResult]:
         try:         
             
@@ -151,6 +161,7 @@ class JudgmentClient(metaclass=SingletonMeta):
                 append=append,
                 judgment_api_key=self.judgment_api_key,
                 organization_id=self.organization_id,
+                tools=tools
             )
             return run_trace_eval(trace_run, override, ignore_errors, function, tracer, examples)
         except ValueError as e:
@@ -173,7 +184,7 @@ class JudgmentClient(metaclass=SingletonMeta):
         ignore_errors: bool = True,
         async_execution: bool = False,
         rules: Optional[List[Rule]] = None
-    ) -> List[ScoringResult]:
+    ) -> Union[List[ScoringResult], asyncio.Task]:
         """
         Executes an evaluation of `Example`s using one or more `Scorer`s
         
@@ -480,7 +491,7 @@ class JudgmentClient(metaclass=SingletonMeta):
             
         return response.json()["slug"]
     
-    async def assert_test(
+    def assert_test(
         self, 
         scorers: List[Union[APIJudgmentScorer, JudgevalScorer]],
         examples: Optional[List[Example]] = None,
@@ -495,6 +506,7 @@ class JudgmentClient(metaclass=SingletonMeta):
         rules: Optional[List[Rule]] = None,
         function: Optional[Callable] = None,
         tracer: Optional[Union[Tracer, BaseCallbackHandler]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
         async_execution: bool = False
     ) -> None:
         """
@@ -513,6 +525,14 @@ class JudgmentClient(metaclass=SingletonMeta):
             override (bool): Whether to override an existing evaluation run with the same name
             rules (Optional[List[Rule]]): Rules to evaluate against scoring results
         """
+
+        # Check for enable_param_checking and tools
+        for scorer in scorers:
+            if hasattr(scorer, "kwargs") and scorer.kwargs is not None:
+                if scorer.kwargs.get("enable_param_checking") is True:
+                    if not tools:
+                        raise ValueError(f"You must provide the 'tools' argument to assert_test when using a scorer with enable_param_checking=True. If you do not want to do param checking, explicitly set enable_param_checking=False for the {scorer.__name__} scorer.")
+
         # Validate that exactly one of examples or test_file is provided
         if (examples is None and test_file is None) or (examples is not None and test_file is not None):
             raise ValueError("Exactly one of 'examples' or 'test_file' must be provided, but not both")
@@ -530,10 +550,11 @@ class JudgmentClient(metaclass=SingletonMeta):
                 rules=rules,
                 function=function,
                 tracer=tracer,
-                test_file=test_file
+                test_file=test_file,
+                tools=tools
             )
         else:
-            results = await self.run_evaluation(
+            results = self.run_evaluation(
                 examples=examples,
                 scorers=scorers,
                 model=model,
@@ -547,4 +568,10 @@ class JudgmentClient(metaclass=SingletonMeta):
                 async_execution=async_execution
             )
         
-        assert_test(results)
+        if async_execution:
+            # 'results' is an asyncio.Task here, awaiting it gives List[ScoringResult]
+            actual_results = asyncio.run(results)
+            assert_test(actual_results)  # Call the synchronous imported function
+        else:
+            # 'results' is already List[ScoringResult] here (synchronous path)
+            assert_test(results)  # Call the synchronous imported function