judgeval 0.0.20__tar.gz → 0.0.22__tar.gz

{judgeval-0.0.20 → judgeval-0.0.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.0.20
+Version: 0.0.22
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues
@@ -12,9 +12,15 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.11
 Requires-Dist: anthropic
 Requires-Dist: fastapi
+Requires-Dist: langchain
+Requires-Dist: langchain-anthropic
+Requires-Dist: langchain-core
+Requires-Dist: langchain-huggingface
+Requires-Dist: langchain-openai
 Requires-Dist: litellm
 Requires-Dist: nest-asyncio
 Requires-Dist: openai
+Requires-Dist: openpyxl
 Requires-Dist: pandas
 Requires-Dist: pika
 Requires-Dist: python-dotenv==1.0.1
@@ -23,8 +29,6 @@ Requires-Dist: supabase
 Requires-Dist: together
 Requires-Dist: uvicorn
 Provides-Extra: dev
-Requires-Dist: langfuse==2.50.3; extra == 'dev'
-Requires-Dist: patronus; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.25.0; extra == 'dev'
 Requires-Dist: pytest-mock>=3.14.0; extra == 'dev'
 Requires-Dist: pytest>=8.3.4; extra == 'dev'

{judgeval-0.0.20 → judgeval-0.0.22}/Pipfile

@@ -29,8 +29,6 @@ pytest = "*"
 pytest-asyncio = "*"
 pytest-mock = "*"
 tavily-python = "*"
-patronus = "*"
-langfuse = "==2.50.3"
 
 [requires]
 python_version = "3.11"

{judgeval-0.0.20 → judgeval-0.0.22}/docs/integration/langgraph.mdx

@@ -14,10 +14,11 @@ graph_builder = StateGraph(State)
 # YOUR LANGGRAPH WORKFLOW
 
 handler = JudgevalCallbackHandler(judgment.get_current_trace())
+set_global_handler(handler)
 
 result = graph.invoke({
     "messages": [HumanMessage(content=prompt)]
-}, config=dict(callbacks=[handler]))
+})
 
 ```
 

{judgeval-0.0.20 → judgeval-0.0.22}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.0.20"
+version = "0.0.22"
 authors = [
     { name="Andrew Li", email="andrew@judgmentlabs.ai" },
     { name="Alex Shan", email="alex@judgmentlabs.ai" },
@@ -28,6 +28,12 @@ dependencies = [
     "anthropic",
     "nest-asyncio",
     "pika",
+    "openpyxl",
+    "langchain",
+    "langchain-huggingface",
+    "langchain-openai",
+    "langchain-anthropic",
+    "langchain-core",
 ]
 
 [project.optional-dependencies]
@@ -35,8 +41,6 @@ dev = [
     "pytest>=8.3.4",
     "pytest-asyncio>=0.25.0",
     "pytest-mock>=3.14.0",
-    "langfuse==2.50.3",
-    "patronus",
     "tavily-python"
 ]
 

judgeval-0.0.22/src/demo/custom_example_demo/qodo_example.py

@@ -0,0 +1,39 @@
+from judgeval.data import CustomExample
+from pydantic import field_validator
+
+class QodoExample(CustomExample):
+    code: str
+    original_code: str
+    
+    def __init__(self, **data):
+        super().__init__(**data)
+        
+    @field_validator('code', 'original_code', mode='before')
+    @classmethod
+    def validate_code(cls, v):
+        if v is not None and not isinstance(v, str):
+            raise ValueError(f"Code must be a string or None but got {v} of type {type(v)}")
+        return v
+    
+    def to_dict(self):
+        return {
+            "code": self.code,
+            "original_code": self.original_code,
+            **super().to_dict()
+        }
+    
+    def model_dump(self, **kwargs):
+        """
+        Custom serialization that handles special cases for fields that might fail standard serialization.
+        """
+        data = super().model_dump(**kwargs)
+        
+        # Do any additional serialization here
+        data["code"] = self.code
+        data["original_code"] = self.original_code
+        
+        return data
+
+
+
+

judgeval-0.0.22/src/demo/custom_example_demo/test.py

@@ -0,0 +1,16 @@
+from judgeval.data import CustomExample
+from judgeval import JudgmentClient
+from qodo_example import QodoExample
+
+custom_example = CustomExample(
+    code="print('Hello, world!')",
+    original_code="print('Hello, world!')",
+)
+
+qodo_example = QodoExample(
+    code="print('Hello, world!')",
+    original_code="print('Hello, world!')",
+)
+
+print(f"{custom_example=}")
+print(f"{qodo_example=}")

{judgeval-0.0.20 → judgeval-0.0.22}/src/judgeval/common/tracer.py

@@ -10,6 +10,7 @@ import os
 import time
 import uuid
 import warnings
+from contextvars import ContextVar
 from contextlib import contextmanager
 from collections import defaultdict
 from dataclasses import dataclass, field
@@ -37,6 +38,7 @@ from judgeval.constants import (
     RABBITMQ_PORT,
     RABBITMQ_QUEUE,
     JUDGMENT_TRACES_DELETE_API_URL,
+    JUDGMENT_PROJECT_DELETE_API_URL,
     JUDGMENT_TRACES_ADD_TO_EVAL_QUEUE_API_URL
 )
 from judgeval.judgment_client import JudgmentClient
@@ -54,7 +56,7 @@ from langchain_core.utils.function_calling import convert_to_openai_tool
 from langchain_core.callbacks import CallbackManager, BaseCallbackHandler
 from langchain_core.agents import AgentAction, AgentFinish
 from langchain_core.outputs import LLMResult
-
+from langchain_core.tracers.context import register_configure_hook
 from langchain_core.messages.ai import AIMessage
 from langchain_core.messages.tool import ToolMessage
 from langchain_core.messages.base import BaseMessage
@@ -251,7 +253,8 @@ class TraceManagerClient:
             raise ValueError(f"Failed to save trace data: {response.text}")
         
         if not empty_save and "ui_results_url" in response.json():
-            rprint(f"\n🔍 You can view your trace data here: [rgb(106,0,255)]{response.json()['ui_results_url']}[/]\n")
+            pretty_str = f"\n🔍 You can view your trace data here: [rgb(106,0,255)][link={response.json()['ui_results_url']}]View Trace[/link]\n"
+            rprint(pretty_str)
 
     def delete_trace(self, trace_id: str):
         """
@@ -294,6 +297,27 @@ class TraceManagerClient:
             raise ValueError(f"Failed to delete trace: {response.text}")
         
         return response.json()
+    
+    def delete_project(self, project_name: str):
+        """
+        Deletes a project from the server. Which also deletes all evaluations and traces associated with the project.
+        """
+        response = requests.delete(
+            JUDGMENT_PROJECT_DELETE_API_URL,
+            json={
+                "project_name": project_name,
+            },
+            headers={
+                "Content-Type": "application/json",
+                "Authorization": f"Bearer {self.judgment_api_key}",
+                "X-Organization-Id": self.organization_id
+            }
+        )
+
+        if response.status_code != HTTPStatus.OK:
+            raise ValueError(f"Failed to delete traces: {response.text}")
+            
+        return response.json()
 
 
 class TraceClient:
@@ -1152,3 +1176,18 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
             'args': str(messages),
             'kwargs': kwargs
         })
+
+judgeval_callback_handler_var: ContextVar[Optional[JudgevalCallbackHandler]] = ContextVar(
+    "judgeval_callback_handler", default=None
+)
+
+def set_global_handler(handler: JudgevalCallbackHandler):
+    judgeval_callback_handler_var.set(handler)
+
+def clear_global_handler():
+    judgeval_callback_handler_var.set(None)
+
+register_configure_hook(
+    context_var=judgeval_callback_handler_var,
+    inheritable=True,
+)

{judgeval-0.0.20 → judgeval-0.0.22}/src/judgeval/constants.py

@@ -48,6 +48,7 @@ JUDGMENT_EVAL_LOG_API_URL = f"{ROOT_API}/log_eval_results/"
 JUDGMENT_EVAL_FETCH_API_URL = f"{ROOT_API}/fetch_eval_results/"
 JUDGMENT_EVAL_DELETE_API_URL = f"{ROOT_API}/delete_eval_results_by_project_and_run_name/"
 JUDGMENT_EVAL_DELETE_PROJECT_API_URL = f"{ROOT_API}/delete_eval_results_by_project/"
+JUDGMENT_PROJECT_DELETE_API_URL = f"{ROOT_API}/projects/delete/"
 JUDGMENT_TRACES_FETCH_API_URL = f"{ROOT_API}/traces/fetch/"
 JUDGMENT_TRACES_SAVE_API_URL = f"{ROOT_API}/traces/save/"
 JUDGMENT_TRACES_DELETE_API_URL = f"{ROOT_API}/traces/delete/"

{judgeval-0.0.20 → judgeval-0.0.22}/src/judgeval/data/__init__.py

@@ -2,8 +2,7 @@ from judgeval.data.example import Example, ExampleParams
 from judgeval.data.api_example import ProcessExample, create_process_example
 from judgeval.data.scorer_data import ScorerData, create_scorer_data
 from judgeval.data.result import ScoringResult, generate_scoring_result
-from judgeval.data.ground_truth import GroundTruthExample
-
+from judgeval.data.custom_example import CustomExample
 __all__ = [
     "Example",
     "ExampleParams",
@@ -13,5 +12,5 @@ __all__ = [
     "create_scorer_data",
     "ScoringResult",
     "generate_scoring_result",
-    "GroundTruthExample",
+    "CustomExample",
 ]

judgeval-0.0.22/src/judgeval/data/custom_example.py

@@ -0,0 +1,98 @@
+from pydantic import BaseModel, Field, field_validator
+from typing import Optional, Dict, Any
+from uuid import uuid4
+from datetime import datetime
+import json
+import warnings
+
+# Brainstorming what are the requirements for the fields?
+class CustomExample(BaseModel):
+    name: Optional[str] = None
+    additional_metadata: Optional[Dict[str, Any]] = None
+    example_id: str = Field(default_factory=lambda: str(uuid4()))
+    example_index: Optional[int] = None
+    timestamp: Optional[str] = None
+    trace_id: Optional[str] = None
+    
+    model_config = {
+        "extra": "allow",  # Allow extra fields with any types
+    }
+    
+    def __init__(self, **data):
+        if 'example_id' not in data:
+            data['example_id'] = str(uuid4())
+        # Set timestamp if not provided
+        if 'timestamp' not in data:
+            data['timestamp'] = datetime.now().isoformat()
+        super().__init__(**data)
+        
+    @field_validator('additional_metadata', mode='before')
+    @classmethod
+    def validate_additional_metadata(cls, v):
+        if v is not None and not isinstance(v, dict):
+            raise ValueError(f"Additional metadata must be a dictionary or None but got {v} of type {type(v)}")
+        return v
+    
+    @field_validator('example_index', mode='before')
+    @classmethod
+    def validate_example_index(cls, v):
+        if v is not None and not isinstance(v, int):
+            raise ValueError(f"Example index must be an integer or None but got {v} of type {type(v)}")
+        return v
+    
+    @field_validator('timestamp', mode='before')
+    @classmethod
+    def validate_timestamp(cls, v):
+        if v is not None and not isinstance(v, str):
+            raise ValueError(f"Timestamp must be a string or None but got {v} of type {type(v)}")
+        return v
+    
+    @field_validator('trace_id', mode='before')
+    @classmethod
+    def validate_trace_id(cls, v):
+        if v is not None and not isinstance(v, str):
+            raise ValueError(f"Trace ID must be a string or None but got {v} of type {type(v)}")
+        return v
+        
+    def to_dict(self):
+        return self.model_dump()
+    
+    def __str__(self):
+        return str(self.model_dump())
+    
+    def model_dump(self, **kwargs):
+        """
+        Custom serialization that handles special cases for fields that might fail standard serialization.
+        """
+        data = super().model_dump(**kwargs)
+        
+        # Get all fields including custom ones
+        all_fields = self.__dict__
+        
+        for field_name, value in all_fields.items():
+            try:
+                # Check if the field has its own serialization method
+                if hasattr(value, 'to_dict'):
+                    data[field_name] = value.to_dict()
+                elif hasattr(value, 'model_dump'):
+                    data[field_name] = value.model_dump()
+                # Field is already in data from super().model_dump()
+                elif field_name in data:
+                    continue
+                else:
+                    # Try standard JSON serialization
+                    json.dumps(value)
+                    data[field_name] = value
+            except (TypeError, OverflowError, ValueError):
+                # Handle non-serializable objects
+                try:
+                    # Try converting to string
+                    data[field_name] = str(value)
+                except Exception as _:
+                    # If all else fails, store as None and optionally warn
+                    warnings.warn(f"Could not serialize field {field_name}, setting to None")
+                    data[field_name] = None
+        
+        return data
+    
+    

{judgeval-0.0.20 → judgeval-0.0.22}/src/judgeval/data/datasets/dataset.py

@@ -7,12 +7,11 @@ import yaml
 from dataclasses import dataclass, field
 from typing import List, Union, Literal
 
-from judgeval.data import Example, GroundTruthExample
+from judgeval.data import Example
 from judgeval.common.logger import debug, error, warning, info
 
 @dataclass
 class EvalDataset:
-    ground_truths: List[GroundTruthExample]
     examples: List[Example]
     _alias: Union[str, None] = field(default=None)
     _id: Union[str, None] = field(default=None)
@@ -21,13 +20,11 @@ class EvalDataset:
     def __init__(self, 
                  judgment_api_key: str = os.getenv("JUDGMENT_API_KEY"),  
                  organization_id: str = os.getenv("JUDGMENT_ORG_ID"),
-                 ground_truths: List[GroundTruthExample] = [], 
                  examples: List[Example] = [],
                  ):
-        debug(f"Initializing EvalDataset with {len(ground_truths)} ground truths and {len(examples)} examples")
+        debug(f"Initializing EvalDataset with {len(examples)} examples")
         if not judgment_api_key:
             warning("No judgment_api_key provided")
-        self.ground_truths = ground_truths
         self.examples = examples
         self._alias = None
         self._id = None
@@ -37,38 +34,13 @@ class EvalDataset:
     def add_from_json(self, file_path: str) -> None:
         debug(f"Loading dataset from JSON file: {file_path}")
         """
-        Adds examples and ground truths from a JSON file.
+        Adds examples from a JSON file.
 
-        The format of the JSON file is expected to be a dictionary with two keys: "examples" and "ground_truths". 
-        The value of each key is a list of dictionaries, where each dictionary represents an example or ground truth.
+        The format of the JSON file is expected to be a dictionary with one key: "examples". 
+        The value of the key is a list of dictionaries, where each dictionary represents an example.
 
         The JSON file is expected to have the following format:
         {
-            "ground_truths": [
-                {
-                    "input": "test input",
-                    "actual_output": null,
-                    "expected_output": "expected output",
-                    "context": [
-                    "context1"
-                ],
-                "retrieval_context": [
-                    "retrieval1"
-                ],
-                "additional_metadata": {
-                    "key": "value"
-                },
-                "comments": "test comment",
-                "tools_called": [
-                    "tool1"
-                ],
-                "expected_tools": [
-                    "tool1"
-                ],
-                "source_file": "test.py",
-                "trace_id": "094121"
-            }
-        ],
         "examples": [
             {
                 "input": "test input",
@@ -103,7 +75,6 @@ class EvalDataset:
             with open(file_path, "r") as file:
                 payload = json.load(file)
                 examples = payload.get("examples", [])
-                ground_truths = payload.get("ground_truths", [])
         except FileNotFoundError:
             error(f"JSON file not found: {file_path}")
             raise FileNotFoundError(f"The file {file_path} was not found.")
@@ -111,21 +82,17 @@ class EvalDataset:
             error(f"Invalid JSON file: {file_path}")
             raise ValueError(f"The file {file_path} is not a valid JSON file.")
 
-        info(f"Added {len(examples)} examples and {len(ground_truths)} ground truths from JSON")
+        info(f"Added {len(examples)} examples from JSON")
         new_examples = [Example(**e) for e in examples]
         for e in new_examples:
             self.add_example(e)
-
-        new_ground_truths = [GroundTruthExample(**g) for g in ground_truths]
-        for g in new_ground_truths:
-            self.add_ground_truth(g)
   
     def add_from_csv(
         self, 
         file_path: str,
         ) -> None:
         """
-        Add Examples and GroundTruthExamples from a CSV file.
+        Add Examples from a CSV file.
         """
         try:
             import pandas as pd
@@ -144,14 +111,14 @@ class EvalDataset:
         "expected_tools", "name", "comments", "source_file", "example", \
         "trace_id"
 
-        We want to collect the examples and ground truths separately which can
+        We want to collect the examples separately which can
         be determined by the "example" column. If the value is True, then it is an
-        example, otherwise it is a ground truth.
+        example
 
         We also assume that if there are multiple retrieval contexts or contexts, they are separated by semicolons.
         This can be adjusted using the `context_delimiter` and `retrieval_context_delimiter` parameters.
         """
-        examples, ground_truths = [], []
+        examples = []
 
         for _, row in df.iterrows():
             data = {
@@ -174,49 +141,20 @@ class EvalDataset:
                     examples.append(e)
                 else:
                     raise ValueError("Every example must have an 'input' and 'actual_output' field.")
-            else:
-                # GroundTruthExample has `comments` and `source_file` fields
-                data["comments"] = row["comments"] if pd.notna(row["comments"]) else None
-                data["source_file"] = row["source_file"] if pd.notna(row["source_file"]) else None
-                # every GroundTruthExample has `input` field
-                if data["input"] is not None:
-                    g = GroundTruthExample(**data)
-                    ground_truths.append(g)
-                else:
-                    raise ValueError("Every ground truth must have an 'input' field.")
+           
 
         for e in examples:
             self.add_example(e)
 
-        for g in ground_truths:
-            self.add_ground_truth(g)
-
     def add_from_yaml(self, file_path: str) -> None:
         debug(f"Loading dataset from YAML file: {file_path}")
         """
-        Adds examples and ground truths from a YAML file.
+        Adds examples from a YAML file.
 
-        The format of the YAML file is expected to be a dictionary with two keys: "examples" and "ground_truths". 
-        The value of each key is a list of dictionaries, where each dictionary represents an example or ground truth.
+        The format of the YAML file is expected to be a dictionary with one key: "examples". 
+        The value of the key is a list of dictionaries, where each dictionary represents an example.
 
         The YAML file is expected to have the following format:
-        ground_truths:
-          - input: "test input"
-            actual_output: null
-            expected_output: "expected output"
-            context:
-              - "context1"
-            retrieval_context:
-              - "retrieval1"
-            additional_metadata:
-              key: "value"
-            comments: "test comment"
-            tools_called:
-              - "tool1"
-            expected_tools:
-              - "tool1"
-            source_file: "test.py"
-            trace_id: "094121"
         examples:
           - input: "test input"
             actual_output: "test output"
@@ -244,7 +182,6 @@ class EvalDataset:
                 if payload is None:
                     raise ValueError("The YAML file is empty.")
                 examples = payload.get("examples", [])
-                ground_truths = payload.get("ground_truths", [])
         except FileNotFoundError:
             error(f"YAML file not found: {file_path}")
             raise FileNotFoundError(f"The file {file_path} was not found.")
@@ -252,25 +189,18 @@ class EvalDataset:
             error(f"Invalid YAML file: {file_path}")
             raise ValueError(f"The file {file_path} is not a valid YAML file.")
 
-        info(f"Added {len(examples)} examples and {len(ground_truths)} ground truths from YAML")
+        info(f"Added {len(examples)} examples from YAML")
         new_examples = [Example(**e) for e in examples]
         for e in new_examples:
             self.add_example(e)
 
-        new_ground_truths = [GroundTruthExample(**g) for g in ground_truths]
-        for g in new_ground_truths:
-            self.add_ground_truth(g)
-
     def add_example(self, e: Example) -> None:
         self.examples = self.examples + [e]
         # TODO if we need to add rank, then we need to do it here
-
-    def add_ground_truth(self, g: GroundTruthExample) -> None:
-        self.ground_truths = self.ground_truths + [g]
     
     def save_as(self, file_type: Literal["json", "csv", "yaml"], dir_path: str, save_name: str = None) -> None:
         """
-        Saves the dataset as a file. Save both the ground truths and examples.
+        Saves the dataset as a file. Save only the examples.
 
         Args:
             file_type (Literal["json", "csv"]): The file type to save the dataset as.
@@ -285,7 +215,6 @@ class EvalDataset:
             with open(complete_path, "w") as file:
                 json.dump(
                     {
-                        "ground_truths": [g.to_dict() for g in self.ground_truths],
                         "examples": [e.to_dict() for e in self.examples],
                     },
                     file,
@@ -319,24 +248,7 @@ class EvalDataset:
                         ]
                     )
                 
-                for g in self.ground_truths:
-                    writer.writerow(
-                        [
-                            g.input,
-                            g.actual_output,
-                            g.expected_output,
-                            ";".join(g.context),
-                            ";".join(g.retrieval_context),
-                            g.additional_metadata,
-                            ";".join(g.tools_called),
-                            ";".join(g.expected_tools),
-                            None,  # GroundTruthExample does not have name
-                            g.comments,
-                            g.source_file,
-                            False,  # Adding a GroundTruthExample, not an Example
-                            g.trace_id
-                        ]
-                    )
+                
         elif file_type == "yaml":
             with open(complete_path, "w") as file:
                 yaml_data = {
@@ -358,24 +270,6 @@ class EvalDataset:
                         }
                         for e in self.examples
                     ],
-                    "ground_truths": [
-                        {
-                            "input": g.input,
-                            "actual_output": g.actual_output,
-                            "expected_output": g.expected_output,
-                            "context": g.context,
-                            "retrieval_context": g.retrieval_context,
-                            "additional_metadata": g.additional_metadata,
-                            "tools_called": g.tools_called,
-                            "expected_tools": g.expected_tools,
-                            "name": None,  # GroundTruthExample does not have name
-                            "comments": g.comments,
-                            "source_file": g.source_file,
-                            "example": False,  # Adding a GroundTruthExample, not an Example
-                            "trace_id": g.trace_id
-                        }
-                        for g in self.ground_truths
-                    ]
                 }
                 yaml.dump(yaml_data, file, default_flow_style=False)
         else:
@@ -391,7 +285,6 @@ class EvalDataset:
     def __str__(self):
         return (
             f"{self.__class__.__name__}("
-            f"ground_truths={self.ground_truths}, "
             f"examples={self.examples}, "
             f"_alias={self._alias}, "
             f"_id={self._id}"