judgeval 0.0.21__tar.gz → 0.0.23__tar.gz

{judgeval-0.0.21 → judgeval-0.0.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.0.21
+Version: 0.0.23
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues

{judgeval-0.0.21 → judgeval-0.0.23}/docs/evaluation/scorers/custom_scorers.mdx

@@ -116,9 +116,9 @@ class SampleScorer(JudgevalScorer):
 ```
 
 
-### 4. Implement the `success_check()` method
+### 4. Implement the `_success_check()` method
 
-When executing an evaluation run, `judgeval` will check if your scorer has passed the `success_check()` method. 
+When executing an evaluation run, `judgeval` will check if your scorer has passed the `_success_check()` method. 
 
 You can implement this method in any way you want, but **it should return a `bool`.** Here's a perfectly valid implementation:
 
@@ -126,7 +126,7 @@ You can implement this method in any way you want, but **it should return a `boo
 class SampleScorer(JudgevalScorer):
     ...
 
-    def success_check(self, example):
+    def _success_check(self):
         if self.error is not None:
             return False
         return self.score >= self.threshold  # or you can do self.success if set

{judgeval-0.0.21 → judgeval-0.0.23}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.0.21"
+version = "0.0.23"
 authors = [
     { name="Andrew Li", email="andrew@judgmentlabs.ai" },
     { name="Alex Shan", email="alex@judgmentlabs.ai" },

judgeval-0.0.23/src/demo/cookbooks/JNPR_Mist/test.py

@@ -0,0 +1,21 @@
+from judgeval import JudgmentClient
+from judgeval.data import Example
+from judgeval.scorers import FaithfulnessScorer
+
+client = JudgmentClient()
+
+example = Example(
+    input="What if these shoes don't fit?",
+    actual_output="We offer a 30-day full refund at no extra cost.",
+    retrieval_context=["All customers are eligible for a 30 day full refund at no extra cost."],
+)
+
+scorer = FaithfulnessScorer(threshold=0.5)
+results = client.run_evaluation(
+    examples=[example],
+    scorers=[scorer],
+    model="gpt-4o",
+    eval_run_name="TestRun",
+    project_name="TestProject",
+)
+print(results)

judgeval-0.0.23/src/demo/cookbooks/linkd/text2sql.py

@@ -0,0 +1,14 @@
+"""
+ClassifierScorer implementation for basic Text-to-SQL evaluation.
+
+Takes a natural language query, a corresponding LLM-generated SQL query, and a table schema + (optional) metadata.
+Determines if the LLM-generated SQL query is valid and works for the natural language query.
+"""
+from judgeval.scorers import ClassifierScorer
+from judgeval import JudgmentClient
+from judgeval.scorers.judgeval_scorers.classifiers.text2sql.text2sql_scorer import Text2SQLScorer
+
+judgment_client = JudgmentClient()
+
+print(judgment_client.push_classifier_scorer(Text2SQLScorer, slug="text2sql-eric-linkd"))
+print(judgment_client.fetch_classifier_scorer("text2sql-eric-linkd"))

judgeval-0.0.23/src/demo/custom_example_demo/qodo_example.py

@@ -0,0 +1,39 @@
+from judgeval.data import CustomExample
+from pydantic import field_validator
+
+class QodoExample(CustomExample):
+    code: str
+    original_code: str
+    
+    def __init__(self, **data):
+        super().__init__(**data)
+        
+    @field_validator('code', 'original_code', mode='before')
+    @classmethod
+    def validate_code(cls, v):
+        if v is not None and not isinstance(v, str):
+            raise ValueError(f"Code must be a string or None but got {v} of type {type(v)}")
+        return v
+    
+    def to_dict(self):
+        return {
+            "code": self.code,
+            "original_code": self.original_code,
+            **super().to_dict()
+        }
+    
+    def model_dump(self, **kwargs):
+        """
+        Custom serialization that handles special cases for fields that might fail standard serialization.
+        """
+        data = super().model_dump(**kwargs)
+        
+        # Do any additional serialization here
+        data["code"] = self.code
+        data["original_code"] = self.original_code
+        
+        return data
+
+
+
+

judgeval-0.0.23/src/demo/custom_example_demo/test.py

@@ -0,0 +1,16 @@
+from judgeval.data import CustomExample
+from judgeval import JudgmentClient
+from qodo_example import QodoExample
+
+custom_example = CustomExample(
+    code="print('Hello, world!')",
+    original_code="print('Hello, world!')",
+)
+
+qodo_example = QodoExample(
+    code="print('Hello, world!')",
+    original_code="print('Hello, world!')",
+)
+
+print(f"{custom_example=}")
+print(f"{qodo_example=}")

{judgeval-0.0.21 → judgeval-0.0.23}/src/judgeval/data/__init__.py

@@ -2,8 +2,7 @@ from judgeval.data.example import Example, ExampleParams
 from judgeval.data.api_example import ProcessExample, create_process_example
 from judgeval.data.scorer_data import ScorerData, create_scorer_data
 from judgeval.data.result import ScoringResult, generate_scoring_result
-from judgeval.data.ground_truth import GroundTruthExample
-
+from judgeval.data.custom_example import CustomExample
 __all__ = [
     "Example",
     "ExampleParams",
@@ -13,5 +12,5 @@ __all__ = [
     "create_scorer_data",
     "ScoringResult",
     "generate_scoring_result",
-    "GroundTruthExample",
+    "CustomExample",
 ]

judgeval-0.0.23/src/judgeval/data/custom_example.py

@@ -0,0 +1,98 @@
+from pydantic import BaseModel, Field, field_validator
+from typing import Optional, Dict, Any
+from uuid import uuid4
+from datetime import datetime
+import json
+import warnings
+
+# Brainstorming what are the requirements for the fields?
+class CustomExample(BaseModel):
+    name: Optional[str] = None
+    additional_metadata: Optional[Dict[str, Any]] = None
+    example_id: str = Field(default_factory=lambda: str(uuid4()))
+    example_index: Optional[int] = None
+    timestamp: Optional[str] = None
+    trace_id: Optional[str] = None
+    
+    model_config = {
+        "extra": "allow",  # Allow extra fields with any types
+    }
+    
+    def __init__(self, **data):
+        if 'example_id' not in data:
+            data['example_id'] = str(uuid4())
+        # Set timestamp if not provided
+        if 'timestamp' not in data:
+            data['timestamp'] = datetime.now().isoformat()
+        super().__init__(**data)
+        
+    @field_validator('additional_metadata', mode='before')
+    @classmethod
+    def validate_additional_metadata(cls, v):
+        if v is not None and not isinstance(v, dict):
+            raise ValueError(f"Additional metadata must be a dictionary or None but got {v} of type {type(v)}")
+        return v
+    
+    @field_validator('example_index', mode='before')
+    @classmethod
+    def validate_example_index(cls, v):
+        if v is not None and not isinstance(v, int):
+            raise ValueError(f"Example index must be an integer or None but got {v} of type {type(v)}")
+        return v
+    
+    @field_validator('timestamp', mode='before')
+    @classmethod
+    def validate_timestamp(cls, v):
+        if v is not None and not isinstance(v, str):
+            raise ValueError(f"Timestamp must be a string or None but got {v} of type {type(v)}")
+        return v
+    
+    @field_validator('trace_id', mode='before')
+    @classmethod
+    def validate_trace_id(cls, v):
+        if v is not None and not isinstance(v, str):
+            raise ValueError(f"Trace ID must be a string or None but got {v} of type {type(v)}")
+        return v
+        
+    def to_dict(self):
+        return self.model_dump()
+    
+    def __str__(self):
+        return str(self.model_dump())
+    
+    def model_dump(self, **kwargs):
+        """
+        Custom serialization that handles special cases for fields that might fail standard serialization.
+        """
+        data = super().model_dump(**kwargs)
+        
+        # Get all fields including custom ones
+        all_fields = self.__dict__
+        
+        for field_name, value in all_fields.items():
+            try:
+                # Check if the field has its own serialization method
+                if hasattr(value, 'to_dict'):
+                    data[field_name] = value.to_dict()
+                elif hasattr(value, 'model_dump'):
+                    data[field_name] = value.model_dump()
+                # Field is already in data from super().model_dump()
+                elif field_name in data:
+                    continue
+                else:
+                    # Try standard JSON serialization
+                    json.dumps(value)
+                    data[field_name] = value
+            except (TypeError, OverflowError, ValueError):
+                # Handle non-serializable objects
+                try:
+                    # Try converting to string
+                    data[field_name] = str(value)
+                except Exception as _:
+                    # If all else fails, store as None and optionally warn
+                    warnings.warn(f"Could not serialize field {field_name}, setting to None")
+                    data[field_name] = None
+        
+        return data
+    
+    

{judgeval-0.0.21 → judgeval-0.0.23}/src/judgeval/data/datasets/dataset.py

@@ -7,12 +7,11 @@ import yaml
 from dataclasses import dataclass, field
 from typing import List, Union, Literal
 
-from judgeval.data import Example, GroundTruthExample
+from judgeval.data import Example
 from judgeval.common.logger import debug, error, warning, info
 
 @dataclass
 class EvalDataset:
-    ground_truths: List[GroundTruthExample]
     examples: List[Example]
     _alias: Union[str, None] = field(default=None)
     _id: Union[str, None] = field(default=None)
@@ -21,13 +20,11 @@ class EvalDataset:
     def __init__(self, 
                  judgment_api_key: str = os.getenv("JUDGMENT_API_KEY"),  
                  organization_id: str = os.getenv("JUDGMENT_ORG_ID"),
-                 ground_truths: List[GroundTruthExample] = [], 
                  examples: List[Example] = [],
                  ):
-        debug(f"Initializing EvalDataset with {len(ground_truths)} ground truths and {len(examples)} examples")
+        debug(f"Initializing EvalDataset with {len(examples)} examples")
         if not judgment_api_key:
             warning("No judgment_api_key provided")
-        self.ground_truths = ground_truths
         self.examples = examples
         self._alias = None
         self._id = None
@@ -37,38 +34,13 @@ class EvalDataset:
     def add_from_json(self, file_path: str) -> None:
         debug(f"Loading dataset from JSON file: {file_path}")
         """
-        Adds examples and ground truths from a JSON file.
+        Adds examples from a JSON file.
 
-        The format of the JSON file is expected to be a dictionary with two keys: "examples" and "ground_truths". 
-        The value of each key is a list of dictionaries, where each dictionary represents an example or ground truth.
+        The format of the JSON file is expected to be a dictionary with one key: "examples". 
+        The value of the key is a list of dictionaries, where each dictionary represents an example.
 
         The JSON file is expected to have the following format:
         {
-            "ground_truths": [
-                {
-                    "input": "test input",
-                    "actual_output": null,
-                    "expected_output": "expected output",
-                    "context": [
-                    "context1"
-                ],
-                "retrieval_context": [
-                    "retrieval1"
-                ],
-                "additional_metadata": {
-                    "key": "value"
-                },
-                "comments": "test comment",
-                "tools_called": [
-                    "tool1"
-                ],
-                "expected_tools": [
-                    "tool1"
-                ],
-                "source_file": "test.py",
-                "trace_id": "094121"
-            }
-        ],
         "examples": [
             {
                 "input": "test input",
@@ -103,7 +75,6 @@ class EvalDataset:
             with open(file_path, "r") as file:
                 payload = json.load(file)
                 examples = payload.get("examples", [])
-                ground_truths = payload.get("ground_truths", [])
         except FileNotFoundError:
             error(f"JSON file not found: {file_path}")
             raise FileNotFoundError(f"The file {file_path} was not found.")
@@ -111,21 +82,17 @@ class EvalDataset:
             error(f"Invalid JSON file: {file_path}")
             raise ValueError(f"The file {file_path} is not a valid JSON file.")
 
-        info(f"Added {len(examples)} examples and {len(ground_truths)} ground truths from JSON")
+        info(f"Added {len(examples)} examples from JSON")
         new_examples = [Example(**e) for e in examples]
         for e in new_examples:
             self.add_example(e)
-
-        new_ground_truths = [GroundTruthExample(**g) for g in ground_truths]
-        for g in new_ground_truths:
-            self.add_ground_truth(g)
   
     def add_from_csv(
         self, 
         file_path: str,
         ) -> None:
         """
-        Add Examples and GroundTruthExamples from a CSV file.
+        Add Examples from a CSV file.
         """
         try:
             import pandas as pd
@@ -144,14 +111,14 @@ class EvalDataset:
         "expected_tools", "name", "comments", "source_file", "example", \
         "trace_id"
 
-        We want to collect the examples and ground truths separately which can
+        We want to collect the examples separately which can
         be determined by the "example" column. If the value is True, then it is an
-        example, otherwise it is a ground truth.
+        example
 
         We also assume that if there are multiple retrieval contexts or contexts, they are separated by semicolons.
         This can be adjusted using the `context_delimiter` and `retrieval_context_delimiter` parameters.
         """
-        examples, ground_truths = [], []
+        examples = []
 
         for _, row in df.iterrows():
             data = {
@@ -174,49 +141,20 @@ class EvalDataset:
                     examples.append(e)
                 else:
                     raise ValueError("Every example must have an 'input' and 'actual_output' field.")
-            else:
-                # GroundTruthExample has `comments` and `source_file` fields
-                data["comments"] = row["comments"] if pd.notna(row["comments"]) else None
-                data["source_file"] = row["source_file"] if pd.notna(row["source_file"]) else None
-                # every GroundTruthExample has `input` field
-                if data["input"] is not None:
-                    g = GroundTruthExample(**data)
-                    ground_truths.append(g)
-                else:
-                    raise ValueError("Every ground truth must have an 'input' field.")
+           
 
         for e in examples:
             self.add_example(e)
 
-        for g in ground_truths:
-            self.add_ground_truth(g)
-
     def add_from_yaml(self, file_path: str) -> None:
         debug(f"Loading dataset from YAML file: {file_path}")
         """
-        Adds examples and ground truths from a YAML file.
+        Adds examples from a YAML file.
 
-        The format of the YAML file is expected to be a dictionary with two keys: "examples" and "ground_truths". 
-        The value of each key is a list of dictionaries, where each dictionary represents an example or ground truth.
+        The format of the YAML file is expected to be a dictionary with one key: "examples". 
+        The value of the key is a list of dictionaries, where each dictionary represents an example.
 
         The YAML file is expected to have the following format:
-        ground_truths:
-          - input: "test input"
-            actual_output: null
-            expected_output: "expected output"
-            context:
-              - "context1"
-            retrieval_context:
-              - "retrieval1"
-            additional_metadata:
-              key: "value"
-            comments: "test comment"
-            tools_called:
-              - "tool1"
-            expected_tools:
-              - "tool1"
-            source_file: "test.py"
-            trace_id: "094121"
         examples:
           - input: "test input"
             actual_output: "test output"
@@ -244,7 +182,6 @@ class EvalDataset:
                 if payload is None:
                     raise ValueError("The YAML file is empty.")
                 examples = payload.get("examples", [])
-                ground_truths = payload.get("ground_truths", [])
         except FileNotFoundError:
             error(f"YAML file not found: {file_path}")
             raise FileNotFoundError(f"The file {file_path} was not found.")
@@ -252,25 +189,18 @@ class EvalDataset:
             error(f"Invalid YAML file: {file_path}")
             raise ValueError(f"The file {file_path} is not a valid YAML file.")
 
-        info(f"Added {len(examples)} examples and {len(ground_truths)} ground truths from YAML")
+        info(f"Added {len(examples)} examples from YAML")
         new_examples = [Example(**e) for e in examples]
         for e in new_examples:
             self.add_example(e)
 
-        new_ground_truths = [GroundTruthExample(**g) for g in ground_truths]
-        for g in new_ground_truths:
-            self.add_ground_truth(g)
-
     def add_example(self, e: Example) -> None:
         self.examples = self.examples + [e]
         # TODO if we need to add rank, then we need to do it here
-
-    def add_ground_truth(self, g: GroundTruthExample) -> None:
-        self.ground_truths = self.ground_truths + [g]
     
     def save_as(self, file_type: Literal["json", "csv", "yaml"], dir_path: str, save_name: str = None) -> None:
         """
-        Saves the dataset as a file. Save both the ground truths and examples.
+        Saves the dataset as a file. Save only the examples.
 
         Args:
             file_type (Literal["json", "csv"]): The file type to save the dataset as.
@@ -285,7 +215,6 @@ class EvalDataset:
             with open(complete_path, "w") as file:
                 json.dump(
                     {
-                        "ground_truths": [g.to_dict() for g in self.ground_truths],
                         "examples": [e.to_dict() for e in self.examples],
                     },
                     file,
@@ -319,24 +248,7 @@ class EvalDataset:
                         ]
                     )
                 
-                for g in self.ground_truths:
-                    writer.writerow(
-                        [
-                            g.input,
-                            g.actual_output,
-                            g.expected_output,
-                            ";".join(g.context),
-                            ";".join(g.retrieval_context),
-                            g.additional_metadata,
-                            ";".join(g.tools_called),
-                            ";".join(g.expected_tools),
-                            None,  # GroundTruthExample does not have name
-                            g.comments,
-                            g.source_file,
-                            False,  # Adding a GroundTruthExample, not an Example
-                            g.trace_id
-                        ]
-                    )
+                
         elif file_type == "yaml":
             with open(complete_path, "w") as file:
                 yaml_data = {
@@ -358,24 +270,6 @@ class EvalDataset:
                         }
                         for e in self.examples
                     ],
-                    "ground_truths": [
-                        {
-                            "input": g.input,
-                            "actual_output": g.actual_output,
-                            "expected_output": g.expected_output,
-                            "context": g.context,
-                            "retrieval_context": g.retrieval_context,
-                            "additional_metadata": g.additional_metadata,
-                            "tools_called": g.tools_called,
-                            "expected_tools": g.expected_tools,
-                            "name": None,  # GroundTruthExample does not have name
-                            "comments": g.comments,
-                            "source_file": g.source_file,
-                            "example": False,  # Adding a GroundTruthExample, not an Example
-                            "trace_id": g.trace_id
-                        }
-                        for g in self.ground_truths
-                    ]
                 }
                 yaml.dump(yaml_data, file, default_flow_style=False)
         else:
@@ -391,7 +285,6 @@ class EvalDataset:
     def __str__(self):
         return (
             f"{self.__class__.__name__}("
-            f"ground_truths={self.ground_truths}, "
             f"examples={self.examples}, "
             f"_alias={self._alias}, "
             f"_id={self._id}"

{judgeval-0.0.21 → judgeval-0.0.23}/src/judgeval/data/datasets/eval_dataset_client.py

@@ -11,7 +11,7 @@ from judgeval.constants import (
     JUDGMENT_DATASETS_EDIT_API_URL,
     JUDGMENT_DATASETS_EXPORT_JSONL_API_URL
 )
-from judgeval.data import Example, GroundTruthExample
+from judgeval.data import Example
 from judgeval.data.datasets import EvalDataset
 
 
@@ -35,7 +35,6 @@ class EvalDatasetClient:
         Mock request:
         dataset = {
             "alias": alias,
-            "ground_truths": [...],
             "examples": [...],
             "overwrite": overwrite
         } ==>
@@ -55,7 +54,6 @@ class EvalDatasetClient:
             )
             content = {
                     "alias": alias,
-                    "ground_truths": [g.to_dict() for g in dataset.ground_truths],
                     "examples": [e.to_dict() for e in dataset.examples],
                     "overwrite": overwrite,
                 }
@@ -102,7 +100,6 @@ class EvalDatasetClient:
         } 
         ==>
         {
-            "ground_truths": [...],
             "examples": [...],
             "_alias": alias,
             "_id": "..."  # ID of the dataset
@@ -142,7 +139,6 @@ class EvalDatasetClient:
 
                 info(f"Successfully pulled dataset with alias '{alias}'")
                 payload = response.json()
-                dataset.ground_truths = [GroundTruthExample(**g) for g in payload.get("ground_truths", [])]
                 dataset.examples = [Example(**e) for e in payload.get("examples", [])]
                 dataset._alias = payload.get("_alias")
                 dataset._id = payload.get("_id")
@@ -164,8 +160,8 @@ class EvalDatasetClient:
         } 
         ==>
         {
-            "test_dataset_1": {"examples_count": len(dataset1.examples), "ground_truths_count": len(dataset1.ground_truths)},
-            "test_dataset_2": {"examples_count": len(dataset2.examples), "ground_truths_count": len(dataset2.ground_truths)},
+            "test_dataset_1": {"examples_count": len(dataset1.examples)},
+            "test_dataset_2": {"examples_count": len(dataset2.examples)},
             ...
         }
         """
@@ -209,15 +205,14 @@ class EvalDatasetClient:
 
                 return payload
 
-    def edit_dataset(self, alias: str, examples: List[Example], ground_truths: List[GroundTruthExample]) -> bool:
+    def edit_dataset(self, alias: str, examples: List[Example]) -> bool:
         """
-        Edits the dataset on Judgment platform by adding new examples and ground truths
+        Edits the dataset on Judgment platform by adding new examples
 
         Mock request:
         {
             "alias": alias,
             "examples": [...],
-            "ground_truths": [...],
             "judgment_api_key": self.judgment_api_key
         }
         """
@@ -234,7 +229,6 @@ class EvalDatasetClient:
             content = {
                 "alias": alias,
                 "examples": [e.to_dict() for e in examples],
-                "ground_truths": [g.to_dict() for g in ground_truths],
             }
 
             try:

{judgeval-0.0.21 → judgeval-0.0.23}/src/judgeval/judgment_client.py

@@ -10,7 +10,6 @@ from judgeval.data.datasets import EvalDataset, EvalDatasetClient
 from judgeval.data import (
     ScoringResult, 
     Example,
-    GroundTruthExample
 )
 from judgeval.scorers import (
     APIJudgmentScorer, 
@@ -283,11 +282,11 @@ class JudgmentClient:
         """
         return self.eval_dataset_client.pull_all_user_dataset_stats()
     
-    def edit_dataset(self, alias: str, examples: List[Example], ground_truths: List[GroundTruthExample]) -> bool:
+    def edit_dataset(self, alias: str, examples: List[Example]) -> bool:
         """
-        Edits the dataset on Judgment platform by adding new examples and ground truths
+        Edits the dataset on Judgment platform by adding new examples
         """
-        return self.eval_dataset_client.edit_dataset(alias, examples, ground_truths)
+        return self.eval_dataset_client.edit_dataset(alias, examples)
     
     # Maybe add option where you can pass in the EvaluationRun object and it will pull the eval results from the backend
     def pull_eval(self, project_name: str, eval_run_name: str) -> List[Dict[str, Union[str, List[ScoringResult]]]]:

{judgeval-0.0.21 → judgeval-0.0.23}/src/judgeval/scorers/judgeval_scorer.py

@@ -116,8 +116,8 @@ class JudgevalScorer:
         For unit testing, determines whether the test case passes or fails
         """
         warning("Attempting to call unimplemented success_check method")
-        error("success_check method not implemented")
-        raise NotImplementedError("You must implement the `passes` method in your custom scorer")
+        error("_success_check method not implemented")
+        raise NotImplementedError("You must implement the `_success_check` method in your custom scorer")
 
     def __str__(self):
         debug("Converting JudgevalScorer instance to string representation")