judgeval 0.0.11__py3-none-any.whl → 0.22.2__py3-none-any.whl

judgeval/data/api_example.py

@@ -1,111 +0,0 @@
-from typing import List, Optional, Dict, Any, Union
-from pydantic import BaseModel, Field, ConfigDict, model_validator
-
-from judgeval.data.example import Example
-from judgeval.data.scorer_data import ScorerData
-from judgeval.common.logger import debug, error
-
-class ProcessExample(BaseModel):
-    """
-    ProcessExample is an `Example` object that contains intermediate information 
-    about an undergoing evaluation on the original `Example`. It is used purely for
-    internal operations and keeping track of the evaluation process.
-    """
-    name: str
-    input: Optional[str] = None
-    actual_output: Optional[str] = None
-    expected_output: Optional[str] = None
-    context: Optional[list] = None
-    retrieval_context: Optional[list] = None
-    tools_called: Optional[list] = None
-    expected_tools: Optional[list] = None
-
-    # make these optional, not all test cases in a conversation will be evaluated
-    success: Optional[bool] = None
-    scorers_data: Optional[List[ScorerData]] = None
-    run_duration: Optional[float] = None 
-    evaluation_cost: Optional[float] = None
-
-    order: Optional[int] =  None
-    # These should map 1 to 1 from golden
-    additional_metadata: Optional[Dict] = None
-    comments: Optional[str] = None
-    trace_id: Optional[str] = None
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-
-    def update_scorer_data(self, scorer_data: ScorerData):
-        """
-        Updates scorer data field of test case after the scorers have been
-        evaluated on this test case.
-        """
-        debug(f"Updating scorer data for example '{self.name}' with scorer: {scorer_data}")
-        # self.scorers_data is a list of ScorerData objects that contain the 
-        # evaluation results of each scorer on this test case
-        if self.scorers_data is None:
-            self.scorers_data = [scorer_data]
-        else:
-            self.scorers_data.append(scorer_data)
-
-        if self.success is None:
-            # self.success will be None when it is a message
-            # in that case we will be setting success for the first time
-            self.success = scorer_data.success
-        else:
-            if scorer_data.success is False:
-                debug(f"Example '{self.name}' marked as failed due to scorer: {scorer_data}")
-                self.success = False
-
-    def update_run_duration(self, run_duration: float):
-        self.run_duration = run_duration
-
-    @model_validator(mode="before")
-    def check_input(cls, values: Dict[str, Any]):
-        input = values.get("input")
-        actual_output = values.get("actual_output")
-
-        if (input is None or actual_output is None):
-            error(f"Validation error: Required fields missing. input={input}, actual_output={actual_output}")
-            raise ValueError(
-                "'input' and 'actual_output' must be provided."
-            )
-
-        return values
-    
-
-def create_process_example(
-    example: Example,
-) -> ProcessExample:
-    """
-    When an LLM Test Case is executed, we track its progress using an ProcessExample.
-
-    This will track things like the success of the test case, as well as the metadata (such as verdicts and claims in Faithfulness).
-    """
-    success = True
-    if example.name is not None:
-        name = example.name
-    else:
-        name = "Test Case Placeholder"
-        debug(f"No name provided for example, using default name: {name}")
-    order = None
-    scorers_data = []
-
-    debug(f"Creating ProcessExample for: {name}")
-    process_ex = ProcessExample(
-        name=name,
-        input=example.input,
-        actual_output=example.actual_output,
-        expected_output=example.expected_output,
-        context=example.context,
-        retrieval_context=example.retrieval_context,
-        tools_called=example.tools_called,
-        expected_tools=example.expected_tools,
-        success=success,
-        scorers_data=scorers_data,
-        run_duration=None,
-        evaluation_cost=None,
-        order=order,
-        additional_metadata=example.additional_metadata,
-        trace_id=example.trace_id
-    )
-    return process_ex
-

judgeval/data/datasets/__init__.py

@@ -1,5 +0,0 @@
-from judgeval.data.datasets.dataset import EvalDataset
-from judgeval.data.datasets.ground_truth import GroundTruthExample
-from judgeval.data.datasets.eval_dataset_client import EvalDatasetClient
-
-__all__ = ["EvalDataset", "EvalDatasetClient", "GroundTruthExample"]

judgeval/data/datasets/dataset.py

@@ -1,286 +0,0 @@
-import ast
-import csv
-import datetime 
-import json
-from dataclasses import dataclass, field
-import os
-from typing import List, Optional, Union, Literal
-
-from judgeval.data.datasets.ground_truth import GroundTruthExample
-from judgeval.data import Example
-from judgeval.common.logger import debug, error, warning, info
-
-@dataclass
-class EvalDataset:
-    ground_truths: List[GroundTruthExample]
-    examples: List[Example]
-    _alias: Union[str, None] = field(default=None)
-    _id: Union[str, None] = field(default=None)
-    judgment_api_key: str = field(default="")
-
-    def __init__(self, 
-                 judgment_api_key: str = os.getenv("JUDGMENT_API_KEY"),  
-                 ground_truths: List[GroundTruthExample] = [], 
-                 examples: List[Example] = [],
-                 ):
-        debug(f"Initializing EvalDataset with {len(ground_truths)} ground truths and {len(examples)} examples")
-        if not judgment_api_key:
-            warning("No judgment_api_key provided")
-        self.ground_truths = ground_truths
-        self.examples = examples
-        self._alias = None
-        self._id = None
-        self.judgment_api_key = judgment_api_key
-        
-
-    def add_from_json(self, file_path: str) -> None:
-        debug(f"Loading dataset from JSON file: {file_path}")
-        """
-        Adds examples and ground truths from a JSON file.
-
-        The format of the JSON file is expected to be a dictionary with two keys: "examples" and "ground_truths". 
-        The value of each key is a list of dictionaries, where each dictionary represents an example or ground truth.
-
-        The JSON file is expected to have the following format:
-        {
-            "ground_truths": [
-                {
-                    "input": "test input",
-                    "actual_output": null,
-                    "expected_output": "expected output",
-                    "context": [
-                    "context1"
-                ],
-                "retrieval_context": [
-                    "retrieval1"
-                ],
-                "additional_metadata": {
-                    "key": "value"
-                },
-                "comments": "test comment",
-                "tools_called": [
-                    "tool1"
-                ],
-                "expected_tools": [
-                    "tool1"
-                ],
-                "source_file": "test.py",
-                "trace_id": "094121"
-            }
-        ],
-        "examples": [
-            {
-                "input": "test input",
-                "actual_output": "test output",
-                "expected_output": "expected output",
-                "context": [
-                    "context1",
-                    "context2"
-                ],
-                "retrieval_context": [
-                    "retrieval1"
-                ],
-                "additional_metadata": {
-                    "key": "value"
-                },
-                "tools_called": [
-                    "tool1"
-                ],
-                "expected_tools": [
-                    "tool1",
-                    "tool2"
-                ],
-                "name": "test example",
-                "example_id": null,
-                "timestamp": "20241230_160117",
-                "trace_id": "123"
-            }
-            ]
-        }
-        """
-        try:
-            with open(file_path, "r") as file:
-                payload = json.load(file)
-                examples = payload.get("examples", [])
-                ground_truths = payload.get("ground_truths", [])
-        except FileNotFoundError:
-            error(f"JSON file not found: {file_path}")
-            raise FileNotFoundError(f"The file {file_path} was not found.")
-        except json.JSONDecodeError:
-            error(f"Invalid JSON file: {file_path}")
-            raise ValueError(f"The file {file_path} is not a valid JSON file.")
-
-        info(f"Added {len(examples)} examples and {len(ground_truths)} ground truths from JSON")
-        new_examples = [Example(**e) for e in examples]
-        for e in new_examples:
-            self.add_example(e)
-
-        new_ground_truths = [GroundTruthExample(**g) for g in ground_truths]
-        for g in new_ground_truths:
-            self.add_ground_truth(g)
-  
-    def add_from_csv(
-        self, 
-        file_path: str,
-        ) -> None:
-        """
-        Add Examples and GroundTruthExamples from a CSV file.
-        """
-        try:
-            import pandas as pd
-        except ModuleNotFoundError:
-            raise ModuleNotFoundError(
-                "Please install pandas to use this method. 'pip install pandas'"
-            )
-        
-        # Pandas naturally reads numbers in data files as ints, not strings (can lead to unexpected behavior)
-        df = pd.read_csv(file_path, dtype={'trace_id': str})
-        """
-        Expect the CSV to have headers
-
-        "input", "actual_output", "expected_output", "context", \
-        "retrieval_context", "additional_metadata", "tools_called", \
-        "expected_tools", "name", "comments", "source_file", "example", \
-        "trace_id"
-
-        We want to collect the examples and ground truths separately which can
-        be determined by the "example" column. If the value is True, then it is an
-        example, otherwise it is a ground truth.
-
-        We also assume that if there are multiple retrieval contexts or contexts, they are separated by semicolons.
-        This can be adjusted using the `context_delimiter` and `retrieval_context_delimiter` parameters.
-        """
-        examples, ground_truths = [], []
-
-        for _, row in df.iterrows():
-            data = {
-                "input": row["input"],
-                "actual_output": row["actual_output"] if pd.notna(row["actual_output"]) else None,
-                "expected_output": row["expected_output"] if pd.notna(row["expected_output"]) else None,
-                "context": row["context"].split(";") if pd.notna(row["context"]) else [],
-                "retrieval_context": row["retrieval_context"].split(";") if pd.notna(row["retrieval_context"]) else [],
-                "additional_metadata": ast.literal_eval(row["additional_metadata"]) if pd.notna(row["additional_metadata"]) else dict(),
-                "tools_called": row["tools_called"].split(";") if pd.notna(row["tools_called"]) else [],
-                "expected_tools": row["expected_tools"].split(";") if pd.notna(row["expected_tools"]) else [],
-                "trace_id": row["trace_id"] if pd.notna(row["trace_id"]) else None
-            }
-            if row["example"]:
-                data["name"] = row["name"] if pd.notna(row["name"]) else None
-                # every Example has `input` and `actual_output` fields
-                if data["input"] is not None and data["actual_output"] is not None:
-                    e = Example(**data)
-                    examples.append(e)
-                else:
-                    raise ValueError("Every example must have an 'input' and 'actual_output' field.")
-            else:
-                # GroundTruthExample has `comments` and `source_file` fields
-                data["comments"] = row["comments"] if pd.notna(row["comments"]) else None
-                data["source_file"] = row["source_file"] if pd.notna(row["source_file"]) else None
-                # every GroundTruthExample has `input` field
-                if data["input"] is not None:
-                    g = GroundTruthExample(**data)
-                    ground_truths.append(g)
-                else:
-                    raise ValueError("Every ground truth must have an 'input' field.")
-
-        for e in examples:
-            self.add_example(e)
-
-        for g in ground_truths:
-            self.add_ground_truth(g)
-
-    def add_example(self, e: Example) -> None:
-        self.examples = self.examples + [e]
-        # TODO if we need to add rank, then we need to do it here
-
-    def add_ground_truth(self, g: GroundTruthExample) -> None:
-        self.ground_truths = self.ground_truths + [g]
-    
-    def save_as(self, file_type: Literal["json", "csv"], dir_path: str, save_name: str = None) -> None:
-        """
-        Saves the dataset as a file. Save both the ground truths and examples.
-
-        Args:
-            file_type (Literal["json", "csv"]): The file type to save the dataset as.
-            dir_path (str): The directory path to save the file to.
-            save_name (str, optional): The name of the file to save. Defaults to None.
-        """
-        if not os.path.exists(dir_path):
-            os.makedirs(dir_path)
-        file_name = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") if save_name is None else save_name
-        complete_path = os.path.join(dir_path, f"{file_name}.{file_type}")
-        if file_type == "json":
-            with open(complete_path, "w") as file:
-                json.dump(
-                    {
-                        "ground_truths": [g.to_dict() for g in self.ground_truths],
-                        "examples": [e.to_dict() for e in self.examples],
-                    },
-                    file,
-                    indent=4,
-                )
-        elif file_type == "csv":
-            with open(complete_path, "w", newline="") as file:
-                writer = csv.writer(file)
-                writer.writerow([
-                    "input", "actual_output", "expected_output", "context", \
-                    "retrieval_context", "additional_metadata", "tools_called", \
-                    "expected_tools", "name", "comments", "source_file", "example", \
-                    "trace_id"
-                ])
-                for e in self.examples:
-                    writer.writerow(
-                        [
-                            e.input,
-                            e.actual_output,
-                            e.expected_output,
-                            ";".join(e.context),
-                            ";".join(e.retrieval_context),
-                            e.additional_metadata,
-                            ";".join(e.tools_called),
-                            ";".join(e.expected_tools),
-                            e.name,
-                            None,  # Example does not have comments
-                            None,  # Example does not have source file
-                            True,  # Adding an Example
-                            e.trace_id
-                        ]
-                    )
-                
-                for g in self.ground_truths:
-                    writer.writerow(
-                        [
-                            g.input,
-                            g.actual_output,
-                            g.expected_output,
-                            ";".join(g.context),
-                            ";".join(g.retrieval_context),
-                            g.additional_metadata,
-                            ";".join(g.tools_called),
-                            ";".join(g.expected_tools),
-                            None,  # GroundTruthExample does not have name
-                            g.comments,
-                            g.source_file,
-                            False,  # Adding a GroundTruthExample, not an Example
-                            g.trace_id
-                        ]
-                    )
-        else:
-            ACCEPTABLE_FILE_TYPES = ["json", "csv"]
-            raise TypeError(f"Invalid file type: {file_type}. Please choose from {ACCEPTABLE_FILE_TYPES}")
-        
-    def __iter__(self):
-        return iter(self.examples)
-    
-    def __len__(self):
-        return len(self.examples)
-    
-    def __str__(self):
-        return (
-            f"{self.__class__.__name__}("
-            f"ground_truths={self.ground_truths}, "
-            f"examples={self.examples}, "
-            f"_alias={self._alias}, "
-            f"_id={self._id}"
-            f")"
-        )

judgeval/data/datasets/eval_dataset_client.py

@@ -1,193 +0,0 @@
-
-from typing import Optional
-import requests
-from rich.progress import Progress, SpinnerColumn, TextColumn
-
-from judgeval.common.logger import debug, error, warning, info
-from judgeval.constants import (
-    JUDGMENT_DATASETS_PUSH_API_URL,
-    JUDGMENT_DATASETS_PULL_API_URL, 
-    JUDGMENT_DATASETS_PULL_ALL_API_URL
-)
-from judgeval.data import Example
-from judgeval.data.datasets import EvalDataset
-from judgeval.data.datasets.ground_truth import GroundTruthExample
-
-
-
-
-class EvalDatasetClient:
-    def __init__(self, judgment_api_key: str):
-        self.judgment_api_key = judgment_api_key
-
-    def create_dataset(self) -> EvalDataset:
-        return EvalDataset(judgment_api_key=self.judgment_api_key)
-    
-    def push(self, dataset: EvalDataset, alias: str,overwrite: Optional[bool] = False) -> bool:
-        debug(f"Pushing dataset with alias '{alias}' (overwrite={overwrite})")
-        if overwrite:
-            warning(f"Overwrite enabled for alias '{alias}'")
-        """
-        Pushes the dataset to Judgment platform
-
-        Mock request:
-        dataset = {
-            "alias": alias,
-            "ground_truths": [...],
-            "examples": [...],
-            "overwrite": overwrite
-        } ==>
-        {
-            "_alias": alias,
-            "_id": "..."  # ID of the dataset
-        }
-        """
-        with Progress(
-            SpinnerColumn(style="rgb(106,0,255)"),
-            TextColumn("[progress.description]{task.description}"),
-            transient=False,
-        ) as progress:
-            task_id = progress.add_task(
-                f"Pushing [rgb(106,0,255)]'{alias}' to Judgment...",
-                total=100,
-            )
-            content = {
-                    "alias": alias,
-                    "ground_truths": [g.to_dict() for g in dataset.ground_truths],
-                    "examples": [e.to_dict() for e in dataset.examples],
-                    "overwrite": overwrite,
-                    "judgment_api_key": dataset.judgment_api_key
-                }
-            try:
-                response = requests.post(
-                    JUDGMENT_DATASETS_PUSH_API_URL, 
-                    json=content
-                )
-                if response.status_code == 500:
-                    error(f"Server error during push: {content.get('message')}")
-                    return False
-                response.raise_for_status()
-            except requests.exceptions.HTTPError as err:
-                if response.status_code == 422:
-                    error(f"Validation error during push: {err.response.json()}")
-                else:
-                    error(f"HTTP error during push: {err}")
-            
-            info(f"Successfully pushed dataset with alias '{alias}'")
-            payload = response.json()
-            dataset._alias = payload.get("_alias")
-            dataset._id = payload.get("_id")
-            progress.update(
-                    task_id,
-                    description=f"{progress.tasks[task_id].description} [rgb(25,227,160)]Done!)",
-                )
-            return True
-        
-    def pull(self, alias: str) -> EvalDataset:
-        debug(f"Pulling dataset with alias '{alias}'")
-        """
-        Pulls the dataset from Judgment platform
-
-        Mock request:
-        {
-            "alias": alias,
-            "user_id": user_id
-        } 
-        ==>
-        {
-            "ground_truths": [...],
-            "examples": [...],
-            "_alias": alias,
-            "_id": "..."  # ID of the dataset
-        }
-        """
-        # Make a POST request to the Judgment API to get the dataset
-        dataset = self.create_dataset()
-
-        with Progress(
-                SpinnerColumn(style="rgb(106,0,255)"),
-                TextColumn("[progress.description]{task.description}"),
-                transient=False,
-            ) as progress:
-                task_id = progress.add_task(
-                    f"Pulling [rgb(106,0,255)]'{alias}'[/rgb(106,0,255)] from Judgment...",
-                    total=100,
-                )
-                request_body = {
-                    "alias": alias,
-                    "judgment_api_key": self.judgment_api_key
-                }
-
-                try:
-                    response = requests.post(
-                        JUDGMENT_DATASETS_PULL_API_URL, 
-                        json=request_body
-                    )
-                    response.raise_for_status()
-                except requests.exceptions.RequestException as e:
-                    error(f"Error pulling dataset: {str(e)}")
-                    raise
-
-                info(f"Successfully pulled dataset with alias '{alias}'")
-                payload = response.json()
-                dataset.ground_truths = [GroundTruthExample(**g) for g in payload.get("ground_truths", [])]
-                dataset.examples = [Example(**e) for e in payload.get("examples", [])]
-                dataset._alias = payload.get("_alias")
-                dataset._id = payload.get("_id")
-                progress.update(
-                    task_id,
-                    description=f"{progress.tasks[task_id].description} [rgb(25,227,160)]Done!)",
-                )
-
-                return dataset
-
-    def pull_all_user_dataset_stats(self) -> dict:
-        debug(f"Pulling user datasets stats for user_id: {self.judgment_api_key}'")
-        """
-        Pulls the user datasets stats from Judgment platform
-
-        Mock request:
-        {
-            "user_id": user_id
-        } 
-        ==>
-        {
-            "test_dataset_1": {"examples_count": len(dataset1.examples), "ground_truths_count": len(dataset1.ground_truths)},
-            "test_dataset_2": {"examples_count": len(dataset2.examples), "ground_truths_count": len(dataset2.ground_truths)},
-            ...
-        }
-        """
-        # Make a POST request to the Judgment API to get the dataset
-
-        with Progress(
-                SpinnerColumn(style="rgb(106,0,255)"),
-                TextColumn("[progress.description]{task.description}"),
-                transient=False,
-            ) as progress:
-                task_id = progress.add_task(
-                    f"Pulling [rgb(106,0,255)]' datasets'[/rgb(106,0,255)] from Judgment...",
-                    total=100,
-                )
-                request_body = {
-                    "judgment_api_key": self.judgment_api_key
-                }
-
-                try:
-                    response = requests.post(
-                        JUDGMENT_DATASETS_PULL_ALL_API_URL, 
-                        json=request_body
-                    )
-                    response.raise_for_status()
-                except requests.exceptions.RequestException as e:
-                    error(f"Error pulling dataset: {str(e)}")
-                    raise
-
-                info(f"Successfully pulled datasets for userid: {self.judgment_api_key}'")
-                payload = response.json()
-
-                progress.update(
-                    task_id,
-                    description=f"{progress.tasks[task_id].description} [rgb(25,227,160)]Done!)",
-                )
-
-                return payload

judgeval/data/datasets/ground_truth.py

@@ -1,54 +0,0 @@
-from pydantic import BaseModel
-from typing import Optional, Dict, List
-
-
-class GroundTruthExample(BaseModel):
-    """
-    GroundTruthExample is the atomic unit of a `Dataset`. It is essentially the same
-    as an `Example`, but the `actual_output` field is optional to enable users to 
-    run their workflow on the `input` field at test-time to evaluate their current 
-    workflow's performance.
-    """
-    input: str
-    actual_output: Optional[str] = None
-    expected_output: Optional[str] = None
-    context: Optional[List[str]] = None
-    retrieval_context: Optional[List[str]] = None
-    additional_metadata: Optional[Dict] = None
-    comments: Optional[str] = None
-    tools_called: Optional[List[str]] = None
-    expected_tools: Optional[List[str]] = None
-    source_file: Optional[str] = None
-    trace_id: Optional[str] = None
-
-    def to_dict(self):
-        return {
-            "input": self.input,
-            "actual_output": self.actual_output,
-            "expected_output": self.expected_output,
-            "context": self.context,
-            "retrieval_context": self.retrieval_context,
-            "additional_metadata": self.additional_metadata,
-            "comments": self.comments,
-            "tools_called": self.tools_called,
-            "expected_tools": self.expected_tools,
-            "source_file": self.source_file,
-            "trace_id": self.trace_id,
-        }
-    
-    def __str__(self):
-        return (
-            f"{self.__class__.__name__}("
-            f"input={self.input}, "
-            f"actual_output={self.actual_output}, "
-            f"expected_output={self.expected_output}, "
-            f"context={self.context}, "
-            f"retrieval_context={self.retrieval_context}, "
-            f"additional_metadata={self.additional_metadata}, "
-            f"comments={self.comments}, "
-            f"tools_called={self.tools_called}, "
-            f"expected_tools={self.expected_tools}, "
-            f"source_file={self.source_file}, "
-            f"trace_id={self.trace_id}"
-            f")"
-        )