judgeval 0.1.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

judgeval/data/datasets/__init__.py

@@ -1,4 +0,0 @@
-from judgeval.data.datasets.dataset import EvalDataset
-from judgeval.data.datasets.eval_dataset_client import EvalDatasetClient
-
-__all__ = ["EvalDataset", "EvalDatasetClient"]

judgeval/data/datasets/dataset.py

@@ -1,341 +0,0 @@
-import ast
-import csv
-import datetime
-import json
-import os
-import yaml
-from dataclasses import dataclass, field
-from typing import List, Union, Literal, Optional
-
-from judgeval.data import Example, Trace
-from judgeval.common.logger import judgeval_logger
-from judgeval.utils.file_utils import get_examples_from_yaml
-
-
-@dataclass
-class EvalDataset:
-    examples: List[Example]
-    traces: List[Trace]
-    _alias: Union[str, None] = field(default=None)
-    _id: Union[str, None] = field(default=None)
-    judgment_api_key: str = field(default="")
-    organization_id: str = field(default="")
-
-    def __init__(
-        self,
-        judgment_api_key: str = os.getenv("JUDGMENT_API_KEY", ""),
-        organization_id: str = os.getenv("JUDGMENT_ORG_ID", ""),
-        examples: Optional[List[Example]] = None,
-        traces: Optional[List[Trace]] = None,
-    ):
-        if not judgment_api_key:
-            judgeval_logger.error("No judgment_api_key provided")
-        self.examples = examples or []
-        self.traces = traces or []
-        self._alias = None
-        self._id = None
-        self.judgment_api_key = judgment_api_key
-        self.organization_id = organization_id
-
-    def add_from_json(self, file_path: str) -> None:
-        """
-        Adds examples from a JSON file.
-
-        The format of the JSON file is expected to be a dictionary with one key: "examples".
-        The value of the key is a list of dictionaries, where each dictionary represents an example.
-
-        The JSON file is expected to have the following format:
-        {
-        "examples": [
-            {
-                "input": "test input",
-                "actual_output": "test output",
-                "expected_output": "expected output",
-                "context": [
-                    "context1",
-                    "context2"
-                ],
-                "retrieval_context": [
-                    "retrieval1"
-                ],
-                "additional_metadata": {
-                    "key": "value"
-                },
-                "tools_called": [
-                    "tool1"
-                ],
-                "expected_tools": [
-                    "tool1",
-                    "tool2"
-                ],
-                "name": "test example",
-                "example_id": null,
-                "timestamp": "20241230_160117",
-                "trace_id": "123"
-            }
-            ]
-        }
-        """
-        try:
-            with open(file_path, "r") as file:
-                payload = json.load(file)
-                examples = payload.get("examples", [])
-        except FileNotFoundError:
-            judgeval_logger.error(f"JSON file not found: {file_path}")
-            raise FileNotFoundError(f"The file {file_path} was not found.")
-        except json.JSONDecodeError:
-            judgeval_logger.error(f"Invalid JSON file: {file_path}")
-            raise ValueError(f"The file {file_path} is not a valid JSON file.")
-
-        new_examples = [Example(**e) for e in examples]
-        for e in new_examples:
-            self.add_example(e)
-
-    def add_from_csv(
-        self,
-        file_path: str,
-        header_mapping: dict,
-        primary_delimiter: str = ",",
-        secondary_delimiter: str = ";",
-    ) -> None:
-        """
-        Add Examples from a CSV file.
-
-        Args:
-            file_path (str): Path to the CSV file
-            header_mapping (dict): Dictionary mapping Example headers to custom headers
-            primary_delimiter (str, optional): Main delimiter used in CSV file. Defaults to ","
-            secondary_delimiter (str, optional): Secondary delimiter for list fields. Defaults to ";"
-        """
-        try:
-            import pandas as pd
-        except ModuleNotFoundError:
-            raise ModuleNotFoundError(
-                "Please install pandas to use this method. 'pip install pandas'"
-            )
-
-        # Pandas naturally reads numbers in data files as ints, not strings (can lead to unexpected behavior)
-        df = pd.read_csv(file_path, dtype={"trace_id": str}, sep=primary_delimiter)
-        """
-        The user should pass in a dict mapping from Judgment Example headers to their custom defined headers.
-        Available headers for Example objects are as follows:
-
-        "input", "actual_output", "expected_output", "context", \
-        "retrieval_context", "additional_metadata", "tools_called", \
-        "expected_tools", "name", "comments", "source_file", "example", \
-        "trace_id"
-
-        We want to collect the examples separately which can
-        be determined by the "example" column. If the value is True, then it is an
-        example, and we expect the `input` and `actual_output` fields to be non-null.
-
-        We also assume that if there are multiple retrieval contexts, contexts, or tools called, they are separated by semicolons.
-        This can be adjusted using the `secondary_delimiter` parameter.
-        """
-        examples = []
-
-        def process_csv_row(value, header):
-            """
-            Maps a singular value in the CSV file to the appropriate type based on the header.
-            If value exists and can be split into type List[*], we will split upon the user's provided secondary delimiter.
-            """
-            # check that the CSV value is not null for entry
-            null_replacement = dict() if header == "additional_metadata" else None
-            if pd.isna(value) or value == "":
-                return null_replacement
-            try:
-                value = (
-                    ast.literal_eval(value)
-                    if header == "additional_metadata"
-                    else str(value)
-                )
-            except (ValueError, SyntaxError):
-                value = str(value)
-            if header in [
-                "context",
-                "retrieval_context",
-                "tools_called",
-                "expected_tools",
-            ]:
-                # attempt to split the value by the secondary delimiter
-                value = value.split(secondary_delimiter)
-
-            return value
-
-        for _, row in df.iterrows():
-            data = {
-                header: process_csv_row(row[header_mapping[header]], header)
-                for header in header_mapping
-            }
-            if "example" in header_mapping and row[header_mapping["example"]]:
-                if "name" in header_mapping:
-                    data["name"] = (
-                        row[header_mapping["name"]]
-                        if pd.notna(row[header_mapping["name"]])
-                        else None
-                    )
-                # every Example has `input` and `actual_output` fields
-                if data["input"] is not None and data["actual_output"] is not None:
-                    e = Example(**data)
-                    examples.append(e)
-                else:
-                    raise ValueError(
-                        "Every example must have an 'input' and 'actual_output' field."
-                    )
-
-        for e in examples:
-            self.add_example(e)
-
-    def add_from_yaml(self, file_path: str) -> None:
-        """
-        Adds examples from a YAML file.
-
-        The format of the YAML file is expected to be a dictionary with one key: "examples".
-        The value of the key is a list of dictionaries, where each dictionary represents an example.
-
-        The YAML file is expected to have the following format:
-        examples:
-          - input: "test input"
-            actual_output: "test output"
-            expected_output: "expected output"
-            context:
-              - "context1"
-              - "context2"
-            retrieval_context:
-              - "retrieval1"
-            additional_metadata:
-              key: "value"
-            tools_called:
-              - "tool1"
-            expected_tools:
-              - "tool1"
-              - "tool2"
-            name: "test example"
-            example_id: null
-            timestamp: "20241230_160117"
-            trace_id: "123"
-        """
-        examples = get_examples_from_yaml(file_path)
-
-        for e in examples:
-            self.add_example(e)
-
-    def add_example(self, e: Example) -> None:
-        self.examples.append(e)
-        # TODO if we need to add rank, then we need to do it here
-
-    def add_trace(self, t: Trace) -> None:
-        self.traces.append(t)
-
-    def save_as(
-        self,
-        file_type: Literal["json", "csv", "yaml"],
-        dir_path: str,
-        save_name: str | None = None,
-    ) -> None:
-        """
-        Saves the dataset as a file. Save only the examples.
-
-        Args:
-            file_type (Literal["json", "csv"]): The file type to save the dataset as.
-            dir_path (str): The directory path to save the file to.
-            save_name (str, optional): The name of the file to save. Defaults to None.
-        """
-        if not os.path.exists(dir_path):
-            os.makedirs(dir_path)
-        file_name = (
-            datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
-            if save_name is None
-            else save_name
-        )
-        complete_path = os.path.join(dir_path, f"{file_name}.{file_type}")
-        if file_type == "json":
-            with open(complete_path, "w") as file:
-                json.dump(
-                    {
-                        "examples": [e.to_dict() for e in self.examples],
-                    },
-                    file,
-                    indent=4,
-                )
-        elif file_type == "csv":
-            with open(complete_path, "w", newline="") as file:
-                writer = csv.writer(file)
-                writer.writerow(
-                    [
-                        "input",
-                        "actual_output",
-                        "expected_output",
-                        "context",
-                        "retrieval_context",
-                        "additional_metadata",
-                        "tools_called",
-                        "expected_tools",
-                        "name",
-                        "comments",
-                        "source_file",
-                        "example",
-                        "trace_id",
-                    ]
-                )
-                for e in self.examples:
-                    writer.writerow(
-                        [
-                            e.input,
-                            e.actual_output,
-                            e.expected_output,
-                            ";".join(e.context),
-                            ";".join(e.retrieval_context),
-                            e.additional_metadata,
-                            ";".join(e.tools_called),
-                            ";".join(e.expected_tools),
-                            e.name,
-                            None,  # Example does not have comments
-                            None,  # Example does not have source file
-                            True,  # Adding an Example
-                        ]
-                    )
-
-        elif file_type == "yaml":
-            with open(complete_path, "w") as file:
-                yaml_data = {
-                    "examples": [
-                        {
-                            "input": e.input,
-                            "actual_output": e.actual_output,
-                            "expected_output": e.expected_output,
-                            "context": e.context,
-                            "retrieval_context": e.retrieval_context,
-                            "additional_metadata": e.additional_metadata,
-                            "tools_called": e.tools_called,
-                            "expected_tools": e.expected_tools,
-                            "name": e.name,
-                            "comments": None,  # Example does not have comments
-                            "source_file": None,  # Example does not have source file
-                            "example": True,  # Adding an Example
-                        }
-                        for e in self.examples
-                    ],
-                }
-                yaml.dump(yaml_data, file, default_flow_style=False)
-        else:
-            ACCEPTABLE_FILE_TYPES = ["json", "csv", "yaml"]
-            raise TypeError(
-                f"Invalid file type: {file_type}. Please choose from {ACCEPTABLE_FILE_TYPES}"
-            )
-
-    def __iter__(self):
-        return iter(self.examples)
-
-    def __len__(self):
-        return len(self.examples)
-
-    def __str__(self):
-        return (
-            f"{self.__class__.__name__}("
-            f"examples={self.examples}, "
-            f"traces={self.traces}, "
-            f"_alias={self._alias}, "
-            f"_id={self._id}"
-            f")"
-        )

judgeval/data/datasets/eval_dataset_client.py

@@ -1,214 +0,0 @@
-from typing import Optional, List
-from rich.progress import Progress, SpinnerColumn, TextColumn
-from judgeval.common.logger import judgeval_logger
-from judgeval.common.api import JudgmentApiClient
-from judgeval.data import Example, Trace
-from judgeval.data.datasets import EvalDataset
-
-
-class EvalDatasetClient:
-    def __init__(self, judgment_api_key: str, organization_id: str):
-        self.api_client = JudgmentApiClient(judgment_api_key, organization_id)
-
-    def create_dataset(self) -> EvalDataset:
-        return EvalDataset(judgment_api_key=self.api_client.api_key)
-
-    def push(
-        self,
-        dataset: EvalDataset,
-        alias: str,
-        project_name: str,
-        overwrite: Optional[bool] = False,
-    ) -> bool:
-        if overwrite:
-            judgeval_logger.warning(f"Overwrite enabled for alias '{alias}'")
-        """
-        Pushes the dataset to Judgment platform
-
-        Mock request:
-        dataset = {
-            "alias": alias,
-            "examples": [...],
-            "overwrite": overwrite
-        } ==>
-        {
-            "_alias": alias,
-            "_id": "..."  # ID of the dataset
-        }
-        """
-        with Progress(
-            SpinnerColumn(style="rgb(106,0,255)"),
-            TextColumn("[progress.description]{task.description}"),
-            transient=False,
-        ) as progress:
-            task_id = progress.add_task(
-                f"Pushing [rgb(106,0,255)]'{alias}' to Judgment...",
-                total=100,
-            )
-            try:
-                payload = self.api_client.push_dataset(
-                    dataset_alias=alias,
-                    project_name=project_name,
-                    examples=[e.to_dict() for e in dataset.examples],
-                    traces=[t.model_dump() for t in dataset.traces],
-                    overwrite=overwrite or False,
-                )
-            except Exception as e:
-                judgeval_logger.error(f"Error during push: {e}")
-                raise
-            dataset._alias = payload.get("_alias")
-            dataset._id = payload.get("_id")
-            progress.update(
-                task_id,
-                description=f"{progress.tasks[task_id].description} [rgb(25,227,160)]Done!)",
-            )
-            return True
-
-    def append_examples(
-        self, alias: str, examples: List[Example], project_name: str
-    ) -> bool:
-        """
-        Appends the dataset to Judgment platform
-
-        Mock request:
-        dataset = {
-            "alias": alias,
-            "examples": [...],
-            "project_name": project_name
-        } ==>
-        {
-            "_alias": alias,
-            "_id": "..."  # ID of the dataset
-        }
-        """
-        with Progress(
-            SpinnerColumn(style="rgb(106,0,255)"),
-            TextColumn("[progress.description]{task.description}"),
-            transient=False,
-        ) as progress:
-            task_id = progress.add_task(
-                f"Appending [rgb(106,0,255)]'{alias}' to Judgment...",
-                total=100,
-            )
-            try:
-                self.api_client.append_examples(
-                    dataset_alias=alias,
-                    project_name=project_name,
-                    examples=[e.to_dict() for e in examples],
-                )
-            except Exception as e:
-                judgeval_logger.error(f"Error during append: {e}")
-                raise
-
-            progress.update(
-                task_id,
-                description=f"{progress.tasks[task_id].description} [rgb(25,227,160)]Done!)",
-            )
-            return True
-
-    def pull(self, alias: str, project_name: str) -> EvalDataset:
-        """
-        Pulls the dataset from Judgment platform
-
-        Mock request:
-        {
-            "alias": alias,
-            "project_name": project_name
-        }
-        ==>
-        {
-            "examples": [...],
-            "_alias": alias,
-            "_id": "..."  # ID of the dataset
-        }
-        """
-        # Make a POST request to the Judgment API to get the dataset
-        dataset = self.create_dataset()
-
-        with Progress(
-            SpinnerColumn(style="rgb(106,0,255)"),
-            TextColumn("[progress.description]{task.description}"),
-            transient=False,
-        ) as progress:
-            task_id = progress.add_task(
-                f"Pulling [rgb(106,0,255)]'{alias}'[/rgb(106,0,255)] from Judgment...",
-                total=100,
-            )
-            try:
-                payload = self.api_client.pull_dataset(
-                    dataset_alias=alias,
-                    project_name=project_name,
-                )
-            except Exception as e:
-                judgeval_logger.error(f"Error pulling dataset: {str(e)}")
-                raise
-            dataset.examples = [Example(**e) for e in payload.get("examples", [])]
-            dataset.traces = [Trace(**t) for t in payload.get("traces", [])]
-            dataset._alias = payload.get("alias")
-            dataset._id = payload.get("id")
-            progress.update(
-                task_id,
-                description=f"{progress.tasks[task_id].description} [rgb(25,227,160)]Done!)",
-            )
-
-            return dataset
-
-    def delete(self, alias: str, project_name: str) -> bool:
-        with Progress(
-            SpinnerColumn(style="rgb(106,0,255)"),
-            TextColumn("[progress.description]{task.description}"),
-            transient=False,
-        ) as progress:
-            progress.add_task(
-                f"Deleting [rgb(106,0,255)]'{alias}'[/rgb(106,0,255)] from Judgment...",
-                total=100,
-            )
-            try:
-                self.api_client.delete_dataset(
-                    dataset_alias=alias,
-                    project_name=project_name,
-                )
-            except Exception as e:
-                judgeval_logger.error(f"Error deleting dataset: {str(e)}")
-                raise
-
-            return True
-
-    def pull_project_dataset_stats(self, project_name: str) -> dict:
-        """
-        Pulls the project datasets stats from Judgment platform
-
-        Mock request:
-        {
-            "project_name": project_name
-        }
-        ==>
-        {
-            "test_dataset_1": {"examples_count": len(dataset1.examples)},
-            "test_dataset_2": {"examples_count": len(dataset2.examples)},
-            ...
-        }
-        """
-        # Make a POST request to the Judgment API to get the dataset
-
-        with Progress(
-            SpinnerColumn(style="rgb(106,0,255)"),
-            TextColumn("[progress.description]{task.description}"),
-            transient=False,
-        ) as progress:
-            task_id = progress.add_task(
-                "Pulling [rgb(106,0,255)]' datasets'[/rgb(106,0,255)] from Judgment...",
-                total=100,
-            )
-            try:
-                payload = self.api_client.get_project_dataset_stats(project_name)
-            except Exception as e:
-                judgeval_logger.error(f"Error pulling dataset: {str(e)}")
-                raise
-
-            progress.update(
-                task_id,
-                description=f"{progress.tasks[task_id].description} [rgb(25,227,160)]Done!)",
-            )
-
-            return payload

judgeval/data/tool.py

@@ -1,5 +0,0 @@
-from judgeval.data.judgment_types import ToolJudgmentType
-
-
-class Tool(ToolJudgmentType):
-    pass

judgeval/data/trace_run.py

@@ -1,37 +0,0 @@
-from pydantic import BaseModel
-from typing import List, Optional, Dict, Any, Union
-from judgeval.data import Trace
-from judgeval.scorers import APIScorerConfig, BaseScorer
-from judgeval.rules import Rule
-
-
-class TraceRun(BaseModel):
-    """
-    Stores example and evaluation scorers together for running an eval task
-
-    Args:
-        project_name (str): The name of the project the evaluation results belong to
-        eval_name (str): A name for this evaluation run
-        traces (List[Trace]): The traces to evaluate
-        scorers (List[Union[JudgmentScorer, BaseScorer]]): A list of scorers to use for evaluation
-        model (str): The model used as a judge when using LLM as a Judge
-        metadata (Optional[Dict[str, Any]]): Additional metadata to include for this evaluation run, e.g. comments, dataset name, purpose, etc.
-        rules (Optional[List[Rule]]): Rules to evaluate against scoring results
-        append (Optional[bool]): Whether to append to existing evaluation results
-        tools (Optional[List[Dict[str, Any]]]): List of tools to use for evaluation
-    """
-
-    organization_id: Optional[str] = None
-    project_name: Optional[str] = None
-    eval_name: Optional[str] = None
-    traces: Optional[List[Trace]] = None
-    scorers: List[Union[APIScorerConfig, BaseScorer]]
-    model: Optional[str] = "gpt-4.1"
-    trace_span_id: Optional[str] = None
-    append: Optional[bool] = False
-    override: Optional[bool] = False
-    rules: Optional[List[Rule]] = None
-    tools: Optional[List[Dict[str, Any]]] = None
-
-    class Config:
-        arbitrary_types_allowed = True

judgeval/evaluation_run.py

@@ -1,75 +0,0 @@
-from typing import List, Optional, Union
-from pydantic import BaseModel, field_validator, Field
-
-from judgeval.data import Example
-from judgeval.scorers import BaseScorer, APIScorerConfig
-from judgeval.constants import ACCEPTABLE_MODELS
-
-
-class EvaluationRun(BaseModel):
-    """
-    Stores example and evaluation scorers together for running an eval task
-
-    Args:
-        project_name (str): The name of the project the evaluation results belong to
-        eval_name (str): A name for this evaluation run
-        examples (List[Example]): The examples to evaluate
-        scorers (List[Union[JudgmentScorer, BaseScorer]]): A list of scorers to use for evaluation
-        model (str): The model used as a judge when using LLM as a Judge
-        metadata (Optional[Dict[str, Any]]): Additional metadata to include for this evaluation run, e.g. comments, dataset name, purpose, etc.
-    """
-
-    organization_id: Optional[str] = None
-    project_name: Optional[str] = Field(default=None, validate_default=True)
-    eval_name: Optional[str] = Field(default=None, validate_default=True)
-    examples: List[Example]
-    scorers: List[Union[APIScorerConfig, BaseScorer]]
-    model: Optional[str] = "gpt-4.1"
-    trace_span_id: Optional[str] = None
-    # API Key will be "" until user calls client.run_eval(), then API Key will be set
-    override: Optional[bool] = False
-    append: Optional[bool] = False
-
-    def model_dump(self, **kwargs):
-        data = super().model_dump(**kwargs)
-
-        data["scorers"] = [
-            scorer.model_dump() for scorer in self.scorers
-        ]  # Pydantic has problems with properly calling model_dump() on the scorers, so we need to do it manually
-
-        return data
-
-    @field_validator("examples")
-    def validate_examples(cls, v):
-        if not v:
-            raise ValueError("Examples cannot be empty.")
-        return v
-
-    @field_validator("scorers", mode="before")
-    def validate_scorers(cls, v):
-        if not v:
-            raise ValueError("Scorers cannot be empty.")
-        if not all(
-            isinstance(scorer, BaseScorer) or isinstance(scorer, APIScorerConfig)
-            for scorer in v
-        ):
-            raise ValueError(
-                "All scorers must be of type BaseScorer or APIScorerConfig."
-            )
-        return v
-
-    @field_validator("model")
-    def validate_model(cls, v, values):
-        if not v:
-            raise ValueError("Model cannot be empty.")
-
-        # Check if model is string or list of strings
-        if isinstance(v, str):
-            if v not in ACCEPTABLE_MODELS:
-                raise ValueError(
-                    f"Model name {v} not recognized. Please select a valid model name.)"
-                )
-            return v
-
-    class Config:
-        arbitrary_types_allowed = True