judgeval 0.0.44__py3-none-any.whl → 0.0.46__py3-none-any.whl

judgeval/data/datasets/eval_dataset_client.py

@@ -1,24 +1,21 @@
-
 from typing import Optional, List
-import requests
+from requests import Response, exceptions
+from judgeval.utils.requests import requests
 from rich.progress import Progress, SpinnerColumn, TextColumn
 
 from judgeval.common.logger import debug, error, warning, info
 from judgeval.constants import (
     JUDGMENT_DATASETS_PUSH_API_URL,
     JUDGMENT_DATASETS_APPEND_EXAMPLES_API_URL,
-    JUDGMENT_DATASETS_PULL_API_URL, 
+    JUDGMENT_DATASETS_PULL_API_URL,
     JUDGMENT_DATASETS_PROJECT_STATS_API_URL,
     JUDGMENT_DATASETS_DELETE_API_URL,
-    JUDGMENT_DATASETS_INSERT_API_URL,
-    JUDGMENT_DATASETS_EXPORT_JSONL_API_URL
+    JUDGMENT_DATASETS_EXPORT_JSONL_API_URL,
 )
 from judgeval.data import Example, Trace
 from judgeval.data.datasets import EvalDataset
 
 
-
-
 class EvalDatasetClient:
     def __init__(self, judgment_api_key: str, organization_id: str):
         self.judgment_api_key = judgment_api_key
@@ -26,8 +23,14 @@ class EvalDatasetClient:
 
     def create_dataset(self) -> EvalDataset:
         return EvalDataset(judgment_api_key=self.judgment_api_key)
-    
-    def push(self, dataset: EvalDataset, alias: str, project_name: str, overwrite: Optional[bool] = False) -> bool:
+
+    def push(
+        self,
+        dataset: EvalDataset,
+        alias: str,
+        project_name: str,
+        overwrite: Optional[bool] = False,
+    ) -> bool:
         debug(f"Pushing dataset with alias '{alias}' (overwrite={overwrite})")
         if overwrite:
             warning(f"Overwrite enabled for alias '{alias}'")
@@ -55,45 +58,46 @@ class EvalDatasetClient:
                 total=100,
             )
             content = {
-                    "dataset_alias": alias,
-                    "project_name": project_name,
-                    "examples": [e.to_dict() for e in dataset.examples],
-                    "traces": [t.model_dump() for t in dataset.traces],
-                    "overwrite": overwrite,
-                }
+                "dataset_alias": alias,
+                "project_name": project_name,
+                "examples": [e.to_dict() for e in dataset.examples],
+                "traces": [t.model_dump() for t in dataset.traces],
+                "overwrite": overwrite,
+            }
             try:
                 response = requests.post(
-                    JUDGMENT_DATASETS_PUSH_API_URL, 
+                    JUDGMENT_DATASETS_PUSH_API_URL,
                     json=content,
                     headers={
                         "Content-Type": "application/json",
                         "Authorization": f"Bearer {self.judgment_api_key}",
-                        "X-Organization-Id": self.organization_id
+                        "X-Organization-Id": self.organization_id,
                     },
-                    verify=True
+                    verify=True,
                 )
                 if response.status_code != 200:
                     error(f"Server error during push: {response.json()}")
                     raise Exception(f"Server error during push: {response.json()}")
                 response.raise_for_status()
-            except requests.exceptions.HTTPError as err:
+            except exceptions.HTTPError as err:
                 if response.status_code == 422:
                     error(f"Validation error during push: {err.response.json()}")
                 else:
                     error(f"HTTP error during push: {err}")
-            
+
             info(f"Successfully pushed dataset with alias '{alias}'")
             payload = response.json()
             dataset._alias = payload.get("_alias")
             dataset._id = payload.get("_id")
             progress.update(
-                    task_id,
-                    description=f"{progress.tasks[task_id].description} [rgb(25,227,160)]Done!)",
-                )
+                task_id,
+                description=f"{progress.tasks[task_id].description} [rgb(25,227,160)]Done!)",
+            )
             return True
-        
 
-    def append_examples(self, alias: str, examples: List[Example], project_name: str) -> bool:
+    def append_examples(
+        self, alias: str, examples: List[Example], project_name: str
+    ) -> bool:
         debug(f"Appending dataset with alias '{alias}'")
         """
         Appends the dataset to Judgment platform
@@ -119,37 +123,37 @@ class EvalDatasetClient:
                 total=100,
             )
             content = {
-                    "dataset_alias": alias,
-                    "project_name": project_name,
-                    "examples": [e.to_dict() for e in examples],
-                }
+                "dataset_alias": alias,
+                "project_name": project_name,
+                "examples": [e.to_dict() for e in examples],
+            }
             try:
                 response = requests.post(
-                    JUDGMENT_DATASETS_APPEND_EXAMPLES_API_URL, 
+                    JUDGMENT_DATASETS_APPEND_EXAMPLES_API_URL,
                     json=content,
                     headers={
                         "Content-Type": "application/json",
                         "Authorization": f"Bearer {self.judgment_api_key}",
-                        "X-Organization-Id": self.organization_id
+                        "X-Organization-Id": self.organization_id,
                     },
-                    verify=True
+                    verify=True,
                 )
                 if response.status_code != 200:
                     error(f"Server error during append: {response.json()}")
                     raise Exception(f"Server error during append: {response.json()}")
                 response.raise_for_status()
-            except requests.exceptions.HTTPError as err:
+            except exceptions.HTTPError as err:
                 if response.status_code == 422:
                     error(f"Validation error during append: {err.response.json()}")
                 else:
                     error(f"HTTP error during append: {err}")
-            
+
             progress.update(
-                    task_id,
-                    description=f"{progress.tasks[task_id].description} [rgb(25,227,160)]Done!)",
-                )
+                task_id,
+                description=f"{progress.tasks[task_id].description} [rgb(25,227,160)]Done!)",
+            )
             return True
-        
+
     def pull(self, alias: str, project_name: str) -> EvalDataset:
         debug(f"Pulling dataset with alias '{alias}'")
         """
@@ -171,81 +175,75 @@ class EvalDatasetClient:
         dataset = self.create_dataset()
 
         with Progress(
-                SpinnerColumn(style="rgb(106,0,255)"),
-                TextColumn("[progress.description]{task.description}"),
-                transient=False,
-            ) as progress:
-                task_id = progress.add_task(
-                    f"Pulling [rgb(106,0,255)]'{alias}'[/rgb(106,0,255)] from Judgment...",
-                    total=100,
+            SpinnerColumn(style="rgb(106,0,255)"),
+            TextColumn("[progress.description]{task.description}"),
+            transient=False,
+        ) as progress:
+            task_id = progress.add_task(
+                f"Pulling [rgb(106,0,255)]'{alias}'[/rgb(106,0,255)] from Judgment...",
+                total=100,
+            )
+            request_body = {"dataset_alias": alias, "project_name": project_name}
+
+            try:
+                response = requests.post(
+                    JUDGMENT_DATASETS_PULL_API_URL,
+                    json=request_body,
+                    headers={
+                        "Content-Type": "application/json",
+                        "Authorization": f"Bearer {self.judgment_api_key}",
+                        "X-Organization-Id": self.organization_id,
+                    },
+                    verify=True,
                 )
-                request_body = {
-                    "dataset_alias": alias,
-                    "project_name": project_name
-                }
+                response.raise_for_status()
+            except exceptions.RequestException as e:
+                error(f"Error pulling dataset: {str(e)}")
+                raise
 
-                try:
-                    response = requests.post(
-                        JUDGMENT_DATASETS_PULL_API_URL, 
-                        json=request_body,
-                        headers={
-                            "Content-Type": "application/json",
-                            "Authorization": f"Bearer {self.judgment_api_key}",
-                            "X-Organization-Id": self.organization_id
-                        },
-                        verify=True
-                    )
-                    response.raise_for_status()
-                except requests.exceptions.RequestException as e:
-                    error(f"Error pulling dataset: {str(e)}")
-                    raise
+            info(f"Successfully pulled dataset with alias '{alias}'")
+            payload = response.json()
+            dataset.examples = [Example(**e) for e in payload.get("examples", [])]
+            dataset.traces = [Trace(**t) for t in payload.get("traces", [])]
+            dataset._alias = payload.get("alias")
+            dataset._id = payload.get("id")
+            progress.update(
+                task_id,
+                description=f"{progress.tasks[task_id].description} [rgb(25,227,160)]Done!)",
+            )
 
-                info(f"Successfully pulled dataset with alias '{alias}'")
-                payload = response.json()
-                dataset.examples = [Example(**e) for e in payload.get("examples", [])]
-                dataset.traces = [Trace(**t) for t in payload.get("traces", [])]
-                dataset._alias = payload.get("alias")
-                dataset._id = payload.get("id")
-                progress.update(
-                    task_id,
-                    description=f"{progress.tasks[task_id].description} [rgb(25,227,160)]Done!)",
-                )
+            return dataset
 
-                return dataset
-    
     def delete(self, alias: str, project_name: str) -> bool:
         with Progress(
-                SpinnerColumn(style="rgb(106,0,255)"),
-                TextColumn("[progress.description]{task.description}"),
-                transient=False,
-            ) as progress:
-                task_id = progress.add_task(
-                    f"Deleting [rgb(106,0,255)]'{alias}'[/rgb(106,0,255)] from Judgment...",
-                    total=100,
+            SpinnerColumn(style="rgb(106,0,255)"),
+            TextColumn("[progress.description]{task.description}"),
+            transient=False,
+        ) as progress:
+            progress.add_task(
+                f"Deleting [rgb(106,0,255)]'{alias}'[/rgb(106,0,255)] from Judgment...",
+                total=100,
+            )
+            request_body = {"dataset_alias": alias, "project_name": project_name}
+
+            try:
+                response = requests.post(
+                    JUDGMENT_DATASETS_DELETE_API_URL,
+                    json=request_body,
+                    headers={
+                        "Content-Type": "application/json",
+                        "Authorization": f"Bearer {self.judgment_api_key}",
+                        "X-Organization-Id": self.organization_id,
+                    },
+                    verify=True,
                 )
-                request_body = {
-                    "dataset_alias": alias,
-                    "project_name": project_name
-                }
+                response.raise_for_status()
+            except exceptions.RequestException as e:
+                error(f"Error deleting dataset: {str(e)}")
+                raise
 
-                try:
-                    response = requests.post(
-                        JUDGMENT_DATASETS_DELETE_API_URL, 
-                        json=request_body,
-                        headers={
-                            "Content-Type": "application/json",
-                            "Authorization": f"Bearer {self.judgment_api_key}",
-                            "X-Organization-Id": self.organization_id
-                        },
-                        verify=True
-                    )
-                    response.raise_for_status()
-                except requests.exceptions.RequestException as e:
-                    error(f"Error deleting dataset: {str(e)}")
-                    raise
+            return True
 
-                return True
-        
     def pull_project_dataset_stats(self, project_name: str) -> dict:
         debug(f"Pulling project datasets stats for project_name: {project_name}'")
         """
@@ -265,91 +263,43 @@ class EvalDatasetClient:
         # Make a POST request to the Judgment API to get the dataset
 
         with Progress(
-                SpinnerColumn(style="rgb(106,0,255)"),
-                TextColumn("[progress.description]{task.description}"),
-                transient=False,
-            ) as progress:
-                task_id = progress.add_task(
-                    f"Pulling [rgb(106,0,255)]' datasets'[/rgb(106,0,255)] from Judgment...",
-                    total=100,
-                )
-                request_body = {
-                    "project_name": project_name
-                }
-
-                try:
-                    response = requests.post(
-                        JUDGMENT_DATASETS_PROJECT_STATS_API_URL, 
-                        json=request_body,
-                        headers={
-                            "Content-Type": "application/json",
-                            "Authorization": f"Bearer {self.judgment_api_key}",
-                            "X-Organization-Id": self.organization_id
-                        },
-                        verify=True
-                    )
-                    response.raise_for_status()
-                except requests.exceptions.RequestException as e:
-                    error(f"Error pulling dataset: {str(e)}")
-                    raise
-
-                info(f"Successfully pulled datasets for userid: {self.judgment_api_key}'")
-                payload = response.json()
-
-                progress.update(
-                    task_id,
-                    description=f"{progress.tasks[task_id].description} [rgb(25,227,160)]Done!)",
-                )
-
-                return payload
-
-    def insert_dataset(self, alias: str, examples: List[Example], project_name: str) -> bool:
-        """
-        Edits the dataset on Judgment platform by adding new examples
-
-        Mock request:
-        {
-            "alias": alias,
-            "examples": [...],
-            "project_name": project_name
-        }
-        """
-        with Progress(
-                SpinnerColumn(style="rgb(106,0,255)"),
-                TextColumn("[progress.description]{task.description}"),
-                transient=False,
-            ) as progress:
+            SpinnerColumn(style="rgb(106,0,255)"),
+            TextColumn("[progress.description]{task.description}"),
+            transient=False,
+        ) as progress:
             task_id = progress.add_task(
-                f"Editing dataset [rgb(106,0,255)]'{alias}'[/rgb(106,0,255)] on Judgment...",
+                "Pulling [rgb(106,0,255)]' datasets'[/rgb(106,0,255)] from Judgment...",
                 total=100,
             )
-            
-            content = {
-                "dataset_alias": alias,
-                "examples": [e.to_dict() for e in examples],
-                "project_name": project_name
-            }
+            request_body = {"project_name": project_name}
 
             try:
                 response = requests.post(
-                    JUDGMENT_DATASETS_INSERT_API_URL,
-                    json=content,
+                    JUDGMENT_DATASETS_PROJECT_STATS_API_URL,
+                    json=request_body,
                     headers={
                         "Content-Type": "application/json",
                         "Authorization": f"Bearer {self.judgment_api_key}",
-                        "X-Organization-Id": self.organization_id
+                        "X-Organization-Id": self.organization_id,
                     },
-                    verify=True
+                    verify=True,
                 )
                 response.raise_for_status()
-            except requests.exceptions.RequestException as e:
-                error(f"Error editing dataset: {str(e)}")
-                return False
-        
-            info(f"Successfully edited dataset '{alias}'")
-            return True
+            except exceptions.RequestException as e:
+                error(f"Error pulling dataset: {str(e)}")
+                raise
+
+            info(f"Successfully pulled datasets for userid: {self.judgment_api_key}'")
+            payload = response.json()
+
+            progress.update(
+                task_id,
+                description=f"{progress.tasks[task_id].description} [rgb(25,227,160)]Done!)",
+            )
 
-    def export_jsonl(self, alias: str, project_name: str) -> requests.Response:
+            return payload
+
+    def export_jsonl(self, alias: str, project_name: str) -> Response:
         """Export dataset in JSONL format from Judgment platform"""
         debug(f"Exporting dataset with alias '{alias}' as JSONL")
         with Progress(
@@ -368,13 +318,13 @@ class EvalDatasetClient:
                     headers={
                         "Content-Type": "application/json",
                         "Authorization": f"Bearer {self.judgment_api_key}",
-                        "X-Organization-Id": self.organization_id
+                        "X-Organization-Id": self.organization_id,
                     },
                     stream=True,
-                    verify=True
+                    verify=True,
                 )
                 response.raise_for_status()
-            except requests.exceptions.HTTPError as err:
+            except exceptions.HTTPError as err:
                 if err.response.status_code == 404:
                     error(f"Dataset not found: {alias}")
                 else:
@@ -383,11 +333,11 @@ class EvalDatasetClient:
             except Exception as e:
                 error(f"Error during export: {str(e)}")
                 raise
-                
+
             info(f"Successfully exported dataset with alias '{alias}'")
             progress.update(
                 task_id,
                 description=f"{progress.tasks[task_id].description} [rgb(25,227,160)]Done!)",
             )
-            
+
             return response

judgeval/data/example.py

@@ -2,14 +2,12 @@
 Classes for representing examples in a dataset.
 """
 
-
 from typing import Optional, Any, Dict, List, Union
 from uuid import uuid4
 from pydantic import BaseModel, Field, field_validator
 from enum import Enum
 from datetime import datetime
 from judgeval.data.tool import Tool
-import time
 
 
 class ExampleParams(Enum):
@@ -38,103 +36,129 @@ class Example(BaseModel):
     example_index: Optional[int] = None
     created_at: Optional[str] = None
     trace_id: Optional[str] = None
-    
+
     def __init__(self, **data):
-        if 'example_id' not in data:
-            data['example_id'] = str(uuid4())
+        if "example_id" not in data:
+            data["example_id"] = str(uuid4())
         # Set timestamp if not provided
-        if 'created_at' not in data:
-            data['created_at'] = datetime.now().isoformat()
+        if "created_at" not in data:
+            data["created_at"] = datetime.now().isoformat()
         super().__init__(**data)
-    
-    @field_validator('input', mode='before')
+
+    @field_validator("input", mode="before")
     @classmethod
     def validate_input(cls, v):
         if v is not None:
             if not isinstance(v, (str, dict)):
-                raise ValueError(f"Input must be a string or dictionary but got {v} of type {type(v)}")
-            
+                raise ValueError(
+                    f"Input must be a string or dictionary but got {v} of type {type(v)}"
+                )
+
             # If it's a string, check that it's not empty
             if isinstance(v, str) and not v:
                 raise ValueError(f"Input string must be non-empty but got '{v}'")
-            
+
             # If it's a dictionary, check that it's not empty
             if isinstance(v, dict) and not v:
                 raise ValueError(f"Input dictionary must be non-empty but got {v}")
-        
+
         return v
-    
-    @field_validator('actual_output', mode='before')
+
+    @field_validator("actual_output", mode="before")
     @classmethod
     def validate_actual_output(cls, v):
         if v is not None:
             if not isinstance(v, (str, list)):
-                raise ValueError(f"Actual output must be a string or a list of strings but got {v} of type {type(v)}")
+                raise ValueError(
+                    f"Actual output must be a string or a list of strings but got {v} of type {type(v)}"
+                )
             if isinstance(v, list) and not all(isinstance(item, str) for item in v):
-                raise ValueError(f"All items in actual_output must be strings but got {v}")
+                raise ValueError(
+                    f"All items in actual_output must be strings but got {v}"
+                )
         return v
-    
-    @field_validator('expected_output', mode='before')
+
+    @field_validator("expected_output", mode="before")
     @classmethod
     def validate_expected_output(cls, v):
         if v is not None and not isinstance(v, (str, list)):
-            raise ValueError(f"Expected output must be a string, a list of strings, or None but got {v} of type {type(v)}")
+            raise ValueError(
+                f"Expected output must be a string, a list of strings, or None but got {v} of type {type(v)}"
+            )
         if isinstance(v, list) and not all(isinstance(item, str) for item in v):
-            raise ValueError(f"All items in expected_output must be strings but got {v}")
+            raise ValueError(
+                f"All items in expected_output must be strings but got {v}"
+            )
         return v
-    
-    @field_validator('expected_tools')
+
+    @field_validator("expected_tools")
     @classmethod
     def validate_expected_tools(cls, v):
         if v is not None:
             if not isinstance(v, list):
-                raise ValueError(f"Expected tools must be a list of Tools or None but got {v} of type {type(v)}")
-            
+                raise ValueError(
+                    f"Expected tools must be a list of Tools or None but got {v} of type {type(v)}"
+                )
+
             # Check that each item in the list is a Tool
             for i, item in enumerate(v):
                 if not isinstance(item, Tool):
-                    raise ValueError(f"Expected tools must be a list of Tools, but item at index {i} is {item} of type {type(item)}")
-        
+                    raise ValueError(
+                        f"Expected tools must be a list of Tools, but item at index {i} is {item} of type {type(item)}"
+                    )
+
         return v
-    
-    @field_validator('context', 'retrieval_context', 'tools_called', mode='before')
+
+    @field_validator("context", "retrieval_context", "tools_called", mode="before")
     @classmethod
     def validate_string_lists(cls, v, info):
         field_name = info.field_name
         if v is not None:
             if not isinstance(v, list):
-                raise ValueError(f"{field_name} must be a list of strings or None but got {v} of type {type(v)}")
+                raise ValueError(
+                    f"{field_name} must be a list of strings or None but got {v} of type {type(v)}"
+                )
             for i, item in enumerate(v):
                 if not isinstance(item, str):
-                    raise ValueError(f"All items in {field_name} must be strings but item at index {i} is {item} of type {type(item)}")
+                    raise ValueError(
+                        f"All items in {field_name} must be strings but item at index {i} is {item} of type {type(item)}"
+                    )
         return v
-    
-    @field_validator('additional_metadata', mode='before')
+
+    @field_validator("additional_metadata", mode="before")
     @classmethod
     def validate_additional_metadata(cls, v):
         if v is not None and not isinstance(v, dict):
-            raise ValueError(f"Additional metadata must be a dictionary or None but got {v} of type {type(v)}")
+            raise ValueError(
+                f"Additional metadata must be a dictionary or None but got {v} of type {type(v)}"
+            )
         return v
-    
-    @field_validator('example_index', mode='before')
+
+    @field_validator("example_index", mode="before")
     @classmethod
     def validate_example_index(cls, v):
         if v is not None and not isinstance(v, int):
-            raise ValueError(f"Example index must be an integer or None but got {v} of type {type(v)}")
+            raise ValueError(
+                f"Example index must be an integer or None but got {v} of type {type(v)}"
+            )
         return v
-    
-    @field_validator('created_at', mode='before')
+
+    @field_validator("created_at", mode="before")
     @classmethod
     def validate_created_at(cls, v):
         if v is not None and not isinstance(v, str):
-            raise ValueError(f"Timestamp must be a string or None but got {v} of type {type(v)}")
+            raise ValueError(
+                f"Timestamp must be a string or None but got {v} of type {type(v)}"
+            )
         return v
-    
-    @field_validator('trace_id', mode='before')
+
+    @field_validator("trace_id", mode="before")
     @classmethod
     def validate_trace_id(cls, v):
         if v is not None and not isinstance(v, str):
-            raise ValueError(f"Trace ID must be a string or None but got {v} of type {type(v)}")
+            raise ValueError(
+                f"Trace ID must be a string or None but got {v} of type {type(v)}"
+            )
         return v
 
     def to_dict(self):