judgeval 0.10.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

judgeval/__init__.py

@@ -6,7 +6,8 @@ from judgeval.data.evaluation_run import ExampleEvaluationRun
 
 
 from typing import List, Optional, Union
-from judgeval.scorers import BaseScorer, ExampleAPIScorerConfig
+from judgeval.scorers import APIScorerConfig
+from judgeval.scorers.example_scorer import ExampleScorer
 from judgeval.data.example import Example
 from judgeval.logger import judgeval_logger
 from judgeval.env import JUDGMENT_API_KEY, JUDGMENT_DEFAULT_GPT_MODEL, JUDGMENT_ORG_ID
@@ -38,7 +39,7 @@ class JudgmentClient(metaclass=SingletonMeta):
     def run_evaluation(
         self,
         examples: List[Example],
-        scorers: List[Union[ExampleAPIScorerConfig, BaseScorer]],
+        scorers: List[Union[APIScorerConfig, ExampleScorer]],
         project_name: str = "default_project",
         eval_run_name: str = "default_eval_run",
         model: str = JUDGMENT_DEFAULT_GPT_MODEL,
@@ -51,10 +52,9 @@ class JudgmentClient(metaclass=SingletonMeta):
                 examples=examples,
                 scorers=scorers,
                 model=model,
-                organization_id=self.organization_id,
             )
 
-            results = run_eval(eval, self.api_key)
+            results = run_eval(eval)
             if assert_test:
                 assert_test_results(results)
 

judgeval/api/__init__.py

@@ -137,6 +137,13 @@ class JudgmentSyncClient:
             payload,
         )
 
+    def datasets_pull_all_for_judgeval(self, payload: DatasetsFetch) -> Any:
+        return self._request(
+            "POST",
+            url_for("/datasets/pull_all_for_judgeval/"),
+            payload,
+        )
+
     def datasets_create_for_judgeval(self, payload: DatasetCreate) -> Any:
         return self._request(
             "POST",
@@ -174,12 +181,12 @@ class JudgmentSyncClient:
             payload,
         )
 
-    def fetch_scorer(
-        self, payload: FetchPromptScorerRequest
-    ) -> FetchPromptScorerResponse:
+    def fetch_scorers(
+        self, payload: FetchPromptScorersRequest
+    ) -> FetchPromptScorersResponse:
         return self._request(
             "POST",
-            url_for("/fetch_scorer/"),
+            url_for("/fetch_scorers/"),
             payload,
         )
 
@@ -339,6 +346,13 @@ class JudgmentAsyncClient:
             payload,
         )
 
+    async def datasets_pull_all_for_judgeval(self, payload: DatasetsFetch) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/datasets/pull_all_for_judgeval/"),
+            payload,
+        )
+
     async def datasets_create_for_judgeval(self, payload: DatasetCreate) -> Any:
         return await self._request(
             "POST",
@@ -378,12 +392,12 @@ class JudgmentAsyncClient:
             payload,
         )
 
-    async def fetch_scorer(
-        self, payload: FetchPromptScorerRequest
-    ) -> FetchPromptScorerResponse:
+    async def fetch_scorers(
+        self, payload: FetchPromptScorersRequest
+    ) -> FetchPromptScorersResponse:
         return await self._request(
             "POST",
-            url_for("/fetch_scorer/"),
+            url_for("/fetch_scorers/"),
             payload,
         )
 

judgeval/api/api_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-09-10T17:42:12+00:00
+#   timestamp: 2025-09-12T16:54:35+00:00
 
 from __future__ import annotations
 from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
@@ -20,6 +20,10 @@ class DatasetFetch(TypedDict):
     project_name: str
 
 
+class DatasetsFetch(TypedDict):
+    project_name: str
+
+
 class ProjectAdd(TypedDict):
     project_name: str
 
@@ -57,8 +61,8 @@ class SavePromptScorerResponse(TypedDict):
     name: str
 
 
-class FetchPromptScorerRequest(TypedDict):
-    name: str
+class FetchPromptScorersRequest(TypedDict):
+    names: NotRequired[Optional[List[str]]]
 
 
 class CustomScorerUploadPayload(TypedDict):
@@ -151,7 +155,7 @@ class ScorerData(TypedDict):
     score: NotRequired[Optional[float]]
     reason: NotRequired[Optional[str]]
     strict_mode: NotRequired[Optional[bool]]
-    evaluation_model: NotRequired[str]
+    evaluation_model: NotRequired[Optional[str]]
     error: NotRequired[Optional[str]]
     additional_metadata: NotRequired[Optional[Dict[str, Any]]]
 
@@ -186,13 +190,13 @@ class OtelTraceSpan(TypedDict):
 
 
 class ExampleEvaluationRun(TypedDict):
-    id: NotRequired[Optional[str]]
-    project_name: NotRequired[Optional[str]]
-    eval_name: NotRequired[Optional[str]]
+    id: NotRequired[str]
+    project_name: str
+    eval_name: str
     custom_scorers: NotRequired[List[BaseScorer]]
     judgment_scorers: NotRequired[List[ScorerConfig]]
     model: str
-    created_at: NotRequired[Optional[str]]
+    created_at: NotRequired[str]
     examples: List[Example]
     trace_span_id: NotRequired[Optional[str]]
     trace_id: NotRequired[Optional[str]]
@@ -203,13 +207,13 @@ class HTTPValidationError(TypedDict):
 
 
 class TraceEvaluationRun(TypedDict):
-    id: NotRequired[Optional[str]]
-    project_name: NotRequired[Optional[str]]
-    eval_name: NotRequired[Optional[str]]
+    id: NotRequired[str]
+    project_name: str
+    eval_name: str
     custom_scorers: NotRequired[List[BaseScorer]]
     judgment_scorers: NotRequired[List[ScorerConfig]]
     model: str
-    created_at: NotRequired[Optional[str]]
+    created_at: NotRequired[str]
     trace_and_span_ids: List[TraceAndSpanId]
     is_offline: NotRequired[bool]
 
@@ -226,21 +230,30 @@ class DatasetReturn(TypedDict):
     examples: NotRequired[Optional[List[Example]]]
 
 
+class DatasetInfo(TypedDict):
+    dataset_id: str
+    name: str
+    created_at: str
+    dataset_kind: DatasetKind
+    entries: int
+    creator: str
+
+
 class DatasetCreate(TypedDict):
     name: str
     dataset_kind: DatasetKind
     project_name: str
-    examples: NotRequired[Optional[List[Example]]]
-    overwrite: NotRequired[Optional[bool]]
+    examples: List[Example]
+    overwrite: bool
 
 
-class FetchPromptScorerResponse(TypedDict):
-    scorer: PromptScorer
+class FetchPromptScorersResponse(TypedDict):
+    scorers: List[PromptScorer]
 
 
 class ScoringResult(TypedDict):
     success: bool
-    scorers_data: Optional[List[ScorerData]]
+    scorers_data: List[ScorerData]
     name: NotRequired[Optional[str]]
     data_object: NotRequired[Optional[Union[OtelTraceSpan, Example]]]
     trace_id: NotRequired[Optional[str]]

judgeval/data/evaluation_run.py

@@ -1,11 +1,11 @@
 from typing import List, Optional, Union, Tuple
-from litellm.files.main import BaseModel
-from pydantic import field_validator, model_validator, Field
+from pydantic import field_validator, model_validator, Field, BaseModel
 from datetime import datetime, timezone
 import uuid
 
 from judgeval.data import Example
-from judgeval.scorers import BaseScorer, APIScorerConfig
+from judgeval.scorers import APIScorerConfig
+from judgeval.scorers.example_scorer import ExampleScorer
 from judgeval.constants import ACCEPTABLE_MODELS
 from judgeval.data.judgment_types import (
     ExampleEvaluationRun as ExampleEvaluationRunJudgmentType,
@@ -14,19 +14,18 @@ from judgeval.data.judgment_types import (
 
 
 class EvaluationRun(BaseModel):
-    id: Optional[str] = Field(default_factory=lambda: str(uuid.uuid4()))
-    created_at: Optional[str] = Field(
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    created_at: str = Field(
         default_factory=lambda: datetime.now(timezone.utc).isoformat()
     )
-    organization_id: Optional[str] = None
-    custom_scorers: Optional[List[BaseScorer]] = None
-    judgment_scorers: Optional[List[APIScorerConfig]] = None
-    scorers: Optional[List[Union[BaseScorer, APIScorerConfig]]] = None
+    custom_scorers: List[ExampleScorer] = Field(default_factory=list)
+    judgment_scorers: List[APIScorerConfig] = Field(default_factory=list)
+    scorers: List[Union[ExampleScorer, APIScorerConfig]] = Field(default_factory=list)
     model: str
 
     def __init__(
         self,
-        scorers: Optional[List[Union[BaseScorer, APIScorerConfig]]] = None,
+        scorers: Optional[List[Union[ExampleScorer, APIScorerConfig]]] = None,
         **kwargs,
     ):
         """
@@ -38,7 +37,7 @@ class EvaluationRun(BaseModel):
         """
         if scorers is not None:
             # Automatically sort scorers into appropriate fields
-            custom_scorers = [s for s in scorers if isinstance(s, BaseScorer)]
+            custom_scorers = [s for s in scorers if isinstance(s, ExampleScorer)]
             judgment_scorers = [s for s in scorers if isinstance(s, APIScorerConfig)]
 
             # Always set both fields as lists (even if empty) to satisfy validation

judgeval/data/judgment_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-09-10T17:42:11+00:00
+#   timestamp: 2025-09-12T16:54:34+00:00
 
 from __future__ import annotations
 from typing import Annotated, Any, Dict, List, Optional, Union
@@ -22,6 +22,10 @@ class DatasetFetch(BaseModel):
     project_name: Annotated[str, Field(title="Project Name")]
 
 
+class DatasetsFetch(BaseModel):
+    project_name: Annotated[str, Field(title="Project Name")]
+
+
 class ProjectAdd(BaseModel):
     project_name: Annotated[str, Field(title="Project Name")]
 
@@ -59,8 +63,8 @@ class SavePromptScorerResponse(BaseModel):
     name: Annotated[str, Field(title="Name")]
 
 
-class FetchPromptScorerRequest(BaseModel):
-    name: Annotated[str, Field(title="Name")]
+class FetchPromptScorersRequest(BaseModel):
+    names: Annotated[Optional[List[str]], Field(title="Names")] = None
 
 
 class CustomScorerUploadPayload(BaseModel):
@@ -210,8 +214,8 @@ class OtelTraceSpan(BaseModel):
 
 class ExampleEvaluationRun(BaseModel):
     id: Annotated[Optional[str], Field(title="Id")] = None
-    project_name: Annotated[Optional[str], Field(title="Project Name")] = None
-    eval_name: Annotated[Optional[str], Field(title="Eval Name")] = None
+    project_name: Annotated[str, Field(title="Project Name")]
+    eval_name: Annotated[str, Field(title="Eval Name")]
     custom_scorers: Annotated[
         Optional[List[BaseScorer]], Field(title="Custom Scorers")
     ] = []
@@ -231,8 +235,8 @@ class HTTPValidationError(BaseModel):
 
 class TraceEvaluationRun(BaseModel):
     id: Annotated[Optional[str], Field(title="Id")] = None
-    project_name: Annotated[Optional[str], Field(title="Project Name")] = None
-    eval_name: Annotated[Optional[str], Field(title="Eval Name")] = None
+    project_name: Annotated[str, Field(title="Project Name")]
+    eval_name: Annotated[str, Field(title="Eval Name")]
     custom_scorers: Annotated[
         Optional[List[BaseScorer]], Field(title="Custom Scorers")
     ] = []
@@ -259,23 +263,30 @@ class DatasetReturn(BaseModel):
     examples: Annotated[Optional[List[Example]], Field(title="Examples")] = None
 
 
+class DatasetInfo(BaseModel):
+    dataset_id: Annotated[str, Field(title="Dataset Id")]
+    name: Annotated[str, Field(title="Name")]
+    created_at: Annotated[str, Field(title="Created At")]
+    dataset_kind: DatasetKind
+    entries: Annotated[int, Field(title="Entries")]
+    creator: Annotated[str, Field(title="Creator")]
+
+
 class DatasetCreate(BaseModel):
     name: Annotated[str, Field(title="Name")]
     dataset_kind: DatasetKind
     project_name: Annotated[str, Field(title="Project Name")]
-    examples: Annotated[Optional[List[Example]], Field(title="Examples")] = None
-    overwrite: Annotated[Optional[bool], Field(title="Overwrite")] = False
+    examples: Annotated[List[Example], Field(title="Examples")]
+    overwrite: Annotated[bool, Field(title="Overwrite")]
 
 
-class FetchPromptScorerResponse(BaseModel):
-    scorer: PromptScorer
+class FetchPromptScorersResponse(BaseModel):
+    scorers: Annotated[List[PromptScorer], Field(title="Scorers")]
 
 
 class ScoringResult(BaseModel):
     success: Annotated[bool, Field(title="Success")]
-    scorers_data: Annotated[Optional[List[ScorerData]], Field(title="Scorers Data")] = (
-        None
-    )
+    scorers_data: Annotated[List[ScorerData], Field(title="Scorers Data")]
     name: Annotated[Optional[str], Field(title="Name")] = None
     data_object: Annotated[
         Optional[Union[OtelTraceSpan, Example]], Field(title="Data Object")

judgeval/data/result.py

@@ -18,6 +18,7 @@ class ScoringResult(JudgmentScoringResult):
 
     # Need to override this so that it uses this repo's Example class
     data_object: Example
+    scorers_data: List[ScorerData]
 
     def model_dump(self, **kwargs):
         data = super().model_dump(**kwargs)

judgeval/data/scorer_data.py

@@ -6,36 +6,11 @@ ScorerData holds the information related to a single, completed Scorer evaluatio
 
 from __future__ import annotations
 
-from judgeval.data.judgment_types import ScorerData as JudgmentScorerData
+from judgeval.data.judgment_types import ScorerData
 from judgeval.scorers import BaseScorer
 from typing import List
 
 
-class ScorerData(JudgmentScorerData):
-    """
-    ScorerData holds the information related to a single, completed Scorer evaluation run.
-
-    For example, if running the Judgment Faithfulness scorer on an example, the ScorerData
-    object will contain whether the example passed its threshold expectation, as well as more detailed
-    information surrounding the evaluation run such as the claims and verdicts generated by the
-    judge model(s).
-    """
-
-    def to_dict(self) -> dict:
-        """Convert the ScorerData instance to a JSON-serializable dictionary."""
-        return {
-            "name": self.name,
-            "threshold": self.threshold,
-            "success": self.success,
-            "score": self.score,
-            "reason": self.reason,
-            "strict_mode": self.strict_mode,
-            "evaluation_model": self.evaluation_model,
-            "error": self.error,
-            "additional_metadata": self.additional_metadata,
-        }
-
-
 def create_scorer_data(scorer: BaseScorer) -> List[ScorerData]:
     """
     After a `scorer` is run, it contains information about the example that was evaluated

judgeval/dataset/__init__.py

@@ -3,7 +3,7 @@ import orjson
 import os
 import yaml
 from dataclasses import dataclass
-from typing import List, Literal, Optional
+from typing import List, Literal
 
 from judgeval.data import Example
 from judgeval.utils.file_utils import get_examples_from_yaml, get_examples_from_json
@@ -11,6 +11,18 @@ from judgeval.api import JudgmentSyncClient
 from judgeval.logger import judgeval_logger
 from judgeval.env import JUDGMENT_API_KEY, JUDGMENT_ORG_ID
 
+from judgeval.api.api_types import DatasetKind
+
+
+@dataclass
+class DatasetInfo:
+    dataset_id: str
+    name: str
+    created_at: str
+    dataset_kind: DatasetKind
+    entries: int
+    creator: str
+
 
 @dataclass
 class Dataset:
@@ -36,13 +48,16 @@ class Dataset:
         if not dataset:
             raise ValueError(f"Dataset {name} not found in project {project_name}")
         examples = dataset.get("examples", [])
+        if examples is None:
+            examples = []
+
         for e in examples:
-            if isinstance(e, dict) and isinstance(e.get("data"), dict):
-                e.update(e.pop("data"))
+            if isinstance(e, dict) and isinstance(e.get("data", {}), dict):
+                e.update(e.pop("data"))  # type: ignore
                 e.pop(
                     "example_id"
                 )  # TODO: remove once scorer data migraiton is complete
-        judgeval_logger.info(f"Succesfully retrieved dataset {name}!")
+        judgeval_logger.info(f"Successfully retrieved dataset {name}!")
         return cls(
             name=name,
             project_name=project_name,
@@ -54,7 +69,7 @@ class Dataset:
         cls,
         name: str,
         project_name: str,
-        examples: Optional[List[Example]] = None,
+        examples: List[Example] = [],
         overwrite: bool = False,
     ):
         if not examples:
@@ -65,19 +80,28 @@ class Dataset:
             {
                 "name": name,
                 "project_name": project_name,
-                "examples": [e.model_dump() for e in examples],
+                "examples": examples,  # type: ignore
                 "dataset_kind": "example",
                 "overwrite": overwrite,
             }
         )
 
-        judgeval_logger.info(f"Succesfull created dataset {name}!")
+        judgeval_logger.info(f"Successfully created dataset {name}!")
         return cls(
             name=name,
             project_name=project_name,
             examples=examples,
         )
 
+    @classmethod
+    def list(cls, project_name: str):
+        client = JudgmentSyncClient(cls.judgment_api_key, cls.organization_id)
+        datasets = client.datasets_pull_all_for_judgeval({"project_name": project_name})
+
+        judgeval_logger.info(f"Fetched all datasets for project {project_name}!")
+
+        return [DatasetInfo(**dataset_info) for dataset_info in datasets]
+
     def add_from_json(self, file_path: str) -> None:
         """
         Adds examples from a JSON file.
@@ -124,7 +148,7 @@ class Dataset:
             {
                 "dataset_name": self.name,
                 "project_name": self.project_name,
-                "examples": [e.model_dump() for e in examples],
+                "examples": examples,  # type: ignore
             }
         )
 

judgeval/env.py

@@ -19,8 +19,17 @@ def optional_env_var(var_name: str, default: str | None = None) -> str | None:
     return os.getenv(var_name, default)
 
 
-JUDGMENT_API_KEY = optional_env_var("JUDGMENT_API_KEY")
-JUDGMENT_ORG_ID = optional_env_var("JUDGMENT_ORG_ID")
+def required_env_var(var_name: str) -> str:
+    value = os.getenv(var_name)
+    if value is None:
+        raise EnvironmentError(
+            f"Environment variable '{var_name}' is required but not set."
+        )
+    return value
+
+
+JUDGMENT_API_KEY = required_env_var("JUDGMENT_API_KEY")
+JUDGMENT_ORG_ID = required_env_var("JUDGMENT_ORG_ID")
 JUDGMENT_API_URL = optional_env_var("JUDGMENT_API_URL", "https://api.judgmentlabs.ai")
 
 JUDGMENT_DEFAULT_GPT_MODEL = optional_env_var("JUDGMENT_DEFAULT_GPT_MODEL", "gpt-4.1")