judgeval 0.10.1__tar.gz → 0.11.0__tar.gz

{judgeval-0.10.1 → judgeval-0.11.0}/.github/workflows/ci.yaml

@@ -47,6 +47,8 @@ jobs:
       - name: Run tests
         run: |
           cd src
+          export JUDGMENT_API_KEY="$JUDGEVAL_GH_JUDGMENT_API_KEY"
+          export JUDGMENT_ORG_ID="$JUDGEVAL_GH_JUDGMENT_ORG_ID"
           uv run pytest tests
 
   run-e2e-tests-staging:

judgeval-0.11.0/.pre-commit-config.yaml

@@ -0,0 +1,23 @@
+repos:
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.8.17
+    hooks:
+      - id: uv-lock
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.13.0
+    hooks:
+      - id: ruff
+        name: ruff (linter)
+        args: [--fix]
+      - id: ruff-format
+        name: ruff (formatter)
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.17.0
+    hooks:
+      - id: mypy
+        language: system
+        # These next two lines allow commits even if mypy fails, REMOVE once we fix all mypy errors
+        verbose: true
+        entry: bash -c 'mypy src/judgeval/ || true'

{judgeval-0.10.1 → judgeval-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.10.1
+Version: 0.11.0
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues

{judgeval-0.10.1 → judgeval-0.11.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.10.1"
+version = "0.11.0"
 authors = [
     { name = "Andrew Li", email = "andrew@judgmentlabs.ai" },
     { name = "Alex Shan", email = "alex@judgmentlabs.ai" },

{judgeval-0.10.1 → judgeval-0.11.0}/scripts/api_generator.py

@@ -33,12 +33,13 @@ JUDGEVAL_PATHS: List[str] = [
     "/add_to_run_eval_queue/traces",
     "/get_evaluation_status/",
     "/save_scorer/",
-    "/fetch_scorer/",
+    "/fetch_scorers/",
     "/scorer_exists/",
     "/upload_custom_scorer/",
     "/datasets/create_for_judgeval/",
     "/datasets/insert_examples_for_judgeval/",
     "/datasets/pull_for_judgeval/",
+    "/datasets/pull_all_for_judgeval/",
     "/projects/resolve/",
     "/e2e_fetch_trace/",
     "/e2e_fetch_span_score/",

{judgeval-0.10.1 → judgeval-0.11.0}/scripts/openapi_transform.py

@@ -32,12 +32,13 @@ JUDGEVAL_PATHS: List[str] = [
     "/add_to_run_eval_queue/traces",
     "/get_evaluation_status/",
     "/save_scorer/",
-    "/fetch_scorer/",
+    "/fetch_scorers/",
     "/scorer_exists/",
     "/upload_custom_scorer/",
     "/datasets/create_for_judgeval/",
     "/datasets/insert_examples_for_judgeval/",
     "/datasets/pull_for_judgeval/",
+    "/datasets/pull_all_for_judgeval/",
     "/projects/resolve/",
     "/e2e_fetch_trace/",
     "/e2e_fetch_span_score/",

{judgeval-0.10.1 → judgeval-0.11.0}/src/judgeval/__init__.py

@@ -6,7 +6,8 @@ from judgeval.data.evaluation_run import ExampleEvaluationRun
 
 
 from typing import List, Optional, Union
-from judgeval.scorers import BaseScorer, ExampleAPIScorerConfig
+from judgeval.scorers import APIScorerConfig
+from judgeval.scorers.example_scorer import ExampleScorer
 from judgeval.data.example import Example
 from judgeval.logger import judgeval_logger
 from judgeval.env import JUDGMENT_API_KEY, JUDGMENT_DEFAULT_GPT_MODEL, JUDGMENT_ORG_ID
@@ -38,7 +39,7 @@ class JudgmentClient(metaclass=SingletonMeta):
     def run_evaluation(
         self,
         examples: List[Example],
-        scorers: List[Union[ExampleAPIScorerConfig, BaseScorer]],
+        scorers: List[Union[APIScorerConfig, ExampleScorer]],
         project_name: str = "default_project",
         eval_run_name: str = "default_eval_run",
         model: str = JUDGMENT_DEFAULT_GPT_MODEL,
@@ -51,10 +52,9 @@ class JudgmentClient(metaclass=SingletonMeta):
                 examples=examples,
                 scorers=scorers,
                 model=model,
-                organization_id=self.organization_id,
             )
 
-            results = run_eval(eval, self.api_key)
+            results = run_eval(eval)
             if assert_test:
                 assert_test_results(results)
 

{judgeval-0.10.1 → judgeval-0.11.0}/src/judgeval/api/__init__.py

@@ -137,12 +137,13 @@ class JudgmentSyncClient:
             payload,
         )
 
-    def datasets_pull_all_for_judgeval(self, payload: DatasetsFetch) -> List[DatasetInfo]:
+    def datasets_pull_all_for_judgeval(self, payload: DatasetsFetch) -> Any:
         return self._request(
             "POST",
             url_for("/datasets/pull_all_for_judgeval/"),
             payload,
         )
+
     def datasets_create_for_judgeval(self, payload: DatasetCreate) -> Any:
         return self._request(
             "POST",
@@ -180,12 +181,12 @@ class JudgmentSyncClient:
             payload,
         )
 
-    def fetch_scorer(
-        self, payload: FetchPromptScorerRequest
-    ) -> FetchPromptScorerResponse:
+    def fetch_scorers(
+        self, payload: FetchPromptScorersRequest
+    ) -> FetchPromptScorersResponse:
         return self._request(
             "POST",
-            url_for("/fetch_scorer/"),
+            url_for("/fetch_scorers/"),
             payload,
         )
 
@@ -345,6 +346,13 @@ class JudgmentAsyncClient:
             payload,
         )
 
+    async def datasets_pull_all_for_judgeval(self, payload: DatasetsFetch) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/datasets/pull_all_for_judgeval/"),
+            payload,
+        )
+
     async def datasets_create_for_judgeval(self, payload: DatasetCreate) -> Any:
         return await self._request(
             "POST",
@@ -384,12 +392,12 @@ class JudgmentAsyncClient:
             payload,
         )
 
-    async def fetch_scorer(
-        self, payload: FetchPromptScorerRequest
-    ) -> FetchPromptScorerResponse:
+    async def fetch_scorers(
+        self, payload: FetchPromptScorersRequest
+    ) -> FetchPromptScorersResponse:
         return await self._request(
             "POST",
-            url_for("/fetch_scorer/"),
+            url_for("/fetch_scorers/"),
             payload,
         )
 

{judgeval-0.10.1 → judgeval-0.11.0}/src/judgeval/api/api_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-09-10T17:42:12+00:00
+#   timestamp: 2025-09-12T16:54:35+00:00
 
 from __future__ import annotations
 from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
@@ -19,6 +19,7 @@ class DatasetFetch(TypedDict):
     dataset_name: str
     project_name: str
 
+
 class DatasetsFetch(TypedDict):
     project_name: str
 
@@ -60,8 +61,8 @@ class SavePromptScorerResponse(TypedDict):
     name: str
 
 
-class FetchPromptScorerRequest(TypedDict):
-    name: str
+class FetchPromptScorersRequest(TypedDict):
+    names: NotRequired[Optional[List[str]]]
 
 
 class CustomScorerUploadPayload(TypedDict):
@@ -154,7 +155,7 @@ class ScorerData(TypedDict):
     score: NotRequired[Optional[float]]
     reason: NotRequired[Optional[str]]
     strict_mode: NotRequired[Optional[bool]]
-    evaluation_model: NotRequired[str]
+    evaluation_model: NotRequired[Optional[str]]
     error: NotRequired[Optional[str]]
     additional_metadata: NotRequired[Optional[Dict[str, Any]]]
 
@@ -189,13 +190,13 @@ class OtelTraceSpan(TypedDict):
 
 
 class ExampleEvaluationRun(TypedDict):
-    id: NotRequired[Optional[str]]
-    project_name: NotRequired[Optional[str]]
-    eval_name: NotRequired[Optional[str]]
+    id: NotRequired[str]
+    project_name: str
+    eval_name: str
     custom_scorers: NotRequired[List[BaseScorer]]
     judgment_scorers: NotRequired[List[ScorerConfig]]
     model: str
-    created_at: NotRequired[Optional[str]]
+    created_at: NotRequired[str]
     examples: List[Example]
     trace_span_id: NotRequired[Optional[str]]
     trace_id: NotRequired[Optional[str]]
@@ -206,13 +207,13 @@ class HTTPValidationError(TypedDict):
 
 
 class TraceEvaluationRun(TypedDict):
-    id: NotRequired[Optional[str]]
-    project_name: NotRequired[Optional[str]]
-    eval_name: NotRequired[Optional[str]]
+    id: NotRequired[str]
+    project_name: str
+    eval_name: str
     custom_scorers: NotRequired[List[BaseScorer]]
     judgment_scorers: NotRequired[List[ScorerConfig]]
     model: str
-    created_at: NotRequired[Optional[str]]
+    created_at: NotRequired[str]
     trace_and_span_ids: List[TraceAndSpanId]
     is_offline: NotRequired[bool]
 
@@ -228,30 +229,31 @@ class DatasetReturn(TypedDict):
     project_name: str
     examples: NotRequired[Optional[List[Example]]]
 
+
 class DatasetInfo(TypedDict):
     dataset_id: str
     name: str
     created_at: str
     dataset_kind: DatasetKind
     entries: int
-    creator: str 
+    creator: str
 
 
 class DatasetCreate(TypedDict):
     name: str
     dataset_kind: DatasetKind
     project_name: str
-    examples: NotRequired[Optional[List[Example]]]
-    overwrite: NotRequired[Optional[bool]]
+    examples: List[Example]
+    overwrite: bool
 
 
-class FetchPromptScorerResponse(TypedDict):
-    scorer: PromptScorer
+class FetchPromptScorersResponse(TypedDict):
+    scorers: List[PromptScorer]
 
 
 class ScoringResult(TypedDict):
     success: bool
-    scorers_data: Optional[List[ScorerData]]
+    scorers_data: List[ScorerData]
     name: NotRequired[Optional[str]]
     data_object: NotRequired[Optional[Union[OtelTraceSpan, Example]]]
     trace_id: NotRequired[Optional[str]]

{judgeval-0.10.1 → judgeval-0.11.0}/src/judgeval/data/evaluation_run.py

@@ -1,11 +1,11 @@
 from typing import List, Optional, Union, Tuple
-from litellm.files.main import BaseModel
-from pydantic import field_validator, model_validator, Field
+from pydantic import field_validator, model_validator, Field, BaseModel
 from datetime import datetime, timezone
 import uuid
 
 from judgeval.data import Example
-from judgeval.scorers import BaseScorer, APIScorerConfig
+from judgeval.scorers import APIScorerConfig
+from judgeval.scorers.example_scorer import ExampleScorer
 from judgeval.constants import ACCEPTABLE_MODELS
 from judgeval.data.judgment_types import (
     ExampleEvaluationRun as ExampleEvaluationRunJudgmentType,
@@ -14,19 +14,18 @@ from judgeval.data.judgment_types import (
 
 
 class EvaluationRun(BaseModel):
-    id: Optional[str] = Field(default_factory=lambda: str(uuid.uuid4()))
-    created_at: Optional[str] = Field(
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    created_at: str = Field(
         default_factory=lambda: datetime.now(timezone.utc).isoformat()
     )
-    organization_id: Optional[str] = None
-    custom_scorers: Optional[List[BaseScorer]] = None
-    judgment_scorers: Optional[List[APIScorerConfig]] = None
-    scorers: Optional[List[Union[BaseScorer, APIScorerConfig]]] = None
+    custom_scorers: List[ExampleScorer] = Field(default_factory=list)
+    judgment_scorers: List[APIScorerConfig] = Field(default_factory=list)
+    scorers: List[Union[ExampleScorer, APIScorerConfig]] = Field(default_factory=list)
     model: str
 
     def __init__(
         self,
-        scorers: Optional[List[Union[BaseScorer, APIScorerConfig]]] = None,
+        scorers: Optional[List[Union[ExampleScorer, APIScorerConfig]]] = None,
         **kwargs,
     ):
         """
@@ -38,7 +37,7 @@ class EvaluationRun(BaseModel):
         """
         if scorers is not None:
             # Automatically sort scorers into appropriate fields
-            custom_scorers = [s for s in scorers if isinstance(s, BaseScorer)]
+            custom_scorers = [s for s in scorers if isinstance(s, ExampleScorer)]
             judgment_scorers = [s for s in scorers if isinstance(s, APIScorerConfig)]
 
             # Always set both fields as lists (even if empty) to satisfy validation

{judgeval-0.10.1 → judgeval-0.11.0}/src/judgeval/data/judgment_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-09-10T17:42:11+00:00
+#   timestamp: 2025-09-12T16:54:34+00:00
 
 from __future__ import annotations
 from typing import Annotated, Any, Dict, List, Optional, Union
@@ -22,6 +22,10 @@ class DatasetFetch(BaseModel):
     project_name: Annotated[str, Field(title="Project Name")]
 
 
+class DatasetsFetch(BaseModel):
+    project_name: Annotated[str, Field(title="Project Name")]
+
+
 class ProjectAdd(BaseModel):
     project_name: Annotated[str, Field(title="Project Name")]
 
@@ -59,8 +63,8 @@ class SavePromptScorerResponse(BaseModel):
     name: Annotated[str, Field(title="Name")]
 
 
-class FetchPromptScorerRequest(BaseModel):
-    name: Annotated[str, Field(title="Name")]
+class FetchPromptScorersRequest(BaseModel):
+    names: Annotated[Optional[List[str]], Field(title="Names")] = None
 
 
 class CustomScorerUploadPayload(BaseModel):
@@ -210,8 +214,8 @@ class OtelTraceSpan(BaseModel):
 
 class ExampleEvaluationRun(BaseModel):
     id: Annotated[Optional[str], Field(title="Id")] = None
-    project_name: Annotated[Optional[str], Field(title="Project Name")] = None
-    eval_name: Annotated[Optional[str], Field(title="Eval Name")] = None
+    project_name: Annotated[str, Field(title="Project Name")]
+    eval_name: Annotated[str, Field(title="Eval Name")]
     custom_scorers: Annotated[
         Optional[List[BaseScorer]], Field(title="Custom Scorers")
     ] = []
@@ -231,8 +235,8 @@ class HTTPValidationError(BaseModel):
 
 class TraceEvaluationRun(BaseModel):
     id: Annotated[Optional[str], Field(title="Id")] = None
-    project_name: Annotated[Optional[str], Field(title="Project Name")] = None
-    eval_name: Annotated[Optional[str], Field(title="Eval Name")] = None
+    project_name: Annotated[str, Field(title="Project Name")]
+    eval_name: Annotated[str, Field(title="Eval Name")]
     custom_scorers: Annotated[
         Optional[List[BaseScorer]], Field(title="Custom Scorers")
     ] = []
@@ -259,23 +263,30 @@ class DatasetReturn(BaseModel):
     examples: Annotated[Optional[List[Example]], Field(title="Examples")] = None
 
 
+class DatasetInfo(BaseModel):
+    dataset_id: Annotated[str, Field(title="Dataset Id")]
+    name: Annotated[str, Field(title="Name")]
+    created_at: Annotated[str, Field(title="Created At")]
+    dataset_kind: DatasetKind
+    entries: Annotated[int, Field(title="Entries")]
+    creator: Annotated[str, Field(title="Creator")]
+
+
 class DatasetCreate(BaseModel):
     name: Annotated[str, Field(title="Name")]
     dataset_kind: DatasetKind
     project_name: Annotated[str, Field(title="Project Name")]
-    examples: Annotated[Optional[List[Example]], Field(title="Examples")] = None
-    overwrite: Annotated[Optional[bool], Field(title="Overwrite")] = False
+    examples: Annotated[List[Example], Field(title="Examples")]
+    overwrite: Annotated[bool, Field(title="Overwrite")]
 
 
-class FetchPromptScorerResponse(BaseModel):
-    scorer: PromptScorer
+class FetchPromptScorersResponse(BaseModel):
+    scorers: Annotated[List[PromptScorer], Field(title="Scorers")]
 
 
 class ScoringResult(BaseModel):
     success: Annotated[bool, Field(title="Success")]
-    scorers_data: Annotated[Optional[List[ScorerData]], Field(title="Scorers Data")] = (
-        None
-    )
+    scorers_data: Annotated[List[ScorerData], Field(title="Scorers Data")]
     name: Annotated[Optional[str], Field(title="Name")] = None
     data_object: Annotated[
         Optional[Union[OtelTraceSpan, Example]], Field(title="Data Object")

{judgeval-0.10.1 → judgeval-0.11.0}/src/judgeval/data/result.py

@@ -18,6 +18,7 @@ class ScoringResult(JudgmentScoringResult):
 
     # Need to override this so that it uses this repo's Example class
     data_object: Example
+    scorers_data: List[ScorerData]
 
     def model_dump(self, **kwargs):
         data = super().model_dump(**kwargs)

{judgeval-0.10.1 → judgeval-0.11.0}/src/judgeval/data/scorer_data.py

@@ -6,36 +6,11 @@ ScorerData holds the information related to a single, completed Scorer evaluatio
 
 from __future__ import annotations
 
-from judgeval.data.judgment_types import ScorerData as JudgmentScorerData
+from judgeval.data.judgment_types import ScorerData
 from judgeval.scorers import BaseScorer
 from typing import List
 
 
-class ScorerData(JudgmentScorerData):
-    """
-    ScorerData holds the information related to a single, completed Scorer evaluation run.
-
-    For example, if running the Judgment Faithfulness scorer on an example, the ScorerData
-    object will contain whether the example passed its threshold expectation, as well as more detailed
-    information surrounding the evaluation run such as the claims and verdicts generated by the
-    judge model(s).
-    """
-
-    def to_dict(self) -> dict:
-        """Convert the ScorerData instance to a JSON-serializable dictionary."""
-        return {
-            "name": self.name,
-            "threshold": self.threshold,
-            "success": self.success,
-            "score": self.score,
-            "reason": self.reason,
-            "strict_mode": self.strict_mode,
-            "evaluation_model": self.evaluation_model,
-            "error": self.error,
-            "additional_metadata": self.additional_metadata,
-        }
-
-
 def create_scorer_data(scorer: BaseScorer) -> List[ScorerData]:
     """
     After a `scorer` is run, it contains information about the example that was evaluated

{judgeval-0.10.1 → judgeval-0.11.0}/src/judgeval/dataset/__init__.py

@@ -3,7 +3,7 @@ import orjson
 import os
 import yaml
 from dataclasses import dataclass
-from typing import List, Literal, Optional
+from typing import List, Literal
 
 from judgeval.data import Example
 from judgeval.utils.file_utils import get_examples_from_yaml, get_examples_from_json
@@ -13,15 +13,17 @@ from judgeval.env import JUDGMENT_API_KEY, JUDGMENT_ORG_ID
 
 from judgeval.api.api_types import DatasetKind
 
+
 @dataclass
 class DatasetInfo:
     dataset_id: str
-    name: str 
+    name: str
     created_at: str
     dataset_kind: DatasetKind
     entries: int
     creator: str
 
+
 @dataclass
 class Dataset:
     examples: List[Example]
@@ -46,9 +48,12 @@ class Dataset:
         if not dataset:
             raise ValueError(f"Dataset {name} not found in project {project_name}")
         examples = dataset.get("examples", [])
+        if examples is None:
+            examples = []
+
         for e in examples:
-            if isinstance(e, dict) and isinstance(e.get("data"), dict):
-                e.update(e.pop("data"))
+            if isinstance(e, dict) and isinstance(e.get("data", {}), dict):
+                e.update(e.pop("data"))  # type: ignore
                 e.pop(
                     "example_id"
                 )  # TODO: remove once scorer data migraiton is complete
@@ -64,7 +69,7 @@ class Dataset:
         cls,
         name: str,
         project_name: str,
-        examples: Optional[List[Example]] = None,
+        examples: List[Example] = [],
         overwrite: bool = False,
     ):
         if not examples:
@@ -75,7 +80,7 @@ class Dataset:
             {
                 "name": name,
                 "project_name": project_name,
-                "examples": [e.model_dump() for e in examples],
+                "examples": examples,  # type: ignore
                 "dataset_kind": "example",
                 "overwrite": overwrite,
             }
@@ -87,18 +92,14 @@ class Dataset:
             project_name=project_name,
             examples=examples,
         )
+
     @classmethod
-    def list(
-        cls,
-        project_name: str
-    ):
+    def list(cls, project_name: str):
         client = JudgmentSyncClient(cls.judgment_api_key, cls.organization_id)
-        datasets = client.datasets_pull_all_for_judgeval(
-            {"project_name": project_name}
-        )
-        
+        datasets = client.datasets_pull_all_for_judgeval({"project_name": project_name})
+
         judgeval_logger.info(f"Fetched all datasets for project {project_name}!")
-                             
+
         return [DatasetInfo(**dataset_info) for dataset_info in datasets]
 
     def add_from_json(self, file_path: str) -> None:
@@ -147,7 +148,7 @@ class Dataset:
             {
                 "dataset_name": self.name,
                 "project_name": self.project_name,
-                "examples": [e.model_dump() for e in examples],
+                "examples": examples,  # type: ignore
             }
         )
 

{judgeval-0.10.1 → judgeval-0.11.0}/src/judgeval/env.py

@@ -19,8 +19,17 @@ def optional_env_var(var_name: str, default: str | None = None) -> str | None:
     return os.getenv(var_name, default)
 
 
-JUDGMENT_API_KEY = optional_env_var("JUDGMENT_API_KEY")
-JUDGMENT_ORG_ID = optional_env_var("JUDGMENT_ORG_ID")
+def required_env_var(var_name: str) -> str:
+    value = os.getenv(var_name)
+    if value is None:
+        raise EnvironmentError(
+            f"Environment variable '{var_name}' is required but not set."
+        )
+    return value
+
+
+JUDGMENT_API_KEY = required_env_var("JUDGMENT_API_KEY")
+JUDGMENT_ORG_ID = required_env_var("JUDGMENT_ORG_ID")
 JUDGMENT_API_URL = optional_env_var("JUDGMENT_API_URL", "https://api.judgmentlabs.ai")
 
 JUDGMENT_DEFAULT_GPT_MODEL = optional_env_var("JUDGMENT_DEFAULT_GPT_MODEL", "gpt-4.1")