judgeval 0.14.0__tar.gz → 0.14.1__tar.gz

{judgeval-0.14.0 → judgeval-0.14.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.14.0
+Version: 0.14.1
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues

{judgeval-0.14.0 → judgeval-0.14.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.14.0"
+version = "0.14.1"
 authors = [
     { name = "Andrew Li", email = "andrew@judgmentlabs.ai" },
     { name = "Alex Shan", email = "alex@judgmentlabs.ai" },

{judgeval-0.14.0 → judgeval-0.14.1}/src/judgeval/api/api_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-09-24T18:25:18+00:00
+#   timestamp: 2025-09-29T19:54:47+00:00
 
 from __future__ import annotations
 from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
@@ -54,6 +54,8 @@ class SavePromptScorerRequest(TypedDict):
     threshold: float
     model: NotRequired[str]
     is_trace: NotRequired[bool]
+    options: NotRequired[Optional[Dict[str, float]]]
+    description: NotRequired[Optional[str]]
 
 
 class SavePromptScorerResponse(TypedDict):
@@ -143,6 +145,8 @@ class PromptScorer(TypedDict):
     prompt: str
     threshold: float
     model: NotRequired[str]
+    options: NotRequired[Optional[Dict[str, float]]]
+    description: NotRequired[Optional[str]]
     created_at: NotRequired[Optional[str]]
     updated_at: NotRequired[Optional[str]]
     is_trace: NotRequired[Optional[bool]]

{judgeval-0.14.0 → judgeval-0.14.1}/src/judgeval/data/judgment_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-09-24T18:25:17+00:00
+#   timestamp: 2025-09-29T19:54:46+00:00
 
 from __future__ import annotations
 from typing import Annotated, Any, Dict, List, Optional, Union
@@ -56,6 +56,8 @@ class SavePromptScorerRequest(BaseModel):
     threshold: Annotated[float, Field(title="Threshold")]
     model: Annotated[Optional[str], Field(title="Model")] = "gpt-5"
     is_trace: Annotated[Optional[bool], Field(title="Is Trace")] = False
+    options: Annotated[Optional[Dict[str, float]], Field(title="Options")] = None
+    description: Annotated[Optional[str], Field(title="Description")] = None
 
 
 class SavePromptScorerResponse(BaseModel):
@@ -156,6 +158,8 @@ class PromptScorer(BaseModel):
     prompt: Annotated[str, Field(title="Prompt")]
     threshold: Annotated[float, Field(title="Threshold")]
     model: Annotated[Optional[str], Field(title="Model")] = "gpt-5"
+    options: Annotated[Optional[Dict[str, float]], Field(title="Options")] = None
+    description: Annotated[Optional[str], Field(title="Description")] = None
     created_at: Annotated[Optional[AwareDatetime], Field(title="Created At")] = None
     updated_at: Annotated[Optional[AwareDatetime], Field(title="Updated At")] = None
     is_trace: Annotated[Optional[bool], Field(title="Is Trace")] = False

{judgeval-0.14.0 → judgeval-0.14.1}/src/judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py

@@ -4,19 +4,21 @@ from judgeval.scorers.api_scorer import (
     TraceAPIScorerConfig,
 )
 from judgeval.constants import APIScorerType
-from typing import Dict, Any
+from typing import Dict, Any, Optional
 from judgeval.api import JudgmentSyncClient
 from judgeval.exceptions import JudgmentAPIError
 import os
 from judgeval.logger import judgeval_logger
 from abc import ABC
 from judgeval.env import JUDGMENT_DEFAULT_GPT_MODEL
+from copy import copy
 
 
 def push_prompt_scorer(
     name: str,
     prompt: str,
     threshold: float,
+    options: Optional[Dict[str, float]] = None,
     model: str = JUDGMENT_DEFAULT_GPT_MODEL,
     judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
     organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
@@ -29,6 +31,7 @@ def push_prompt_scorer(
                 "name": name,
                 "prompt": prompt,
                 "threshold": threshold,
+                "options": options,
                 "model": model,
                 "is_trace": is_trace,
             }
@@ -98,6 +101,7 @@ def scorer_exists(
 class BasePromptScorer(ABC, APIScorerConfig):
     score_type: APIScorerType
     prompt: str
+    options: Optional[Dict[str, float]] = None
     judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or ""
     organization_id: str = os.getenv("JUDGMENT_ORG_ID") or ""
 
@@ -124,6 +128,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
             name=name,
             prompt=scorer_config["prompt"],
             threshold=scorer_config["threshold"],
+            options=scorer_config.get("options"),
             model=scorer_config.get("model"),
             judgment_api_key=judgment_api_key,
             organization_id=organization_id,
@@ -135,6 +140,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
         name: str,
         prompt: str,
         threshold: float = 0.5,
+        options: Optional[Dict[str, float]] = None,
         model: str = JUDGMENT_DEFAULT_GPT_MODEL,
         judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
         organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
@@ -150,6 +156,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
                 name,
                 prompt,
                 threshold,
+                options,
                 model,
                 judgment_api_key,
                 organization_id,
@@ -161,6 +168,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
                 name=name,
                 prompt=prompt,
                 threshold=threshold,
+                options=options,
                 model=model,
                 judgment_api_key=judgment_api_key,
                 organization_id=organization_id,
@@ -199,6 +207,14 @@ class BasePromptScorer(ABC, APIScorerConfig):
         self.push_prompt_scorer()
         judgeval_logger.info(f"Successfully updated model for {self.name}")
 
+    def set_options(self, options: Optional[Dict[str, float]]):
+        """
+        Updates the options of the scorer.
+        """
+        self.options = options
+        self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated options for {self.name}")
+
     def append_to_prompt(self, prompt_addition: str):
         """
         Appends a string to the prompt.
@@ -226,6 +242,12 @@ class BasePromptScorer(ABC, APIScorerConfig):
         """
         return self.model
 
+    def get_options(self) -> Dict[str, float] | None:
+        """
+        Returns the options of the scorer.
+        """
+        return copy(self.options) if self.options is not None else None
+
     def get_name(self) -> str | None:
         """
         Returns the name of the scorer.
@@ -241,6 +263,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
             "model": self.model,
             "prompt": self.prompt,
             "threshold": self.threshold,
+            "options": self.options,
         }
 
     def push_prompt_scorer(self):
@@ -251,6 +274,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
             self.name,
             self.prompt,
             self.threshold,
+            self.options,
             self.model,
             self.judgment_api_key,
             self.organization_id,
@@ -258,7 +282,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
         )
 
     def __str__(self):
-        return f"PromptScorer(name={self.name}, model={self.model}, prompt={self.prompt}, threshold={self.threshold})"
+        return f"PromptScorer(name={self.name}, model={self.model}, prompt={self.prompt}, threshold={self.threshold}, options={self.options})"
 
     def model_dump(self, *args, **kwargs) -> Dict[str, Any]:
         base = super().model_dump(*args, **kwargs)

{judgeval-0.14.0 → judgeval-0.14.1}/src/judgeval/version.py

@@ -1,4 +1,4 @@
-__version__ = "0.14.0"
+__version__ = "0.14.1"
 
 
 def get_version() -> str: