judgeval 0.14.0__py3-none-any.whl → 0.14.1__py3-none-any.whl

judgeval/api/api_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-09-24T18:25:18+00:00
+#   timestamp: 2025-09-29T19:54:47+00:00
 
 from __future__ import annotations
 from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
@@ -54,6 +54,8 @@ class SavePromptScorerRequest(TypedDict):
     threshold: float
     model: NotRequired[str]
     is_trace: NotRequired[bool]
+    options: NotRequired[Optional[Dict[str, float]]]
+    description: NotRequired[Optional[str]]
 
 
 class SavePromptScorerResponse(TypedDict):
@@ -143,6 +145,8 @@ class PromptScorer(TypedDict):
     prompt: str
     threshold: float
     model: NotRequired[str]
+    options: NotRequired[Optional[Dict[str, float]]]
+    description: NotRequired[Optional[str]]
     created_at: NotRequired[Optional[str]]
     updated_at: NotRequired[Optional[str]]
     is_trace: NotRequired[Optional[bool]]

judgeval/data/judgment_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-09-24T18:25:17+00:00
+#   timestamp: 2025-09-29T19:54:46+00:00
 
 from __future__ import annotations
 from typing import Annotated, Any, Dict, List, Optional, Union
@@ -56,6 +56,8 @@ class SavePromptScorerRequest(BaseModel):
     threshold: Annotated[float, Field(title="Threshold")]
     model: Annotated[Optional[str], Field(title="Model")] = "gpt-5"
     is_trace: Annotated[Optional[bool], Field(title="Is Trace")] = False
+    options: Annotated[Optional[Dict[str, float]], Field(title="Options")] = None
+    description: Annotated[Optional[str], Field(title="Description")] = None
 
 
 class SavePromptScorerResponse(BaseModel):
@@ -156,6 +158,8 @@ class PromptScorer(BaseModel):
     prompt: Annotated[str, Field(title="Prompt")]
     threshold: Annotated[float, Field(title="Threshold")]
     model: Annotated[Optional[str], Field(title="Model")] = "gpt-5"
+    options: Annotated[Optional[Dict[str, float]], Field(title="Options")] = None
+    description: Annotated[Optional[str], Field(title="Description")] = None
     created_at: Annotated[Optional[AwareDatetime], Field(title="Created At")] = None
     updated_at: Annotated[Optional[AwareDatetime], Field(title="Updated At")] = None
     is_trace: Annotated[Optional[bool], Field(title="Is Trace")] = False

judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py

@@ -4,19 +4,21 @@ from judgeval.scorers.api_scorer import (
     TraceAPIScorerConfig,
 )
 from judgeval.constants import APIScorerType
-from typing import Dict, Any
+from typing import Dict, Any, Optional
 from judgeval.api import JudgmentSyncClient
 from judgeval.exceptions import JudgmentAPIError
 import os
 from judgeval.logger import judgeval_logger
 from abc import ABC
 from judgeval.env import JUDGMENT_DEFAULT_GPT_MODEL
+from copy import copy
 
 
 def push_prompt_scorer(
     name: str,
     prompt: str,
     threshold: float,
+    options: Optional[Dict[str, float]] = None,
     model: str = JUDGMENT_DEFAULT_GPT_MODEL,
     judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
     organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
@@ -29,6 +31,7 @@ def push_prompt_scorer(
                 "name": name,
                 "prompt": prompt,
                 "threshold": threshold,
+                "options": options,
                 "model": model,
                 "is_trace": is_trace,
             }
@@ -98,6 +101,7 @@ def scorer_exists(
 class BasePromptScorer(ABC, APIScorerConfig):
     score_type: APIScorerType
     prompt: str
+    options: Optional[Dict[str, float]] = None
     judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or ""
     organization_id: str = os.getenv("JUDGMENT_ORG_ID") or ""
 
@@ -124,6 +128,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
             name=name,
             prompt=scorer_config["prompt"],
             threshold=scorer_config["threshold"],
+            options=scorer_config.get("options"),
             model=scorer_config.get("model"),
             judgment_api_key=judgment_api_key,
             organization_id=organization_id,
@@ -135,6 +140,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
         name: str,
         prompt: str,
         threshold: float = 0.5,
+        options: Optional[Dict[str, float]] = None,
         model: str = JUDGMENT_DEFAULT_GPT_MODEL,
         judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
         organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
@@ -150,6 +156,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
                 name,
                 prompt,
                 threshold,
+                options,
                 model,
                 judgment_api_key,
                 organization_id,
@@ -161,6 +168,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
                 name=name,
                 prompt=prompt,
                 threshold=threshold,
+                options=options,
                 model=model,
                 judgment_api_key=judgment_api_key,
                 organization_id=organization_id,
@@ -199,6 +207,14 @@ class BasePromptScorer(ABC, APIScorerConfig):
         self.push_prompt_scorer()
         judgeval_logger.info(f"Successfully updated model for {self.name}")
 
+    def set_options(self, options: Optional[Dict[str, float]]):
+        """
+        Updates the options of the scorer.
+        """
+        self.options = options
+        self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated options for {self.name}")
+
     def append_to_prompt(self, prompt_addition: str):
         """
         Appends a string to the prompt.
@@ -226,6 +242,12 @@ class BasePromptScorer(ABC, APIScorerConfig):
         """
         return self.model
 
+    def get_options(self) -> Dict[str, float] | None:
+        """
+        Returns the options of the scorer.
+        """
+        return copy(self.options) if self.options is not None else None
+
     def get_name(self) -> str | None:
         """
         Returns the name of the scorer.
@@ -241,6 +263,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
             "model": self.model,
             "prompt": self.prompt,
             "threshold": self.threshold,
+            "options": self.options,
         }
 
     def push_prompt_scorer(self):
@@ -251,6 +274,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
             self.name,
             self.prompt,
             self.threshold,
+            self.options,
             self.model,
             self.judgment_api_key,
             self.organization_id,
@@ -258,7 +282,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
         )
 
     def __str__(self):
-        return f"PromptScorer(name={self.name}, model={self.model}, prompt={self.prompt}, threshold={self.threshold})"
+        return f"PromptScorer(name={self.name}, model={self.model}, prompt={self.prompt}, threshold={self.threshold}, options={self.options})"
 
     def model_dump(self, *args, **kwargs) -> Dict[str, Any]:
         base = super().model_dump(*args, **kwargs)

judgeval/version.py

@@ -1,4 +1,4 @@
-__version__ = "0.14.0"
+__version__ = "0.14.1"
 
 
 def get_version() -> str:

{judgeval-0.14.0.dist-info → judgeval-0.14.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.14.0
+Version: 0.14.1
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues

{judgeval-0.14.0.dist-info → judgeval-0.14.1.dist-info}/RECORD

@@ -4,14 +4,14 @@ judgeval/constants.py,sha256=JZZJ1MqzZZDVk-5PRPRbmLnM8mXI-RDL5vxa1JFuscs,3408
 judgeval/env.py,sha256=37Mn4g0OkpFxXCZGlO_CLqKJnyX-jx_R24tC28XJzig,2112
 judgeval/exceptions.py,sha256=tTbfe4yoOtPXmn22UQz9-6a-5PT9uOko85xaRRwr0Sw,621
 judgeval/logger.py,sha256=ZWbp0QfT1CJnQIjV-Zle4n489nFCKEmD2-ukx--iiow,1553
-judgeval/version.py,sha256=qFLvC9WMqiLgjMPdAqGbBm-v0rO2ePVeM8dvl-2r7WA,74
+judgeval/version.py,sha256=jxLK8GY7YWWLhTk4egDdn5VKiEty1Qpb-C3dLL2m-To,74
 judgeval/warnings.py,sha256=LbGte14ppiFjrkp-JJYueZ40NWFvMkWRvPXr6r-fUWw,73
 judgeval/api/__init__.py,sha256=3Pm0qQ4ZQj76jUsJVrnuazRnYcqF3pzM_Wv_Z6lOv0w,13216
-judgeval/api/api_types.py,sha256=w45Rh2I31JaqWoUsxCK5_rr_wJ1QkWYxDLjRppXkruE,8785
+judgeval/api/api_types.py,sha256=mtk9xcgYGj1zXV1w_vZ_fbVu9OI4i2IIDLL37lgYnV4,8979
 judgeval/data/__init__.py,sha256=1tU0EN0ThIfQ1fad5I3dKxAfTcZ5U8cvTLcQ6qLVLU0,407
 judgeval/data/evaluation_run.py,sha256=O41p99wNAuCAf6lsLNKzkZ6W-kL9LlzCYxVls7IcKkA,4727
 judgeval/data/example.py,sha256=eGJpF-lyUH734Cg90B7WtU9f8iKoS3VFGeV6R-GVCCc,1039
-judgeval/data/judgment_types.py,sha256=rZ4Uq6Va94bHF0Obn3wKB7E5n77R5N5WTkK5o02EAzg,16285
+judgeval/data/judgment_types.py,sha256=fNRqiGEG_nJhVkucagoxxgFqmpwK0-GlwWOwjmBtpXk,16603
 judgeval/data/result.py,sha256=XufFGSAkBDfevPUmzSgsR9HEqytISkM0U5HkhJmsjpY,2102
 judgeval/data/scorer_data.py,sha256=HeP15ZgftFTJCF8JmDJCLWXRnZJIaGDJCzl7Hg6gWwE,2006
 judgeval/data/trace.py,sha256=zSiR3o6xt8Z46XA3M9fJBtViF0BsPO6yKp9jxdscOSc,3881
@@ -39,7 +39,7 @@ judgeval/scorers/judgeval_scorers/api_scorers/answer_correctness.py,sha256=WUeFy
 judgeval/scorers/judgeval_scorers/api_scorers/answer_relevancy.py,sha256=ciiFBQQC4UDsk9qou9OiKbAR31s82eRUY1ZTt1gdM-0,407
 judgeval/scorers/judgeval_scorers/api_scorers/faithfulness.py,sha256=ucYOI6ztAjfoYmcgTDzN8u5RrehlVqrkeLEfss9b1fk,441
 judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py,sha256=V3RdrWhnR_vLBrtWw7QbgN9K_A-Och7-v9I2fN4z8gY,506
-judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py,sha256=UiX-qUVokx8W3EIr5i4_H6CGQT2pNBEYWSCtTUx1wLE,9196
+judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py,sha256=FbrXNMedeepYp_bADsysapIIZcr09l9EV9QWfGxvanw,10075
 judgeval/tracer/__init__.py,sha256=iqFvWok4QBW-1bs2zCmkhw4Y_o2d2mVeiPUtQbG9Nvc,35995
 judgeval/tracer/constants.py,sha256=ae8tivAW97awJQxdRB9OMqX50wOLX3zqChT_AGkPBu0,85
 judgeval/tracer/keys.py,sha256=ho4-_w4ngTVejdSKUH80sG6vtYt4c7FEKrYpFrDfPLs,2105
@@ -72,8 +72,8 @@ judgeval/utils/serialize.py,sha256=QXR-8Nj5rqOrI9zLx0oRLdk6DW6Bc7j8eyF4zQ7PLxA,6
 judgeval/utils/testing.py,sha256=m5Nexv65tmfSj1XvAPK5Ear7aJ7w5xjDtZN0tLZ_RBk,2939
 judgeval/utils/url.py,sha256=Shf0v3XcbaWpL0m1eGJEEO_z4TsQCnDB2Rl25OTUmiI,195
 judgeval/utils/version_check.py,sha256=ylZQSqV7kLzEOChxvav9SCHUU4OnaCp36tXHLjdzmw0,1072
-judgeval-0.14.0.dist-info/METADATA,sha256=PN_KgV2HavJDSdkoMAnusgLD1kiKNOfXMyWEVFyYx2s,8564
-judgeval-0.14.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-judgeval-0.14.0.dist-info/entry_points.txt,sha256=-eoeD-oDLn4A7MSgeBS9Akwanf3_0r0cgEleBcIOjg0,46
-judgeval-0.14.0.dist-info/licenses/LICENSE.md,sha256=tKmCg7k5QOmxPK19XMfzim04QiQJPmgIm0pAn55IJwk,11352
-judgeval-0.14.0.dist-info/RECORD,,
+judgeval-0.14.1.dist-info/METADATA,sha256=e8rJlBzFrfcadnR6-WiBQaRTKj2LlsnuxAS-Ag_WK1Q,8564
+judgeval-0.14.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+judgeval-0.14.1.dist-info/entry_points.txt,sha256=-eoeD-oDLn4A7MSgeBS9Akwanf3_0r0cgEleBcIOjg0,46
+judgeval-0.14.1.dist-info/licenses/LICENSE.md,sha256=tKmCg7k5QOmxPK19XMfzim04QiQJPmgIm0pAn55IJwk,11352
+judgeval-0.14.1.dist-info/RECORD,,