judgeval 0.3.2__py3-none-any.whl → 0.4.0__py3-none-any.whl

judgeval/common/api/api.py

@@ -1,4 +1,4 @@
-from typing import Literal, List, Dict, Any, Union
+from typing import Literal, List, Dict, Any, Union, Optional
 from requests import exceptions
 from judgeval.common.api.constants import (
     JUDGMENT_TRACES_FETCH_API_URL,
@@ -238,7 +238,7 @@ class JudgmentApiClient:
         }
         return self._do_request("POST", JUDGMENT_CHECK_EXAMPLE_KEYS_API_URL, payload)
 
-    def save_scorer(self, name: str, prompt: str, options: dict):
+    def save_scorer(self, name: str, prompt: str, options: Optional[dict] = None):
         payload: ScorerSavePayload = {
             "name": name,
             "prompt": prompt,

judgeval/common/api/constants.py

@@ -162,7 +162,7 @@ JUDGMENT_SCORER_EXISTS_API_URL = f"{ROOT_API}/scorer_exists/"
 class ScorerSavePayload(TypedDict):
     name: str
     prompt: str
-    options: dict
+    options: Optional[dict]
 
 
 class ScorerFetchPayload(TypedDict):

judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py

@@ -1,27 +1,29 @@
 from judgeval.scorers.api_scorer import APIScorerConfig
 from judgeval.constants import APIScorerType
-from typing import Mapping, Dict, Any
+from typing import Dict, Any, Optional
 from judgeval.common.api import JudgmentApiClient, JudgmentAPIException
 import os
 from judgeval.common.exceptions import JudgmentAPIError
+from copy import copy
+from judgeval.common.logger import judgeval_logger
 
 
 def push_prompt_scorer(
     name: str,
     prompt: str,
-    options: Mapping[str, float],
+    options: Optional[Dict[str, float]] = None,
     judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
     organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
 ) -> str:
     client = JudgmentApiClient(judgment_api_key, organization_id)
     try:
-        r = client.save_scorer(name, prompt, dict(options))
+        r = client.save_scorer(name, prompt, options)
     except JudgmentAPIException as e:
         if e.status_code == 500:
             raise JudgmentAPIError(
                 f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.error_detail}"
             )
-        raise JudgmentAPIError(f"Failed to save classifier scorer: {e.error_detail}")
+        raise JudgmentAPIError(f"Failed to save prompt scorer: {e.error_detail}")
     return r["name"]
 
 
@@ -32,7 +34,7 @@ def fetch_prompt_scorer(
 ):
     client = JudgmentApiClient(judgment_api_key, organization_id)
     try:
-        scorer_config = client.fetch_scorer(name)
+        scorer_config = client.fetch_scorer(name)["scorer"]
         scorer_config.pop("created_at")
         scorer_config.pop("updated_at")
         return scorer_config
@@ -42,7 +44,7 @@ def fetch_prompt_scorer(
                 f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.error_detail}"
             )
         raise JudgmentAPIError(
-            f"Failed to fetch classifier scorer '{name}': {e.error_detail}"
+            f"Failed to fetch prompt scorer '{name}': {e.error_detail}"
         )
 
 
@@ -72,7 +74,7 @@ class PromptScorer(APIScorerConfig):
     """
 
     prompt: str
-    options: Mapping[str, float]
+    options: Optional[Dict[str, float]] = None
     score_type: APIScorerType = APIScorerType.PROMPT_SCORER
     judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or ""
     organization_id: str = os.getenv("JUDGMENT_ORG_ID") or ""
@@ -88,7 +90,7 @@ class PromptScorer(APIScorerConfig):
         return cls(
             name=name,
             prompt=scorer_config["prompt"],
-            options=scorer_config["options"],
+            options=scorer_config.get("options"),
             judgment_api_key=judgment_api_key,
             organization_id=organization_id,
         )
@@ -98,12 +100,13 @@ class PromptScorer(APIScorerConfig):
         cls,
         name: str,
         prompt: str,
-        options: Mapping[str, float],
+        options: Optional[Dict[str, float]] = None,
         judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
         organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
     ):
         if not scorer_exists(name, judgment_api_key, organization_id):
             push_prompt_scorer(name, prompt, options, judgment_api_key, organization_id)
+            judgeval_logger.info(f"Successfully created PromptScorer: {name}")
             return cls(
                 name=name,
                 prompt=prompt,
@@ -117,13 +120,6 @@ class PromptScorer(APIScorerConfig):
             )
 
     # Setter functions. Each setter function pushes the scorer to the DB.
-    def set_name(self, name: str):
-        """
-        Updates the name of the scorer.
-        """
-        self.name = name
-        self.push_prompt_scorer()
-
     def set_threshold(self, threshold: float):
         """
         Updates the threshold of the scorer.
@@ -140,8 +136,9 @@ class PromptScorer(APIScorerConfig):
         """
         self.prompt = prompt
         self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated prompt for {self.name}")
 
-    def set_options(self, options: Mapping[str, float]):
+    def set_options(self, options: Dict[str, float]):
         """
         Updates the options with the new options.
 
@@ -150,6 +147,7 @@ class PromptScorer(APIScorerConfig):
         """
         self.options = options
         self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated options for {self.name}")
 
     def append_to_prompt(self, prompt_addition: str):
         """
@@ -157,6 +155,7 @@ class PromptScorer(APIScorerConfig):
         """
         self.prompt += prompt_addition
         self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully appended to prompt for {self.name}")
 
     # Getters
     def get_prompt(self) -> str | None:
@@ -165,11 +164,11 @@ class PromptScorer(APIScorerConfig):
         """
         return self.prompt
 
-    def get_options(self) -> Mapping[str, float] | None:
+    def get_options(self) -> Dict[str, float] | None:
         """
         Returns the options of the scorer.
         """
-        return self.options
+        return copy(self.options) if self.options is not None else None
 
     def get_name(self) -> str | None:
         """

judgeval/scorers/score.py

@@ -48,6 +48,7 @@ async def safe_a_score_example(
         judgeval_logger.error(f"Error during scoring: {str(e)}")
         scorer.error = str(e)
         scorer.success = False
+        scorer.score = 0
         return
 
 

{judgeval-0.3.2.dist-info → judgeval-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.3.2
+Version: 0.4.0
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues

{judgeval-0.3.2.dist-info → judgeval-0.4.0.dist-info}/RECORD

@@ -12,8 +12,8 @@ judgeval/common/exceptions.py,sha256=OkgDznu2wpBQZMXiZarLJYNk1HIcC8qYW7VypDC3Ook
 judgeval/common/logger.py,sha256=514eFLYWS_UL8VY-zAR2ePUlpQe4rbYlleLASFllLE4,1511
 judgeval/common/utils.py,sha256=oxGDRVWOICKWeyGgsoc36_yAyHSYF4XtH842Mkznwis,34739
 judgeval/common/api/__init__.py,sha256=-E7lpZz1fG8puR_aYUMfPmQ-Vyhd0bgzoaU5EhIuFjQ,114
-judgeval/common/api/api.py,sha256=wty02HYANeOYlM8fHOLc33ux5bu9Ieq7iRqCr-UP0ng,14157
-judgeval/common/api/constants.py,sha256=vAW94pbyTS6rv1TKpt7z6xxMJvTaAxFiy1D4kzuLHeg,4567
+judgeval/common/api/api.py,sha256=0Vd81CXoUYF1a7hOlI2PqXJcQO7eqWEbS8sX2W0CJMY,14184
+judgeval/common/api/constants.py,sha256=DXej0m8HEhb871SdiR8t_o4fzeMoQjHYqb_X0Plj8wY,4577
 judgeval/common/storage/__init__.py,sha256=a-PI7OL-ydyzugGUKmJKRBASnK-Q-gs82L9K9rSyJP8,90
 judgeval/common/storage/s3_storage.py,sha256=0-bNKheqJJyBZ92KGrzQtd1zocIRWBlfn_58L4a-Ay0,3719
 judgeval/common/tracer/__init__.py,sha256=tJCJsmVmrL89Phv88gNCJ-j0ITPez6lh8vhMAAlLNSc,795
@@ -47,7 +47,7 @@ judgeval/scorers/api_scorer.py,sha256=xlhqkeMUBFxl8daSXOTWOYwZjBAz7o6b4sVD5f8cIH
 judgeval/scorers/base_scorer.py,sha256=eDfQk8N8TQfM1ayJDWr0NTdSQxcbk9-VZHd0Igb9EbI,2878
 judgeval/scorers/example_scorer.py,sha256=2n45y3LMV1Q-ARyXLHqvVWETlnY1DqS7OLzPu9IBGz8,716
 judgeval/scorers/exceptions.py,sha256=ACDHK5-TWiF3NTk-wycaedpbrdobm-CvvC1JA_iP-Mk,179
-judgeval/scorers/score.py,sha256=2-M_AmOjIQR2c0qvuB4WIIQD-7zSNdzsWC8ttqltw2g,6601
+judgeval/scorers/score.py,sha256=SPoN5GG5sGKAt2aw_NgDj4-w5Mv1CGgaD9q-Sj1jRII,6626
 judgeval/scorers/utils.py,sha256=HQOYTJtNnsi_aPfMssePAaBbXpAv7LXgwUlWlDFuN2g,3965
 judgeval/scorers/judgeval_scorers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 judgeval/scorers/judgeval_scorers/api_scorers/__init__.py,sha256=GX4KkwPR2p-c0Y5mZingJa8EUfjAbMGhrmRBDBunOGw,1484
@@ -58,14 +58,14 @@ judgeval/scorers/judgeval_scorers/api_scorers/execution_order.py,sha256=NABO_iBd
 judgeval/scorers/judgeval_scorers/api_scorers/faithfulness.py,sha256=ps51bTgQsD9xGYsk1v9bx0WxQMqywSllCE9_xlJkLd8,531
 judgeval/scorers/judgeval_scorers/api_scorers/hallucination.py,sha256=SnFLvU4FGsMeUVUp0SGHSy_6wgfwr_vHPGnZx5YJl_Q,691
 judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py,sha256=aQzu-TiGqG74JDQ927evv5yGmnZw2AOolyHvlIhiUbI,683
-judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py,sha256=TS3uZ6YQfMs2yGCwzlz-yxZ3Rid79MGxEQESZkSX_Vo,7038
+judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py,sha256=nx73DeoVkSqJTP1hYxMsJobG9HVWgMDN5-xFOXt_8Ts,7348
 judgeval/scorers/judgeval_scorers/api_scorers/tool_dependency.py,sha256=Mcp1CjMNyOax9UkvoRdSyUYdO2Os1-Nko43y89m2Luo,594
 judgeval/scorers/judgeval_scorers/api_scorers/tool_order.py,sha256=Z2FLGBC7m_CLx-CMgXVuTvYvN0vY5yOcWA0ImBkeBfY,787
 judgeval/tracer/__init__.py,sha256=wkuXtOGDCrwgPPXlh_sSJmvGuWaAMHyNzk1TzB5f9aI,148
 judgeval/utils/alerts.py,sha256=3w_AjQrgfmOZvfqCridW8WAnHVxHHXokX9jNzVFyGjA,3297
 judgeval/utils/file_utils.py,sha256=PWHRs8dUr8iDwpglSSk4Yjd7C6ZhDzUaO-jV3m7riHM,1987
 judgeval/utils/requests.py,sha256=K3gUKrwL6TvwYKVYO5OeLWdUHn9NiUPmnIXhZEiEaHU,1534
-judgeval-0.3.2.dist-info/METADATA,sha256=3VDQzNPY0wKnaIpzmrP2OLsk6eEGJZH_dSl4QcJRVwE,10348
-judgeval-0.3.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-judgeval-0.3.2.dist-info/licenses/LICENSE.md,sha256=tKmCg7k5QOmxPK19XMfzim04QiQJPmgIm0pAn55IJwk,11352
-judgeval-0.3.2.dist-info/RECORD,,
+judgeval-0.4.0.dist-info/METADATA,sha256=6ZJMXn13RNp-LZfoBUNtmjt7nCvthWU9qT9YNBNe7Wo,10348
+judgeval-0.4.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+judgeval-0.4.0.dist-info/licenses/LICENSE.md,sha256=tKmCg7k5QOmxPK19XMfzim04QiQJPmgIm0pAn55IJwk,11352
+judgeval-0.4.0.dist-info/RECORD,,