judgeval 0.3.1__tar.gz → 0.4.0__tar.gz

{judgeval-0.3.1 → judgeval-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.3.1
+Version: 0.4.0
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues

{judgeval-0.3.1 → judgeval-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.3.1"
+version = "0.4.0"
 authors = [
     { name="Andrew Li", email="andrew@judgmentlabs.ai" },
     { name="Alex Shan", email="alex@judgmentlabs.ai" },

{judgeval-0.3.1 → judgeval-0.4.0}/src/judgeval/common/api/api.py

@@ -1,4 +1,4 @@
-from typing import Literal, List, Dict, Any, Union
+from typing import Literal, List, Dict, Any, Union, Optional
 from requests import exceptions
 from judgeval.common.api.constants import (
     JUDGMENT_TRACES_FETCH_API_URL,
@@ -238,7 +238,7 @@ class JudgmentApiClient:
         }
         return self._do_request("POST", JUDGMENT_CHECK_EXAMPLE_KEYS_API_URL, payload)
 
-    def save_scorer(self, name: str, prompt: str, options: dict):
+    def save_scorer(self, name: str, prompt: str, options: Optional[dict] = None):
         payload: ScorerSavePayload = {
             "name": name,
             "prompt": prompt,

{judgeval-0.3.1 → judgeval-0.4.0}/src/judgeval/common/api/constants.py

@@ -162,7 +162,7 @@ JUDGMENT_SCORER_EXISTS_API_URL = f"{ROOT_API}/scorer_exists/"
 class ScorerSavePayload(TypedDict):
     name: str
     prompt: str
-    options: dict
+    options: Optional[dict]
 
 
 class ScorerFetchPayload(TypedDict):

{judgeval-0.3.1 → judgeval-0.4.0}/src/judgeval/common/tracer/core.py

@@ -418,6 +418,8 @@ class TraceClient:
                 self.start_time or time.time(), timezone.utc
             ).isoformat(),
             "duration": total_duration,
+            "trace_spans": [span.model_dump() for span in self.trace_spans],
+            "evaluation_runs": [run.model_dump() for run in self.evaluation_runs],
             "offline_mode": self.tracer.offline_mode,
             "parent_trace_id": self.parent_trace_id,
             "parent_name": self.parent_name,

{judgeval-0.3.1 → judgeval-0.4.0}/src/judgeval/common/tracer/trace_manager.py

@@ -54,7 +54,6 @@ class TraceManagerClient:
         Returns:
             dict: Server response containing UI URL and other metadata
         """
-        server_response = self.api_client.upsert_trace(trace_data)
 
         if self.tracer and self.tracer.use_s3 and final_save:
             try:
@@ -67,6 +66,11 @@ class TraceManagerClient:
             except Exception as e:
                 judgeval_logger.warning(f"Failed to save trace to S3: {str(e)}")
 
+        trace_data.pop("trace_spans", None)
+        trace_data.pop("evaluation_runs", None)
+
+        server_response = self.api_client.upsert_trace(trace_data)
+
         if not offline_mode and show_link and "ui_results_url" in server_response:
             pretty_str = f"\n🔍 You can view your trace data here: [rgb(106,0,255)][link={server_response['ui_results_url']}]View Trace[/link]\n"
             rprint(pretty_str)

{judgeval-0.3.1 → judgeval-0.4.0}/src/judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py

@@ -1,27 +1,29 @@
 from judgeval.scorers.api_scorer import APIScorerConfig
 from judgeval.constants import APIScorerType
-from typing import Mapping, Dict, Any
+from typing import Dict, Any, Optional
 from judgeval.common.api import JudgmentApiClient, JudgmentAPIException
 import os
 from judgeval.common.exceptions import JudgmentAPIError
+from copy import copy
+from judgeval.common.logger import judgeval_logger
 
 
 def push_prompt_scorer(
     name: str,
     prompt: str,
-    options: Mapping[str, float],
+    options: Optional[Dict[str, float]] = None,
     judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
     organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
 ) -> str:
     client = JudgmentApiClient(judgment_api_key, organization_id)
     try:
-        r = client.save_scorer(name, prompt, dict(options))
+        r = client.save_scorer(name, prompt, options)
     except JudgmentAPIException as e:
         if e.status_code == 500:
             raise JudgmentAPIError(
                 f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.error_detail}"
             )
-        raise JudgmentAPIError(f"Failed to save classifier scorer: {e.error_detail}")
+        raise JudgmentAPIError(f"Failed to save prompt scorer: {e.error_detail}")
     return r["name"]
 
 
@@ -32,7 +34,7 @@ def fetch_prompt_scorer(
 ):
     client = JudgmentApiClient(judgment_api_key, organization_id)
     try:
-        scorer_config = client.fetch_scorer(name)
+        scorer_config = client.fetch_scorer(name)["scorer"]
         scorer_config.pop("created_at")
         scorer_config.pop("updated_at")
         return scorer_config
@@ -42,7 +44,7 @@ def fetch_prompt_scorer(
                 f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.error_detail}"
             )
         raise JudgmentAPIError(
-            f"Failed to fetch classifier scorer '{name}': {e.error_detail}"
+            f"Failed to fetch prompt scorer '{name}': {e.error_detail}"
         )
 
 
@@ -72,7 +74,7 @@ class PromptScorer(APIScorerConfig):
     """
 
     prompt: str
-    options: Mapping[str, float]
+    options: Optional[Dict[str, float]] = None
     score_type: APIScorerType = APIScorerType.PROMPT_SCORER
     judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or ""
     organization_id: str = os.getenv("JUDGMENT_ORG_ID") or ""
@@ -88,7 +90,7 @@ class PromptScorer(APIScorerConfig):
         return cls(
             name=name,
             prompt=scorer_config["prompt"],
-            options=scorer_config["options"],
+            options=scorer_config.get("options"),
             judgment_api_key=judgment_api_key,
             organization_id=organization_id,
         )
@@ -98,12 +100,13 @@ class PromptScorer(APIScorerConfig):
         cls,
         name: str,
         prompt: str,
-        options: Mapping[str, float],
+        options: Optional[Dict[str, float]] = None,
         judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
         organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
     ):
         if not scorer_exists(name, judgment_api_key, organization_id):
             push_prompt_scorer(name, prompt, options, judgment_api_key, organization_id)
+            judgeval_logger.info(f"Successfully created PromptScorer: {name}")
             return cls(
                 name=name,
                 prompt=prompt,
@@ -117,13 +120,6 @@ class PromptScorer(APIScorerConfig):
             )
 
     # Setter functions. Each setter function pushes the scorer to the DB.
-    def set_name(self, name: str):
-        """
-        Updates the name of the scorer.
-        """
-        self.name = name
-        self.push_prompt_scorer()
-
     def set_threshold(self, threshold: float):
         """
         Updates the threshold of the scorer.
@@ -140,8 +136,9 @@ class PromptScorer(APIScorerConfig):
         """
         self.prompt = prompt
         self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated prompt for {self.name}")
 
-    def set_options(self, options: Mapping[str, float]):
+    def set_options(self, options: Dict[str, float]):
         """
         Updates the options with the new options.
 
@@ -150,6 +147,7 @@ class PromptScorer(APIScorerConfig):
         """
         self.options = options
         self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated options for {self.name}")
 
     def append_to_prompt(self, prompt_addition: str):
         """
@@ -157,6 +155,7 @@ class PromptScorer(APIScorerConfig):
         """
         self.prompt += prompt_addition
         self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully appended to prompt for {self.name}")
 
     # Getters
     def get_prompt(self) -> str | None:
@@ -165,11 +164,11 @@ class PromptScorer(APIScorerConfig):
         """
         return self.prompt
 
-    def get_options(self) -> Mapping[str, float] | None:
+    def get_options(self) -> Dict[str, float] | None:
         """
         Returns the options of the scorer.
         """
-        return self.options
+        return copy(self.options) if self.options is not None else None
 
     def get_name(self) -> str | None:
         """

{judgeval-0.3.1 → judgeval-0.4.0}/src/judgeval/scorers/score.py

@@ -48,6 +48,7 @@ async def safe_a_score_example(
         judgeval_logger.error(f"Error during scoring: {str(e)}")
         scorer.error = str(e)
         scorer.success = False
+        scorer.score = 0
         return