judgeval 0.1.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py

@@ -1,28 +1,51 @@
-from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.scorers.api_scorer import (
+    APIScorerConfig,
+    ExampleAPIScorerConfig,
+    TraceAPIScorerConfig,
+)
 from judgeval.constants import APIScorerType
-from typing import Mapping, Dict, Any
-from judgeval.common.api import JudgmentApiClient, JudgmentAPIException
+from typing import Dict, Any, Optional
+from judgeval.api import JudgmentSyncClient
+from judgeval.exceptions import JudgmentAPIError
 import os
-from judgeval.common.exceptions import JudgmentAPIError
+from judgeval.logger import judgeval_logger
+from abc import ABC
+from judgeval.env import JUDGMENT_DEFAULT_GPT_MODEL
+from copy import copy
+from judgeval.utils.decorators.dont_throw import dont_throw
 
 
 def push_prompt_scorer(
     name: str,
     prompt: str,
-    options: Mapping[str, float],
+    threshold: float,
+    options: Optional[Dict[str, float]] = None,
+    model: str = JUDGMENT_DEFAULT_GPT_MODEL,
+    description: Optional[str] = None,
     judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
     organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
+    is_trace: bool = False,
 ) -> str:
-    client = JudgmentApiClient(judgment_api_key, organization_id)
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
     try:
-        r = client.save_scorer(name, prompt, dict(options))
-    except JudgmentAPIException as e:
-        if e.status_code == 500:
-            raise JudgmentAPIError(
-                f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.error_detail}"
-            )
-        raise JudgmentAPIError(f"Failed to save classifier scorer: {e.error_detail}")
-    return r["name"]
+        r = client.save_scorer(
+            payload={
+                "name": name,
+                "prompt": prompt,
+                "threshold": threshold,
+                "options": options,
+                "model": model,
+                "description": description,
+                "is_trace": is_trace,
+            }
+        )
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to save prompt scorer: {e.detail}",
+            response=e.response,
+        )
+    return r["scorer_response"]["name"]
 
 
 def fetch_prompt_scorer(
@@ -30,19 +53,26 @@ def fetch_prompt_scorer(
     judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
     organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
 ):
-    client = JudgmentApiClient(judgment_api_key, organization_id)
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
     try:
-        scorer_config = client.fetch_scorer(name)
-        scorer_config.pop("created_at")
-        scorer_config.pop("updated_at")
-        return scorer_config
-    except JudgmentAPIException as e:
-        if e.status_code == 500:
+        fetched_scorers = client.fetch_scorers({"names": [name]})
+        if len(fetched_scorers["scorers"]) == 0:
+            judgeval_logger.error(f"Prompt scorer '{name}' not found")
             raise JudgmentAPIError(
-                f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.error_detail}"
+                status_code=404,
+                detail=f"Prompt scorer '{name}' not found",
+                response=None,  # type: ignore
             )
+        else:
+            scorer_config = fetched_scorers["scorers"][0]
+            scorer_config.pop("created_at")
+            scorer_config.pop("updated_at")
+            return scorer_config
+    except JudgmentAPIError as e:
         raise JudgmentAPIError(
-            f"Failed to fetch classifier scorer '{name}': {e.error_detail}"
+            status_code=e.status_code,
+            detail=f"Failed to fetch prompt scorer '{name}': {e.detail}",
+            response=e.response,
         )
 
 
@@ -51,33 +81,33 @@ def scorer_exists(
     judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
     organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
 ):
-    client = JudgmentApiClient(judgment_api_key, organization_id)
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
     try:
-        return client.scorer_exists(name)["exists"]
-    except JudgmentAPIException as e:
+        return client.scorer_exists({"name": name})["exists"]
+    except JudgmentAPIError as e:
         if e.status_code == 500:
             raise JudgmentAPIError(
-                f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.error_detail}"
+                status_code=e.status_code,
+                detail=f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.detail}",
+                response=e.response,
             )
-        raise JudgmentAPIError(f"Failed to check if scorer exists: {e.error_detail}")
-
-
-class PromptScorer(APIScorerConfig):
-    """
-    In the Judgment backend, this scorer is implemented as a PromptScorer that takes
-    1. a system role that may involve the Example object
-    2. options for scores on the example
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to check if scorer exists: {e.detail}",
+            response=e.response,
+        )
 
-    and uses a judge to execute the evaluation from the system role and classify into one of the options
-    """
 
+class BasePromptScorer(ABC, APIScorerConfig):
+    score_type: APIScorerType
     prompt: str
-    options: Mapping[str, float]
-    score_type: APIScorerType = APIScorerType.PROMPT_SCORER
+    options: Optional[Dict[str, float]] = None
+    description: Optional[str] = None
     judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or ""
     organization_id: str = os.getenv("JUDGMENT_ORG_ID") or ""
 
     @classmethod
+    @dont_throw
     def get(
         cls,
         name: str,
@@ -85,10 +115,24 @@ class PromptScorer(APIScorerConfig):
         organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
     ):
         scorer_config = fetch_prompt_scorer(name, judgment_api_key, organization_id)
+        if scorer_config["is_trace"] != issubclass(cls, TracePromptScorer):
+            raise JudgmentAPIError(
+                status_code=400,
+                detail=f"Scorer with name {name} is not a {cls.__name__}",
+                response=None,  # type: ignore
+            )
+        if issubclass(cls, TracePromptScorer):
+            score_type = APIScorerType.TRACE_PROMPT_SCORER
+        else:
+            score_type = APIScorerType.PROMPT_SCORER
         return cls(
+            score_type=score_type,
             name=name,
             prompt=scorer_config["prompt"],
-            options=scorer_config["options"],
+            threshold=scorer_config["threshold"],
+            options=scorer_config.get("options"),
+            model=scorer_config.get("model"),
+            description=scorer_config.get("description"),
             judgment_api_key=judgment_api_key,
             organization_id=organization_id,
         )
@@ -98,32 +142,51 @@ class PromptScorer(APIScorerConfig):
         cls,
         name: str,
         prompt: str,
-        options: Mapping[str, float],
+        threshold: float = 0.5,
+        options: Optional[Dict[str, float]] = None,
+        model: str = JUDGMENT_DEFAULT_GPT_MODEL,
+        description: Optional[str] = None,
         judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
         organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
     ):
         if not scorer_exists(name, judgment_api_key, organization_id):
-            push_prompt_scorer(name, prompt, options, judgment_api_key, organization_id)
+            if issubclass(cls, TracePromptScorer):
+                is_trace = True
+                score_type = APIScorerType.TRACE_PROMPT_SCORER
+            else:
+                is_trace = False
+                score_type = APIScorerType.PROMPT_SCORER
+            push_prompt_scorer(
+                name,
+                prompt,
+                threshold,
+                options,
+                model,
+                description,
+                judgment_api_key,
+                organization_id,
+                is_trace,
+            )
+            judgeval_logger.info(f"Successfully created PromptScorer: {name}")
             return cls(
+                score_type=score_type,
                 name=name,
                 prompt=prompt,
+                threshold=threshold,
                 options=options,
+                model=model,
+                description=description,
                 judgment_api_key=judgment_api_key,
                 organization_id=organization_id,
             )
         else:
             raise JudgmentAPIError(
-                f"Scorer with name {name} already exists. Either use the existing scorer with the get() method or use a new name."
+                status_code=400,
+                detail=f"Scorer with name {name} already exists. Either use the existing scorer with the get() method or use a new name.",
+                response=None,  # type: ignore
             )
 
     # Setter functions. Each setter function pushes the scorer to the DB.
-    def set_name(self, name: str):
-        """
-        Updates the name of the scorer.
-        """
-        self.name = name
-        self.push_prompt_scorer()
-
     def set_threshold(self, threshold: float):
         """
         Updates the threshold of the scorer.
@@ -140,16 +203,31 @@ class PromptScorer(APIScorerConfig):
         """
         self.prompt = prompt
         self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated prompt for {self.name}")
 
-    def set_options(self, options: Mapping[str, float]):
+    def set_model(self, model: str):
         """
-        Updates the options with the new options.
+        Updates the model of the scorer.
+        """
+        self.model = model
+        self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated model for {self.name}")
 
-        Sample options:
-        {"yes": 1, "no": 0}
+    def set_options(self, options: Optional[Dict[str, float]]):
+        """
+        Updates the options of the scorer.
         """
         self.options = options
         self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated options for {self.name}")
+
+    def set_description(self, description: Optional[str]):
+        """
+        Updates the description of the scorer.
+        """
+        self.description = description
+        self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated description for {self.name}")
 
     def append_to_prompt(self, prompt_addition: str):
         """
@@ -157,21 +235,40 @@ class PromptScorer(APIScorerConfig):
         """
         self.prompt += prompt_addition
         self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully appended to prompt for {self.name}")
 
     # Getters
-    def get_prompt(self) -> str | None:
+    def get_threshold(self) -> float:
+        """
+        Returns the threshold of the scorer.
+        """
+        return self.threshold
+
+    def get_prompt(self) -> str:
         """
         Returns the prompt of the scorer.
         """
         return self.prompt
 
-    def get_options(self) -> Mapping[str, float] | None:
+    def get_model(self) -> str:
+        """
+        Returns the model of the scorer.
+        """
+        return self.model
+
+    def get_options(self) -> Dict[str, float] | None:
         """
         Returns the options of the scorer.
         """
-        return self.options
+        return copy(self.options) if self.options is not None else None
+
+    def get_description(self) -> str | None:
+        """
+        Returns the description of the scorer.
+        """
+        return self.description
 
-    def get_name(self) -> str | None:
+    def get_name(self) -> str:
         """
         Returns the name of the scorer.
         """
@@ -183,8 +280,11 @@ class PromptScorer(APIScorerConfig):
         """
         return {
             "name": self.name,
+            "model": self.model,
             "prompt": self.prompt,
+            "threshold": self.threshold,
             "options": self.options,
+            "description": self.description,
         }
 
     def push_prompt_scorer(self):
@@ -194,13 +294,17 @@ class PromptScorer(APIScorerConfig):
         push_prompt_scorer(
             self.name,
             self.prompt,
+            self.threshold,
             self.options,
+            self.model,
+            self.description,
             self.judgment_api_key,
             self.organization_id,
+            isinstance(self, TracePromptScorer),
         )
 
     def __str__(self):
-        return f"PromptScorer(name={self.name}, prompt={self.prompt}, options={self.options})"
+        return f"PromptScorer(name={self.name}, model={self.model}, prompt={self.prompt}, threshold={self.threshold}, options={self.options}, description={self.description})"
 
     def model_dump(self, *args, **kwargs) -> Dict[str, Any]:
         base = super().model_dump(*args, **kwargs)
@@ -213,3 +317,11 @@ class PromptScorer(APIScorerConfig):
             k: getattr(self, k) for k in extra_fields if getattr(self, k) is not None
         }
         return base
+
+
+class PromptScorer(BasePromptScorer, ExampleAPIScorerConfig):
+    pass
+
+
+class TracePromptScorer(BasePromptScorer, TraceAPIScorerConfig):
+    pass

judgeval/scorers/score.py

@@ -1,5 +1,5 @@
 """
-Infrastructure for executing evaluations of `Example`s using one or more `BaseScorer`s.
+Infrastructure for executing evaluations of `Example`s using one or more `ExampleScorer`s.
 """
 
 import asyncio
@@ -13,61 +13,67 @@ from judgeval.data import (
     generate_scoring_result,
     create_scorer_data,
 )
-from judgeval.scorers import BaseScorer
+from judgeval.scorers.example_scorer import ExampleScorer
 from judgeval.scorers.utils import clone_scorers
-from judgeval.common.logger import judgeval_logger
+from judgeval.logger import judgeval_logger
 from judgeval.judges import JudgevalJudge
+from judgeval.env import JUDGMENT_DEFAULT_GPT_MODEL
 
 
 async def safe_a_score_example(
-    scorer: BaseScorer,
+    scorer: ExampleScorer,
     example: Example,
 ):
     """
     Scoring task function when not using a progress indicator!
-    "Safely" scores an `Example` using a `BaseScorer` by gracefully handling any exceptions that may occur.
+    "Safely" scores an `Example` using a `ExampleScorer` by gracefully handling any exceptions that may occur.
 
     Args:
-        scorer (BaseScorer): The `BaseScorer` to use for scoring the example.
+        scorer (ExampleScorer): The `ExampleScorer` to use for scoring the example.
         example (Example): The `Example` to be scored.
-
-        ignore_errors (bool): Whether to ignore errors during the evaluation.
-        If set to false, any error will be raised and stop the evaluation.
-        If set to true, the error will be stored in the `error` attribute of the `BaseScorer` and the `success` attribute will be set to False.
-
-        skip_on_missing_params (bool): Whether to skip the test case if required parameters are missing.
     """
     try:
-        scorer.score = await scorer.a_score_example(example)
+        score = await scorer.a_score_example(example)
+        if score is None:
+            raise Exception("a_score_example need to return a score")
+        elif score < 0:
+            judgeval_logger.warning("score cannot be less than 0 , setting to 0")
+            score = 0
+        elif score > 1:
+            judgeval_logger.warning("score cannot be greater than 1 , setting to 1")
+            score = 1
+        else:
+            scorer.score = score
         scorer.success = scorer.success_check()
     except Exception as e:
         judgeval_logger.error(f"Error during scoring: {str(e)}")
         scorer.error = str(e)
         scorer.success = False
+        scorer.score = 0
         return
 
 
 async def a_execute_scoring(
     examples: List[Example],
-    scorers: List[BaseScorer],
-    model: Optional[Union[str, List[str], JudgevalJudge]] = "gpt-4.1",
+    scorers: List[ExampleScorer],
+    model: Optional[Union[str, List[str], JudgevalJudge]] = JUDGMENT_DEFAULT_GPT_MODEL,
     ignore_errors: bool = False,
     throttle_value: int = 0,
     max_concurrent: int = 100,
+    show_progress: bool = True,
 ) -> List[ScoringResult]:
     """
-    Executes evaluations of `Example`s asynchronously using one or more `BaseScorer`s.
-    Each `Example` will be evaluated by all of the `BaseScorer`s in the `scorers` list.
+    Executes evaluations of `Example`s asynchronously using one or more `ExampleScorer`s.
+    Each `Example` will be evaluated by all of the `ExampleScorer`s in the `scorers` list.
 
     Args:
         examples (List[Example]): A list of `Example` objects to be evaluated.
-        scorers (List[BaseScorer]): A list of `BaseScorer` objects to evaluate the examples.
+        scorers (List[ExampleScorer]): A list of `ExampleScorer` objects to evaluate the examples.
         model (Union[str, List[str], JudgevalJudge]): The model to use for evaluation.
         ignore_errors (bool): Whether to ignore errors during evaluation.
         throttle_value (int): The amount of time to wait between starting each task.
         max_concurrent (int): The maximum number of concurrent tasks.
-
-        _use_bar_indicator (bool): Whether to use a progress bar indicator.
+        show_progress (bool): Whether to show the progress bar indicator.
 
     Returns:
         List[ScoringResult]: A list of `ScoringResult` objects containing the evaluation results.
@@ -82,33 +88,50 @@ async def a_execute_scoring(
             except Exception as e:
                 judgeval_logger.error(f"Error executing function: {e}")
                 if kwargs.get("ignore_errors", False):
-                    # Simply return None when ignoring errors, as expected by the test
                     return None
-                # If we're not ignoring errors, propagate the exception
                 raise
 
-    # Add model to scorers
     for scorer in scorers:
-        if not scorer.model:
+        if not scorer.model and isinstance(model, str):
             scorer._add_model(model)
 
-    scoring_results: List[ScoringResult] = [None for _ in examples]
+    scoring_results: List[Optional[ScoringResult]] = [None for _ in examples]
     tasks = []
-    cloned_scorers: List[BaseScorer]
-
-    with tqdm_asyncio(
-        desc=f"Evaluating {len(examples)} example(s) in parallel",
-        unit="Example",
-        total=len(examples),
-        bar_format="{desc}: |{bar}|{percentage:3.0f}% ({n_fmt}/{total_fmt}) [Time Taken: {elapsed}, {rate_fmt}{postfix}]",
-    ) as pbar:
+
+    if show_progress:
+        with tqdm_asyncio(
+            desc=f"Evaluating {len(examples)} example(s) in parallel",
+            unit="Example",
+            total=len(examples),
+            bar_format="{desc}: |{bar}|{percentage:3.0f}% ({n_fmt}/{total_fmt}) [Time Taken: {elapsed}, {rate_fmt}{postfix}]",
+        ) as pbar:
+            for i, ex in enumerate(examples):
+                if isinstance(ex, Example):
+                    if len(scorers) == 0:
+                        pbar.update(1)
+                        continue
+
+                    cloned_scorers = clone_scorers(scorers)  # type: ignore
+                    task = execute_with_semaphore(
+                        func=a_eval_examples_helper,
+                        scorers=cloned_scorers,
+                        example=ex,
+                        scoring_results=scoring_results,
+                        score_index=i,
+                        ignore_errors=ignore_errors,
+                        pbar=pbar,
+                    )
+                    tasks.append(asyncio.create_task(task))
+
+                await asyncio.sleep(throttle_value)
+            await asyncio.gather(*tasks)
+    else:
         for i, ex in enumerate(examples):
             if isinstance(ex, Example):
                 if len(scorers) == 0:
-                    pbar.update(1)
                     continue
 
-                cloned_scorers = clone_scorers(scorers)
+                cloned_scorers = clone_scorers(scorers)  # type: ignore
                 task = execute_with_semaphore(
                     func=a_eval_examples_helper,
                     scorers=cloned_scorers,
@@ -116,19 +139,19 @@ async def a_execute_scoring(
                     scoring_results=scoring_results,
                     score_index=i,
                     ignore_errors=ignore_errors,
-                    pbar=pbar,
+                    pbar=None,
                 )
                 tasks.append(asyncio.create_task(task))
 
             await asyncio.sleep(throttle_value)
         await asyncio.gather(*tasks)
-    return scoring_results
+    return [result for result in scoring_results if result is not None]
 
 
 async def a_eval_examples_helper(
-    scorers: List[BaseScorer],
+    scorers: List[ExampleScorer],
     example: Example,
-    scoring_results: List[ScoringResult],
+    scoring_results: List[Optional[ScoringResult]],
     score_index: int,
     ignore_errors: bool,
     pbar: Optional[tqdm_asyncio] = None,
@@ -137,7 +160,7 @@ async def a_eval_examples_helper(
     Evaluate a single example asynchronously using a list of scorers.
 
     Args:
-        scorers (List[BaseScorer]): List of BaseScorer objects to evaluate the example.
+        scorers (List[ExampleScorer]): List of ExampleScorer objects to evaluate the example.
         example (Example): The example to be evaluated.
         scoring_results (List[ScoringResult]): List to store the scoring results.
         score_index (int): Index at which the result should be stored in scoring_results.
@@ -147,24 +170,18 @@ async def a_eval_examples_helper(
         None
     """
 
-    # scoring the Example
     scoring_start_time = time.perf_counter()
 
     tasks = [safe_a_score_example(scorer, example) for scorer in scorers]
 
     await asyncio.gather(*tasks)
 
-    # Now that all the scoring functions of each scorer have executed, we collect
-    # the results and update the ScoringResult with the scorer data
     success = True
     scorer_data_list = []
     for scorer in scorers:
-        # At this point, the scorer has been executed and already contains data.
         if getattr(scorer, "skipped", False):
             continue
-        scorer_data = create_scorer_data(
-            scorer
-        )  # Fetch scorer data from completed scorer evaluation
+        scorer_data = create_scorer_data(scorer)
         for s in scorer_data:
             success = success and s.success
         scorer_data_list.extend(scorer_data)