judgeval 0.0.52__py3-none-any.whl → 0.0.54__py3-none-any.whl

judgeval/scorers/agent_scorer.py

@@ -0,0 +1,21 @@
+from judgeval.scorers.base_scorer import BaseScorer
+from judgeval.data import Trace
+from typing import List, Optional
+from abc import abstractmethod
+
+from judgeval.common.logger import warning, error
+
+
+class AgentScorer(BaseScorer):
+    @abstractmethod
+    async def a_score_trace(
+        self, trace: Trace, tools: Optional[List] = None, *args, **kwargs
+    ) -> float:
+        """
+        Asynchronously measures the score on a trace
+        """
+        warning("Attempting to call unimplemented a_score_trace method")
+        error("a_score_trace method not implemented")
+        raise NotImplementedError(
+            "You must implement the `a_score_trace` method in your custom scorer"
+        )

judgeval/scorers/api_scorer.py

@@ -6,27 +6,35 @@ Scores `Example`s using ready-made Judgment evaluators.
 
 from pydantic import BaseModel, field_validator
 from typing import List
-from judgeval.common.logger import debug, info, error
 from judgeval.data import ExampleParams
-from judgeval.constants import APIScorer, UNBOUNDED_SCORERS
+from judgeval.constants import APIScorerType, UNBOUNDED_SCORERS
+from judgeval.common.logger import judgeval_logger
 
 
-class APIJudgmentScorer(BaseModel):
+class APIScorerConfig(BaseModel):
     """
-    Class for ready-made, "out-of-the-box" scorer that uses Judgment evaluators to score `Example`s.
+    Scorer config that is used to send to our Judgment server.
 
     Args:
         score_type (APIScorer): The Judgment metric to use for scoring `Example`s
+        name (str): The name of the scorer, usually this is the same as the score_type
         threshold (float): A value between 0 and 1 that determines the scoring threshold
+        strict_mode (bool): Whether to use strict mode for the scorer
+        required_params (List[ExampleParams]): List of the required parameters on examples for the scorer
+        kwargs (dict): Additional keyword arguments to pass to the scorer
     """
 
-    score_type: APIScorer
-    threshold: float
+    score_type: APIScorerType
+    name: str = ""
+    threshold: float = 0.5
+    strict_mode: bool = False
     required_params: List[
         ExampleParams
-    ] = []  # List of the required parameters on examples for the scorer
+    ] = []  # This is used to check if the example has the required parameters before running the scorer
+    kwargs: dict = {}
 
     @field_validator("threshold")
+    @classmethod
     def validate_threshold(cls, v, info):
         """
         Validates that the threshold is between 0 and 1 inclusive.
@@ -34,47 +42,29 @@ class APIJudgmentScorer(BaseModel):
         score_type = info.data.get("score_type")
         if score_type in UNBOUNDED_SCORERS:
             if v < 0:
-                error(f"Threshold for {score_type} must be greater than 0, got: {v}")
+                judgeval_logger.error(
+                    f"Threshold for {score_type} must be greater than 0, got: {v}"
+                )
                 raise ValueError(
                     f"Threshold for {score_type} must be greater than 0, got: {v}"
                 )
         else:
             if not 0 <= v <= 1:
-                error(f"Threshold for {score_type} must be between 0 and 1, got: {v}")
+                judgeval_logger.error(
+                    f"Threshold for {score_type} must be between 0 and 1, got: {v}"
+                )
                 raise ValueError(
                     f"Threshold for {score_type} must be between 0 and 1, got: {v}"
                 )
         return v
 
-    @field_validator("score_type")
-    def convert_to_enum_value(cls, v):
-        """
-        Validates that the `score_type` is a valid `APIScorer` enum value.
-        Converts string values to `APIScorer` enum values.
-        """
-        debug(f"Attempting to convert score_type value: {v}")
-        if isinstance(v, APIScorer):
-            info(f"Using existing APIScorer: {v}")
-            return v
-        elif isinstance(v, str):
-            debug(f"Converting string value to APIScorer enum: {v}")
-            return APIScorer[v.upper()]
-        error(f"Invalid score_type value: {v}")
-        raise ValueError(f"Invalid value for score_type: {v}")
+    @field_validator("name", mode="after")
+    @classmethod
+    def set_name_to_score_type_if_none(cls, v, info):
+        """Set name to score_type if not provided"""
+        if v is None:
+            return info.data.get("score_type")
+        return v
 
     def __str__(self):
         return f"JudgmentScorer(score_type={self.score_type.value}, threshold={self.threshold})"
-
-    def to_dict(self) -> dict:
-        """
-        Converts the scorer configuration to a dictionary format.
-
-        Returns:
-            dict: A dictionary containing the scorer's configuration
-        """
-        return {
-            "score_type": str(
-                self.score_type.value
-            ),  # Convert enum to string for serialization
-            "threshold": self.threshold,
-        }

judgeval/scorers/base_scorer.py

@@ -0,0 +1,98 @@
+"""
+Base class for all scorers.
+"""
+
+from typing import Dict, Optional
+
+from pydantic import BaseModel
+
+
+from judgeval.judges.utils import create_judge
+from typing import Any
+from pydantic import model_validator, Field
+from judgeval.common.logger import judgeval_logger
+
+
+class BaseScorer(BaseModel):
+    """
+    If you want to create a scorer that does not fall under any of the ready-made Judgment scorers,
+    you can create a custom scorer by extending this class. This is best used for special use cases
+    where none of Judgment's scorers are suitable.
+    """
+
+    score_type: str  # type of your scorer (Faithfulness, PromptScorer)
+    threshold: float = (
+        0.5  # The threshold to pass a test while using this scorer as a scorer
+    )
+    name: Optional[str] = (
+        None  # name of your scorer (Faithfulness, PromptScorer-randomslug)
+    )
+    score: Optional[float] = None  # The float score of the scorer run on the test case
+    score_breakdown: Optional[Dict] = None
+    reason: Optional[str] = ""
+    using_native_model: Optional[bool] = None  # Whether the model is a native model
+    success: Optional[bool] = None  # Whether the test case passed or failed
+    model: Optional[Any] = Field(
+        default=None, exclude=True
+    )  # The model used to evaluate the test case
+    evaluation_model: Optional[str] = None  # The model used to evaluate the test case
+    strict_mode: bool = False  # Whether to run the scorer in strict mode
+    error: Optional[str] = None  # The error message if the scorer failed
+    additional_metadata: Optional[Dict] = None  # Additional metadata for the scorer
+    user: Optional[str] = None  # The user ID of the scorer
+
+    @model_validator(mode="before")
+    @classmethod
+    def enforce_strict_threshold(cls, data: dict):
+        if data.get("strict_mode"):
+            data["threshold"] = 1.0
+        return data
+
+    @model_validator(mode="after")
+    @classmethod
+    def default_name(cls, m: "BaseScorer") -> "BaseScorer":
+        if not m.name:
+            # Try to use the class name if it exists and is not empty
+            class_name = getattr(m, "__class__", None)
+            if class_name and getattr(m.__class__, "__name__", None):
+                m.name = m.__class__.__name__
+            else:
+                m.name = m.score_type
+        return m
+
+    def _add_model(self, model: str):
+        """
+        Adds the evaluation model to the BaseScorer instance
+
+        This method is used at eval time
+        """
+        self.model, self.using_native_model = create_judge(model)
+        self.evaluation_model = self.model.get_model_name()
+
+    def success_check(self) -> bool:
+        """
+        For unit testing, determines whether the test case passes or fails
+        """
+        if self.error:
+            return False
+        if self.score is None:
+            return False
+        return self.score >= self.threshold
+
+    def __str__(self):
+        if self.error:
+            judgeval_logger.warning(f"BaseScorer contains error: {self.error}")
+        attributes = {
+            "score_type": self.score_type,
+            "threshold": self.threshold,
+            "score": self.score,
+            "score_breakdown": self.score_breakdown,
+            "reason": self.reason,
+            "success": self.success,
+            "model": self.model,
+            "evaluation_model": self.evaluation_model,
+            "strict_mode": self.strict_mode,
+            "error": self.error,
+            "additional_metadata": self.additional_metadata,
+        }
+        return f"BaseScorer({attributes})"

judgeval/scorers/example_scorer.py

@@ -0,0 +1,19 @@
+from judgeval.scorers.base_scorer import BaseScorer
+from judgeval.data import Example
+from typing import List
+from pydantic import Field
+from judgeval.common.logger import judgeval_logger
+
+
+class ExampleScorer(BaseScorer):
+    score_type: str = "Custom"  # default to custom score type
+    required_params: List[str] = Field(default_factory=list)
+
+    async def a_score_example(self, example: Example, *args, **kwargs) -> float:
+        """
+        Asynchronously measures the score on a single example
+        """
+        judgeval_logger.error("a_score_example method not implemented")
+        raise NotImplementedError(
+            "You must implement the `a_score_example` method in your custom scorer"
+        )

judgeval/scorers/judgeval_scorers/api_scorers/__init__.py

@@ -1,40 +1,21 @@
 from judgeval.scorers.judgeval_scorers.api_scorers.execution_order import (
     ExecutionOrderScorer,
 )
-from judgeval.scorers.judgeval_scorers.api_scorers.json_correctness import (
-    JSONCorrectnessScorer,
-)
-from judgeval.scorers.judgeval_scorers.api_scorers.summarization import (
-    SummarizationScorer,
-)
 from judgeval.scorers.judgeval_scorers.api_scorers.hallucination import (
     HallucinationScorer,
 )
 from judgeval.scorers.judgeval_scorers.api_scorers.faithfulness import (
     FaithfulnessScorer,
 )
-from judgeval.scorers.judgeval_scorers.api_scorers.contextual_relevancy import (
-    ContextualRelevancyScorer,
-)
-from judgeval.scorers.judgeval_scorers.api_scorers.contextual_precision import (
-    ContextualPrecisionScorer,
-)
-from judgeval.scorers.judgeval_scorers.api_scorers.contextual_recall import (
-    ContextualRecallScorer,
-)
 from judgeval.scorers.judgeval_scorers.api_scorers.answer_relevancy import (
     AnswerRelevancyScorer,
 )
 from judgeval.scorers.judgeval_scorers.api_scorers.answer_correctness import (
     AnswerCorrectnessScorer,
 )
-from judgeval.scorers.judgeval_scorers.api_scorers.comparison import ComparisonScorer
 from judgeval.scorers.judgeval_scorers.api_scorers.instruction_adherence import (
     InstructionAdherenceScorer,
 )
-from judgeval.scorers.judgeval_scorers.api_scorers.groundedness import (
-    GroundednessScorer,
-)
 from judgeval.scorers.judgeval_scorers.api_scorers.derailment_scorer import (
     DerailmentScorer,
 )
@@ -57,7 +38,6 @@ __all__ = [
     "ContextualRecallScorer",
     "AnswerRelevancyScorer",
     "AnswerCorrectnessScorer",
-    "ComparisonScorer",
     "InstructionAdherenceScorer",
     "GroundednessScorer",
     "DerailmentScorer",

judgeval/scorers/judgeval_scorers/api_scorers/answer_correctness.py

@@ -6,23 +6,16 @@ TODO add link to docs page for this scorer
 """
 
 # Internal imports
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
+from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.constants import APIScorerType
 from judgeval.data import ExampleParams
+from typing import List
 
 
-class AnswerCorrectnessScorer(APIJudgmentScorer):
-    def __init__(self, threshold: float):
-        super().__init__(
-            threshold=threshold,
-            score_type=APIScorer.ANSWER_CORRECTNESS,
-            required_params=[
-                ExampleParams.INPUT,
-                ExampleParams.ACTUAL_OUTPUT,
-                ExampleParams.EXPECTED_OUTPUT,
-            ],
-        )
-
-    @property
-    def __name__(self):
-        return "Answer Correctness"
+class AnswerCorrectnessScorer(APIScorerConfig):
+    score_type: APIScorerType = APIScorerType.ANSWER_CORRECTNESS
+    required_params: List[ExampleParams] = [
+        ExampleParams.INPUT,
+        ExampleParams.ACTUAL_OUTPUT,
+        ExampleParams.EXPECTED_OUTPUT,
+    ]

judgeval/scorers/judgeval_scorers/api_scorers/answer_relevancy.py

@@ -1,27 +1,12 @@
-"""
-`judgeval` answer relevancy scorer
-
-TODO add link to docs page for this scorer
-
-"""
-
-# Internal imports
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
+from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.constants import APIScorerType
 from judgeval.data import ExampleParams
+from typing import List
 
 
-class AnswerRelevancyScorer(APIJudgmentScorer):
-    def __init__(self, threshold: float):
-        super().__init__(
-            threshold=threshold,
-            score_type=APIScorer.ANSWER_RELEVANCY,
-            required_params=[
-                ExampleParams.INPUT,
-                ExampleParams.ACTUAL_OUTPUT,
-            ],
-        )
-
-    @property
-    def __name__(self):
-        return "Answer Relevancy"
+class AnswerRelevancyScorer(APIScorerConfig):
+    score_type: APIScorerType = APIScorerType.ANSWER_RELEVANCY
+    required_params: List[ExampleParams] = [
+        ExampleParams.INPUT,
+        ExampleParams.ACTUAL_OUTPUT,
+    ]

judgeval/scorers/judgeval_scorers/api_scorers/classifier_scorer.py

@@ -1,9 +1,9 @@
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
-from typing import List, Mapping, Optional
+from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.constants import APIScorerType
+from typing import List, Mapping, Optional, Dict, Any
 
 
-class ClassifierScorer(APIJudgmentScorer):
+class ClassifierScorer(APIScorerConfig):
     """
     In the Judgment backend, this scorer is implemented as a PromptScorer that takes
     1. a system role that may involve the Example object
@@ -16,50 +16,15 @@ class ClassifierScorer(APIJudgmentScorer):
     options = {"positive": 1, "negative": 0}
 
     Args:
-        name (str): The name of the scorer
         slug (str): A unique identifier for the scorer
         conversation (List[dict]): The conversation template with placeholders (e.g., {{actual_output}})
         options (Mapping[str, float]): A mapping of classification options to their corresponding scores
-        threshold (float): The threshold for determining success (default: 0.5)
-        include_reason (bool): Whether to include reasoning in the response (default: True)
-        strict_mode (bool): Whether to use strict mode (default: False)
-        verbose_mode (bool): Whether to include verbose logging (default: False)
     """
 
-    name: Optional[str] = None
     slug: Optional[str] = None
     conversation: Optional[List[dict]] = None
     options: Optional[Mapping[str, float]] = None
-    verbose_mode: bool = False
-    strict_mode: bool = False
-    include_reason: bool = True
-    async_mode: bool = True
-    threshold: float = 0.5
-
-    def __init__(
-        self,
-        name: str,
-        slug: str,
-        conversation: List[dict],
-        options: Mapping[str, float],
-        threshold: float = 0.5,
-        include_reason: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-        async_mode: bool = True,
-    ):
-        super().__init__(
-            threshold=threshold,
-            score_type=APIScorer.CLASSIFIER,
-        )
-        self.name = name
-        self.verbose_mode = verbose_mode
-        self.strict_mode = strict_mode
-        self.include_reason = include_reason
-        self.slug = slug
-        self.conversation = conversation
-        self.options = options
-        self.async_mode = async_mode
+    score_type: APIScorerType = APIScorerType.PROMPT_SCORER
 
     def update_name(self, name: str):
         """
@@ -94,32 +59,15 @@ class ClassifierScorer(APIJudgmentScorer):
     def __str__(self):
         return f"ClassifierScorer(name={self.name}, slug={self.slug}, conversation={self.conversation}, threshold={self.threshold}, options={self.options})"
 
-    # @model_serializer
-    # def serialize_model(self) -> dict:
-    #     """
-    #     Defines how the ClassifierScorer should be serialized when model_dump() is called.
-    #     """
-    #     return {
-    #         "name": self.name,
-    #         "score_type": self.name,
-    #         "conversation": self.conversation,
-    #         "options": self.options,
-    #         "threshold": self.threshold,
-    #         "include_reason": self.include_reason,
-    #         "async_mode": self.async_mode,
-    #         "strict_mode": self.strict_mode,
-    #         "verbose_mode": self.verbose_mode,
-    #     }
-
-    def to_dict(self) -> dict:
-        return {
-            "name": self.name,
-            "score_type": self.name,
-            "conversation": self.conversation,
-            "options": self.options,
-            "threshold": self.threshold,
-            "include_reason": self.include_reason,
-            "async_mode": self.async_mode,
-            "strict_mode": self.strict_mode,
-            "verbose_mode": self.verbose_mode,
+    def model_dump(self, *args, **kwargs) -> Dict[str, Any]:
+        base = super().model_dump(*args, **kwargs)
+        base_fields = set(APIScorerConfig.model_fields.keys())
+        all_fields = set(self.__class__.model_fields.keys())
+
+        extra_fields = all_fields - base_fields - {"kwargs"}
+
+        base["kwargs"] = {
+            k: getattr(self, k) for k in extra_fields if getattr(self, k) is not None
         }
+
+        return base

judgeval/scorers/judgeval_scorers/api_scorers/derailment_scorer.py

@@ -6,17 +6,9 @@ TODO add link to docs page for this scorer
 """
 
 # Internal imports
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
+from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.constants import APIScorerType
 
 
-class DerailmentScorer(APIJudgmentScorer):
-    def __init__(self, threshold: float):
-        super().__init__(
-            threshold=threshold,
-            score_type=APIScorer.DERAILMENT,
-        )
-
-    @property
-    def __name__(self):
-        return "Derailment"
+class DerailmentScorer(APIScorerConfig):
+    score_type: APIScorerType = APIScorerType.DERAILMENT

judgeval/scorers/judgeval_scorers/api_scorers/execution_order.py

@@ -6,13 +6,13 @@ TODO add link to docs page for this scorer
 """
 
 # Internal imports
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
+from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.constants import APIScorerType
 from typing import Optional, Dict
 from judgeval.data import ExampleParams
 
 
-class ExecutionOrderScorer(APIJudgmentScorer):
+class ExecutionOrderScorer(APIScorerConfig):
     kwargs: Optional[Dict] = None
 
     def __init__(
@@ -23,7 +23,7 @@ class ExecutionOrderScorer(APIJudgmentScorer):
     ):
         super().__init__(
             threshold=threshold,
-            score_type=APIScorer.EXECUTION_ORDER,
+            score_type=APIScorerType.EXECUTION_ORDER,
             required_params=[
                 ExampleParams.ACTUAL_OUTPUT,
                 ExampleParams.EXPECTED_OUTPUT,

judgeval/scorers/judgeval_scorers/api_scorers/faithfulness.py

@@ -6,23 +6,16 @@ TODO add link to docs page for this scorer
 """
 
 # Internal imports
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
+from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.constants import APIScorerType
 from judgeval.data import ExampleParams
+from typing import List
 
 
-class FaithfulnessScorer(APIJudgmentScorer):
-    def __init__(self, threshold: float):
-        super().__init__(
-            threshold=threshold,
-            score_type=APIScorer.FAITHFULNESS,
-            required_params=[
-                ExampleParams.INPUT,
-                ExampleParams.ACTUAL_OUTPUT,
-                ExampleParams.RETRIEVAL_CONTEXT,
-            ],
-        )
-
-    @property
-    def __name__(self):
-        return "Faithfulness"
+class FaithfulnessScorer(APIScorerConfig):
+    score_type: APIScorerType = APIScorerType.FAITHFULNESS
+    required_params: List[ExampleParams] = [
+        ExampleParams.INPUT,
+        ExampleParams.ACTUAL_OUTPUT,
+        ExampleParams.RETRIEVAL_CONTEXT,
+    ]

judgeval/scorers/judgeval_scorers/api_scorers/hallucination.py

@@ -6,16 +6,16 @@ TODO add link to docs page for this scorer
 """
 
 # Internal imports
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
+from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.constants import APIScorerType
 from judgeval.data import ExampleParams
 
 
-class HallucinationScorer(APIJudgmentScorer):
+class HallucinationScorer(APIScorerConfig):
     def __init__(self, threshold: float):
         super().__init__(
             threshold=threshold,
-            score_type=APIScorer.HALLUCINATION,
+            score_type=APIScorerType.HALLUCINATION,
             required_params=[
                 ExampleParams.INPUT,
                 ExampleParams.ACTUAL_OUTPUT,

judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py

@@ -6,16 +6,16 @@ TODO add link to docs page for this scorer
 """
 
 # Internal imports
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
+from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.constants import APIScorerType
 from judgeval.data import ExampleParams
 
 
-class InstructionAdherenceScorer(APIJudgmentScorer):
+class InstructionAdherenceScorer(APIScorerConfig):
     def __init__(self, threshold: float):
         super().__init__(
             threshold=threshold,
-            score_type=APIScorer.INSTRUCTION_ADHERENCE,
+            score_type=APIScorerType.INSTRUCTION_ADHERENCE,
             required_params=[
                 ExampleParams.INPUT,
                 ExampleParams.ACTUAL_OUTPUT,

judgeval/scorers/judgeval_scorers/api_scorers/tool_dependency.py

@@ -3,16 +3,16 @@
 """
 
 # Internal imports
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
+from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.constants import APIScorerType
 from typing import Optional, Dict
 
 
-class ToolDependencyScorer(APIJudgmentScorer):
+class ToolDependencyScorer(APIScorerConfig):
     kwargs: Optional[Dict] = None
 
     def __init__(self, threshold: float = 1.0, enable_param_checking: bool = True):
-        super().__init__(threshold=threshold, score_type=APIScorer.TOOL_DEPENDENCY)
+        super().__init__(threshold=threshold, score_type=APIScorerType.TOOL_DEPENDENCY)
         self.kwargs = {"enable_param_checking": enable_param_checking}
 
     @property

judgeval/scorers/judgeval_scorers/api_scorers/tool_order.py

@@ -3,21 +3,25 @@
 """
 
 # Internal imports
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
-from typing import Optional, Dict
+from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.constants import APIScorerType
+from typing import Dict, Any
 
 
-class ToolOrderScorer(APIJudgmentScorer):
-    kwargs: Optional[Dict] = None
+class ToolOrderScorer(APIScorerConfig):
+    score_type: APIScorerType = APIScorerType.TOOL_ORDER
+    threshold: float = 1.0
+    exact_match: bool = False
 
-    def __init__(self, threshold: float = 1.0, exact_match: bool = False):
-        super().__init__(
-            threshold=threshold,
-            score_type=APIScorer.TOOL_ORDER,
-        )
-        self.kwargs = {"exact_match": exact_match}
+    def model_dump(self, *args, **kwargs) -> Dict[str, Any]:
+        base = super().model_dump(*args, **kwargs)
+        base_fields = set(APIScorerConfig.model_fields.keys())
+        all_fields = set(self.__class__.model_fields.keys())
 
-    @property
-    def __name__(self):
-        return "Tool Order"
+        extra_fields = all_fields - base_fields - {"kwargs"}
+
+        base["kwargs"] = {
+            k: getattr(self, k) for k in extra_fields if getattr(self, k) is not None
+        }
+
+        return base