judgeval 0.1.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

judgeval/scorers/agent_scorer.py

@@ -1,21 +1,17 @@
-from judgeval.scorers.base_scorer import BaseScorer
-from judgeval.data import Trace
-from typing import List, Optional
-from abc import abstractmethod
+# from judgeval.scorers.base_scorer import BaseScorer
+# from judgeval.data.judgment_types import Trace as JudgmentTrace
+# from typing import List, Optional
+# from abc import abstractmethod
 
-from judgeval.common.logger import warning, error
 
-
-class AgentScorer(BaseScorer):
-    @abstractmethod
-    async def a_score_trace(
-        self, trace: Trace, tools: Optional[List] = None, *args, **kwargs
-    ) -> float:
-        """
-        Asynchronously measures the score on a trace
-        """
-        warning("Attempting to call unimplemented a_score_trace method")
-        error("a_score_trace method not implemented")
-        raise NotImplementedError(
-            "You must implement the `a_score_trace` method in your custom scorer"
-        )
+# class TraceScorer(BaseScorer):
+#     @abstractmethod
+#     async def a_score_trace(
+#         self, trace: JudgmentTrace, tools: Optional[List] = None, *args, **kwargs
+#     ) -> float:
+#         """
+#         Asynchronously measures the score on a trace
+#         """
+#         raise NotImplementedError(
+#             "You must implement the `a_score_trace` method in your custom scorer"
+#         )

judgeval/scorers/api_scorer.py

@@ -4,11 +4,13 @@ Judgment Scorer class.
 Scores `Example`s using ready-made Judgment evaluators.
 """
 
+from __future__ import annotations
+
 from pydantic import BaseModel, field_validator
 from typing import List
-from judgeval.data import ExampleParams
-from judgeval.constants import APIScorerType, UNBOUNDED_SCORERS
-from judgeval.common.logger import judgeval_logger
+from judgeval.constants import APIScorerType
+from judgeval.data.example import ExampleParams
+from judgeval.env import JUDGMENT_DEFAULT_GPT_MODEL
 
 
 class APIScorerConfig(BaseModel):
@@ -28,9 +30,10 @@ class APIScorerConfig(BaseModel):
     name: str = ""
     threshold: float = 0.5
     strict_mode: bool = False
-    required_params: List[
-        ExampleParams
-    ] = []  # This is used to check if the example has the required parameters before running the scorer
+    model: str = JUDGMENT_DEFAULT_GPT_MODEL
+
+    required_params: List[ExampleParams] = []
+
     kwargs: dict = {}
 
     @field_validator("threshold")
@@ -40,31 +43,26 @@ class APIScorerConfig(BaseModel):
         Validates that the threshold is between 0 and 1 inclusive.
         """
         score_type = info.data.get("score_type")
-        if score_type in UNBOUNDED_SCORERS:
-            if v < 0:
-                judgeval_logger.error(
-                    f"Threshold for {score_type} must be greater than 0, got: {v}"
-                )
-                raise ValueError(
-                    f"Threshold for {score_type} must be greater than 0, got: {v}"
-                )
-        else:
-            if not 0 <= v <= 1:
-                judgeval_logger.error(
-                    f"Threshold for {score_type} must be between 0 and 1, got: {v}"
-                )
-                raise ValueError(
-                    f"Threshold for {score_type} must be between 0 and 1, got: {v}"
-                )
+        if not 0 <= v <= 1:
+            raise ValueError(
+                f"Threshold for {score_type} must be between 0 and 1, got: {v}"
+            )
         return v
 
     @field_validator("name", mode="after")
     @classmethod
     def set_name_to_score_type_if_none(cls, v, info):
-        """Set name to score_type if not provided"""
         if v is None:
             return info.data.get("score_type")
         return v
 
     def __str__(self):
         return f"JudgmentScorer(score_type={self.score_type.value}, threshold={self.threshold})"
+
+
+class ExampleAPIScorerConfig(APIScorerConfig):
+    pass
+
+
+class TraceAPIScorerConfig(APIScorerConfig):
+    pass

judgeval/scorers/base_scorer.py

@@ -2,6 +2,7 @@
 Base class for all scorers.
 """
 
+from __future__ import annotations
 from typing import Dict, Optional
 
 from pydantic import BaseModel
@@ -19,45 +20,63 @@ class BaseScorer(BaseModel):
     where none of Judgment's scorers are suitable.
     """
 
-    score_type: str  # type of your scorer (Faithfulness, PromptScorer)
-    threshold: float = (
-        0.5  # The threshold to pass a test while using this scorer as a scorer
-    )
-    name: Optional[str] = (
-        None  # name of your scorer (Faithfulness, PromptScorer-randomslug)
-    )
-    score: Optional[float] = None  # The float score of the scorer run on the test case
+    # type of your scorer (Faithfulness, PromptScorer)
+    score_type: str
+
+    # The threshold to pass a test while using this scorer as a scorer
+    threshold: float = 0.5
+
+    # name of your scorer (Faithfulness, PromptScorer-randomslug)
+    name: str = ""
+
+    # The name of the class of the scorer
+    class_name: Optional[str] = None
+
+    # The float score of the scorer run on the test case
+    score: Optional[float] = None
+
     score_breakdown: Optional[Dict] = None
     reason: Optional[str] = ""
-    using_native_model: Optional[bool] = None  # Whether the model is a native model
-    success: Optional[bool] = None  # Whether the test case passed or failed
-    model: Optional[str] = None  # The name of the model used to evaluate the test case
-    model_client: Optional[Any] = Field(
-        default=None, exclude=True
-    )  # The model used to evaluate the test case
-    strict_mode: bool = False  # Whether to run the scorer in strict mode
-    error: Optional[str] = None  # The error message if the scorer failed
-    additional_metadata: Optional[Dict] = None  # Additional metadata for the scorer
-    user: Optional[str] = None  # The user ID of the scorer
-
-    @model_validator(mode="before")
-    @classmethod
-    def enforce_strict_threshold(cls, data: dict):
-        if data.get("strict_mode"):
-            data["threshold"] = 1.0
-        return data
+
+    # Whether the model is a native model
+    using_native_model: Optional[bool] = None
+
+    # Whether the test case passed or failed
+    success: bool = False
+
+    # The name of the model used to evaluate the test case
+    model: Optional[str] = None
+
+    # The model used to evaluate the test case
+    model_client: Optional[Any] = Field(default=None, exclude=True)
+
+    # Whether to run the scorer in strict mode
+    strict_mode: bool = False
+
+    # The error message if the scorer failed
+    error: Optional[str] = None
+
+    # Additional metadata for the scorer
+    additional_metadata: Optional[Dict] = None
+
+    # The user ID of the scorer
+    user: Optional[str] = None
+
+    # Whether the scorer is hosted on the server
+    server_hosted: bool = False
+
+    @model_validator(mode="after")
+    def enforce_strict_threshold(self):
+        if self.strict_mode:
+            self.threshold = 1.0
+        return self
 
     @model_validator(mode="after")
-    @classmethod
-    def default_name(cls, m: "BaseScorer") -> "BaseScorer":
-        if not m.name:
-            # Try to use the class name if it exists and is not empty
-            class_name = getattr(m, "__class__", None)
-            if class_name and getattr(m.__class__, "__name__", None):
-                m.name = m.__class__.__name__
-            else:
-                m.name = m.score_type
-        return m
+    def default_name(self):
+        self.class_name = self.__class__.__name__
+        if not self.name:
+            self.name = self.class_name
+        return self
 
     def _add_model(self, model: str):
         """
@@ -66,7 +85,6 @@ class BaseScorer(BaseModel):
         This method is used at eval time
         """
         self.model_client, self.using_native_model = create_judge(model)
-        self.model = self.model_client.get_model_name() or model
 
     def success_check(self) -> bool:
         """

judgeval/scorers/example_scorer.py

@@ -2,18 +2,16 @@ from judgeval.scorers.base_scorer import BaseScorer
 from judgeval.data import Example
 from typing import List
 from pydantic import Field
-from judgeval.common.logger import judgeval_logger
 
 
 class ExampleScorer(BaseScorer):
-    score_type: str = "Custom"  # default to custom score type
+    score_type: str = "Custom"
     required_params: List[str] = Field(default_factory=list)
 
     async def a_score_example(self, example: Example, *args, **kwargs) -> float:
         """
         Asynchronously measures the score on a single example
         """
-        judgeval_logger.error("a_score_example method not implemented")
         raise NotImplementedError(
             "You must implement the `a_score_example` method in your custom scorer"
         )

judgeval/scorers/judgeval_scorers/api_scorers/__init__.py

@@ -1,9 +1,3 @@
-from judgeval.scorers.judgeval_scorers.api_scorers.execution_order import (
-    ExecutionOrderScorer,
-)
-from judgeval.scorers.judgeval_scorers.api_scorers.hallucination import (
-    HallucinationScorer,
-)
 from judgeval.scorers.judgeval_scorers.api_scorers.faithfulness import (
     FaithfulnessScorer,
 )
@@ -16,32 +10,16 @@ from judgeval.scorers.judgeval_scorers.api_scorers.answer_correctness import (
 from judgeval.scorers.judgeval_scorers.api_scorers.instruction_adherence import (
     InstructionAdherenceScorer,
 )
-from judgeval.scorers.judgeval_scorers.api_scorers.derailment_scorer import (
-    DerailmentScorer,
-)
-from judgeval.scorers.judgeval_scorers.api_scorers.tool_order import ToolOrderScorer
 from judgeval.scorers.judgeval_scorers.api_scorers.prompt_scorer import (
+    TracePromptScorer,
     PromptScorer,
 )
-from judgeval.scorers.judgeval_scorers.api_scorers.tool_dependency import (
-    ToolDependencyScorer,
-)
 
 __all__ = [
-    "ExecutionOrderScorer",
-    "JSONCorrectnessScorer",
-    "SummarizationScorer",
-    "HallucinationScorer",
     "FaithfulnessScorer",
-    "ContextualRelevancyScorer",
-    "ContextualPrecisionScorer",
-    "ContextualRecallScorer",
     "AnswerRelevancyScorer",
     "AnswerCorrectnessScorer",
     "InstructionAdherenceScorer",
-    "GroundednessScorer",
-    "DerailmentScorer",
-    "ToolOrderScorer",
+    "TracePromptScorer",
     "PromptScorer",
-    "ToolDependencyScorer",
 ]

judgeval/scorers/judgeval_scorers/api_scorers/answer_correctness.py

@@ -1,18 +1,10 @@
-"""
-`judgeval` answer relevancy scorer
-
-TODO add link to docs page for this scorer
-
-"""
-
-# Internal imports
-from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.scorers.api_scorer import ExampleAPIScorerConfig
 from judgeval.constants import APIScorerType
 from judgeval.data import ExampleParams
 from typing import List
 
 
-class AnswerCorrectnessScorer(APIScorerConfig):
+class AnswerCorrectnessScorer(ExampleAPIScorerConfig):
     score_type: APIScorerType = APIScorerType.ANSWER_CORRECTNESS
     required_params: List[ExampleParams] = [
         ExampleParams.INPUT,

judgeval/scorers/judgeval_scorers/api_scorers/answer_relevancy.py

@@ -1,10 +1,10 @@
-from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.scorers.api_scorer import ExampleAPIScorerConfig
 from judgeval.constants import APIScorerType
 from judgeval.data import ExampleParams
 from typing import List
 
 
-class AnswerRelevancyScorer(APIScorerConfig):
+class AnswerRelevancyScorer(ExampleAPIScorerConfig):
     score_type: APIScorerType = APIScorerType.ANSWER_RELEVANCY
     required_params: List[ExampleParams] = [
         ExampleParams.INPUT,

judgeval/scorers/judgeval_scorers/api_scorers/faithfulness.py

@@ -1,18 +1,10 @@
-"""
-`judgeval` faithfulness scorer
-
-TODO add link to docs page for this scorer
-
-"""
-
-# Internal imports
-from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.scorers.api_scorer import ExampleAPIScorerConfig
 from judgeval.constants import APIScorerType
 from judgeval.data import ExampleParams
 from typing import List
 
 
-class FaithfulnessScorer(APIScorerConfig):
+class FaithfulnessScorer(ExampleAPIScorerConfig):
     score_type: APIScorerType = APIScorerType.FAITHFULNESS
     required_params: List[ExampleParams] = [
         ExampleParams.INPUT,

judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py

@@ -1,17 +1,9 @@
-"""
-`judgeval` instruction adherence scorer
-
-TODO add link to docs page for this scorer
-
-"""
-
-# Internal imports
-from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.scorers.api_scorer import ExampleAPIScorerConfig
 from judgeval.constants import APIScorerType
 from judgeval.data import ExampleParams
 
 
-class InstructionAdherenceScorer(APIScorerConfig):
+class InstructionAdherenceScorer(ExampleAPIScorerConfig):
     def __init__(self, threshold: float):
         super().__init__(
             threshold=threshold,
@@ -21,7 +13,3 @@ class InstructionAdherenceScorer(APIScorerConfig):
                 ExampleParams.ACTUAL_OUTPUT,
             ],
         )
-
-    @property
-    def __name__(self):
-        return "Instruction Adherence"