judgeval 0.0.11__py3-none-any.whl → 0.22.2__py3-none-any.whl

judgeval/scorers/judgeval_scorers/__init__.py

@@ -1,148 +0,0 @@
-from typing import Type, Optional, Any
-from functools import wraps
-
-# Import implementations
-from judgeval.scorers.judgeval_scorers.api_scorers import (
-    ToolCorrectnessScorer as APIToolCorrectnessScorer,
-    JSONCorrectnessScorer as APIJSONCorrectnessScorer,
-    SummarizationScorer as APISummarizationScorer,
-    HallucinationScorer as APIHallucinationScorer,
-    FaithfulnessScorer as APIFaithfulnessScorer,
-    ContextualRelevancyScorer as APIContextualRelevancyScorer,
-    ContextualPrecisionScorer as APIContextualPrecisionScorer,
-    ContextualRecallScorer as APIContextualRecallScorer,
-    AnswerRelevancyScorer as APIAnswerRelevancyScorer,
-    AnswerCorrectnessScorer as APIAnswerCorrectnessScorer,
-)
-
-from judgeval.scorers.judgeval_scorers.local_implementations import (
-    AnswerRelevancyScorer as LocalAnswerRelevancyScorer,
-    ContextualPrecisionScorer as LocalContextualPrecisionScorer,
-    ContextualRecallScorer as LocalContextualRecallScorer,
-    ContextualRelevancyScorer as LocalContextualRelevancyScorer,
-    FaithfulnessScorer as LocalFaithfulnessScorer,
-    JsonCorrectnessScorer as LocalJsonCorrectnessScorer,
-    ToolCorrectnessScorer as LocalToolCorrectnessScorer,
-    HallucinationScorer as LocalHallucinationScorer,
-    SummarizationScorer as LocalSummarizationScorer,
-    AnswerCorrectnessScorer as LocalAnswerCorrectnessScorer
-)
-
-from judgeval.scorers.judgeval_scorers.classifiers import Text2SQLScorer
-
-
-class ScorerWrapper:
-    """
-    Wrapper class that can dynamically load either API or local implementation of a scorer.
-    """
-    def __init__(self, api_implementation: Type, local_implementation: Optional[Type] = None):
-        self.api_implementation = api_implementation
-        self.local_implementation = local_implementation
-        self._instance = None
-        self._init_args = None
-        self._init_kwargs = None
-        
-    def __call__(self, *args, **kwargs):
-        """Store initialization arguments for later use when implementation is loaded"""
-        self._init_args = args
-        self._init_kwargs = kwargs
-        return self
-    
-    def load_implementation(self, use_judgment: bool = True) -> Any:
-        """
-        Load the appropriate implementation based on the use_judgment flag.
-        
-        Args:
-            use_judgment (bool): If True, use API implementation. If False, use local implementation.
-        
-        Returns:
-            Instance of the appropriate implementation
-        
-        Raises:
-            ValueError: If local implementation is requested but not available
-        """
-        if self._instance is not None:
-            return self._instance
-            
-        if use_judgment:
-            implementation = self.api_implementation
-        else:
-            if self.local_implementation is None:
-                raise ValueError("No local implementation available for this scorer")
-            implementation = self.local_implementation
-            
-        args = self._init_args or ()
-        kwargs = self._init_kwargs or {}
-        self._instance = implementation(*args, **kwargs)
-        return self._instance
-    
-    def __getattr__(self, name):
-        """Defer all attribute access to the loaded implementation"""
-        if self._instance is None:
-            raise RuntimeError("Implementation not loaded. Call load_implementation() first")
-        return getattr(self._instance, name)
-
-# Create wrapped versions of all scorers
-
-AnswerCorrectnessScorer = ScorerWrapper(
-    api_implementation=APIAnswerCorrectnessScorer,
-    local_implementation=LocalAnswerCorrectnessScorer
-)
-
-AnswerRelevancyScorer = ScorerWrapper(
-    api_implementation=APIAnswerRelevancyScorer,
-    local_implementation=LocalAnswerRelevancyScorer
-)
-
-ToolCorrectnessScorer = ScorerWrapper(
-    api_implementation=APIToolCorrectnessScorer,
-    local_implementation=LocalToolCorrectnessScorer
-)
-
-JSONCorrectnessScorer = ScorerWrapper(
-    api_implementation=APIJSONCorrectnessScorer,
-    local_implementation=LocalJsonCorrectnessScorer
-)
-
-SummarizationScorer = ScorerWrapper(
-    api_implementation=APISummarizationScorer,
-    local_implementation=LocalSummarizationScorer
-)
-
-HallucinationScorer = ScorerWrapper(
-    api_implementation=APIHallucinationScorer,
-    local_implementation=LocalHallucinationScorer
-)
-
-FaithfulnessScorer = ScorerWrapper(
-    api_implementation=APIFaithfulnessScorer,
-    local_implementation=LocalFaithfulnessScorer
-)
-
-ContextualRelevancyScorer = ScorerWrapper(
-    api_implementation=APIContextualRelevancyScorer,
-    local_implementation=LocalContextualRelevancyScorer
-)
-
-ContextualPrecisionScorer = ScorerWrapper(
-    api_implementation=APIContextualPrecisionScorer,
-    local_implementation=LocalContextualPrecisionScorer
-)
-
-ContextualRecallScorer = ScorerWrapper(
-    api_implementation=APIContextualRecallScorer,
-    local_implementation=LocalContextualRecallScorer
-)
-
-__all__ = [
-    "ToolCorrectnessScorer",
-    "JSONCorrectnessScorer",
-    "SummarizationScorer",
-    "HallucinationScorer",
-    "FaithfulnessScorer",
-    "ContextualRelevancyScorer",
-    "ContextualPrecisionScorer",
-    "ContextualRecallScorer",
-    "AnswerRelevancyScorer",
-    "Text2SQLScorer",
-]

judgeval/scorers/judgeval_scorers/api_scorers/__init__.py

@@ -1,23 +1,25 @@
-from judgeval.scorers.judgeval_scorers.api_scorers.tool_correctness import ToolCorrectnessScorer
-from judgeval.scorers.judgeval_scorers.api_scorers.json_correctness import JSONCorrectnessScorer
-from judgeval.scorers.judgeval_scorers.api_scorers.summarization import SummarizationScorer
-from judgeval.scorers.judgeval_scorers.api_scorers.hallucination import HallucinationScorer
-from judgeval.scorers.judgeval_scorers.api_scorers.faithfulness import FaithfulnessScorer
-from judgeval.scorers.judgeval_scorers.api_scorers.contextual_relevancy import ContextualRelevancyScorer
-from judgeval.scorers.judgeval_scorers.api_scorers.contextual_precision import ContextualPrecisionScorer
-from judgeval.scorers.judgeval_scorers.api_scorers.contextual_recall import ContextualRecallScorer
-from judgeval.scorers.judgeval_scorers.api_scorers.answer_relevancy import AnswerRelevancyScorer
-from judgeval.scorers.judgeval_scorers.api_scorers.answer_correctness import AnswerCorrectnessScorer
+from judgeval.scorers.judgeval_scorers.api_scorers.faithfulness import (
+    FaithfulnessScorer,
+)
+from judgeval.scorers.judgeval_scorers.api_scorers.answer_relevancy import (
+    AnswerRelevancyScorer,
+)
+from judgeval.scorers.judgeval_scorers.api_scorers.answer_correctness import (
+    AnswerCorrectnessScorer,
+)
+from judgeval.scorers.judgeval_scorers.api_scorers.instruction_adherence import (
+    InstructionAdherenceScorer,
+)
+from judgeval.scorers.judgeval_scorers.api_scorers.prompt_scorer import (
+    TracePromptScorer,
+    PromptScorer,
+)
 
 __all__ = [
-    "ToolCorrectnessScorer",
-    "JSONCorrectnessScorer",
-    "SummarizationScorer",
-    "HallucinationScorer",
     "FaithfulnessScorer",
-    "ContextualRelevancyScorer",
-    "ContextualPrecisionScorer",
-    "ContextualRecallScorer",
     "AnswerRelevancyScorer",
     "AnswerCorrectnessScorer",
+    "InstructionAdherenceScorer",
+    "TracePromptScorer",
+    "PromptScorer",
 ]

judgeval/scorers/judgeval_scorers/api_scorers/answer_correctness.py

@@ -1,19 +1,13 @@
-"""
-`judgeval` answer relevancy scorer
-
-TODO add link to docs page for this scorer
-
-"""
-
-# Internal imports
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
-
-
-class AnswerCorrectnessScorer(APIJudgmentScorer):
-    def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.ANSWER_CORRECTNESS)
-
-    @property
-    def __name__(self):
-        return "Answer Correctness"
+from judgeval.scorers.api_scorer import ExampleAPIScorerConfig
+from judgeval.constants import APIScorerType
+from judgeval.data import ExampleParams
+from typing import List
+
+
+class AnswerCorrectnessScorer(ExampleAPIScorerConfig):
+    score_type: APIScorerType = APIScorerType.ANSWER_CORRECTNESS
+    required_params: List[ExampleParams] = [
+        ExampleParams.INPUT,
+        ExampleParams.ACTUAL_OUTPUT,
+        ExampleParams.EXPECTED_OUTPUT,
+    ]

judgeval/scorers/judgeval_scorers/api_scorers/answer_relevancy.py

@@ -1,19 +1,12 @@
-"""
-`judgeval` answer relevancy scorer
-
-TODO add link to docs page for this scorer
-
-"""
-
-# Internal imports
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
-
-
-class AnswerRelevancyScorer(APIJudgmentScorer):
-    def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.ANSWER_RELEVANCY)
-
-    @property
-    def __name__(self):
-        return "Answer Relevancy"
+from judgeval.scorers.api_scorer import ExampleAPIScorerConfig
+from judgeval.constants import APIScorerType
+from judgeval.data import ExampleParams
+from typing import List
+
+
+class AnswerRelevancyScorer(ExampleAPIScorerConfig):
+    score_type: APIScorerType = APIScorerType.ANSWER_RELEVANCY
+    required_params: List[ExampleParams] = [
+        ExampleParams.INPUT,
+        ExampleParams.ACTUAL_OUTPUT,
+    ]

judgeval/scorers/judgeval_scorers/api_scorers/faithfulness.py

@@ -1,19 +1,13 @@
-"""
-`judgeval` faithfulness scorer
-
-TODO add link to docs page for this scorer
-
-"""
-
-# Internal imports
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
-
-
-class FaithfulnessScorer(APIJudgmentScorer):
-    def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.FAITHFULNESS)
-
-    @property
-    def __name__(self):
-        return "Faithfulness"
+from judgeval.scorers.api_scorer import ExampleAPIScorerConfig
+from judgeval.constants import APIScorerType
+from judgeval.data import ExampleParams
+from typing import List
+
+
+class FaithfulnessScorer(ExampleAPIScorerConfig):
+    score_type: APIScorerType = APIScorerType.FAITHFULNESS
+    required_params: List[ExampleParams] = [
+        ExampleParams.INPUT,
+        ExampleParams.ACTUAL_OUTPUT,
+        ExampleParams.RETRIEVAL_CONTEXT,
+    ]

judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py

@@ -0,0 +1,15 @@
+from judgeval.scorers.api_scorer import ExampleAPIScorerConfig
+from judgeval.constants import APIScorerType
+from judgeval.data import ExampleParams
+
+
+class InstructionAdherenceScorer(ExampleAPIScorerConfig):
+    def __init__(self, threshold: float):
+        super().__init__(
+            threshold=threshold,
+            score_type=APIScorerType.INSTRUCTION_ADHERENCE,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+            ],
+        )

judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py

@@ -0,0 +1,327 @@
+from judgeval.scorers.api_scorer import (
+    APIScorerConfig,
+    ExampleAPIScorerConfig,
+    TraceAPIScorerConfig,
+)
+from judgeval.constants import APIScorerType
+from typing import Dict, Any, Optional
+from judgeval.api import JudgmentSyncClient
+from judgeval.exceptions import JudgmentAPIError
+import os
+from judgeval.logger import judgeval_logger
+from abc import ABC
+from judgeval.env import JUDGMENT_DEFAULT_GPT_MODEL
+from copy import copy
+from judgeval.utils.decorators.dont_throw import dont_throw
+
+
+def push_prompt_scorer(
+    name: str,
+    prompt: str,
+    threshold: float,
+    options: Optional[Dict[str, float]] = None,
+    model: str = JUDGMENT_DEFAULT_GPT_MODEL,
+    description: Optional[str] = None,
+    judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
+    organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
+    is_trace: bool = False,
+) -> str:
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        r = client.save_scorer(
+            payload={
+                "name": name,
+                "prompt": prompt,
+                "threshold": threshold,
+                "options": options,
+                "model": model,
+                "description": description,
+                "is_trace": is_trace,
+            }
+        )
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to save prompt scorer: {e.detail}",
+            response=e.response,
+        )
+    return r["scorer_response"]["name"]
+
+
+def fetch_prompt_scorer(
+    name: str,
+    judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
+    organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
+):
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        fetched_scorers = client.fetch_scorers({"names": [name]})
+        if len(fetched_scorers["scorers"]) == 0:
+            judgeval_logger.error(f"Prompt scorer '{name}' not found")
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Prompt scorer '{name}' not found",
+                response=None,  # type: ignore
+            )
+        else:
+            scorer_config = fetched_scorers["scorers"][0]
+            scorer_config.pop("created_at")
+            scorer_config.pop("updated_at")
+            return scorer_config
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to fetch prompt scorer '{name}': {e.detail}",
+            response=e.response,
+        )
+
+
+def scorer_exists(
+    name: str,
+    judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
+    organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
+):
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        return client.scorer_exists({"name": name})["exists"]
+    except JudgmentAPIError as e:
+        if e.status_code == 500:
+            raise JudgmentAPIError(
+                status_code=e.status_code,
+                detail=f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.detail}",
+                response=e.response,
+            )
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to check if scorer exists: {e.detail}",
+            response=e.response,
+        )
+
+
+class BasePromptScorer(ABC, APIScorerConfig):
+    score_type: APIScorerType
+    prompt: str
+    options: Optional[Dict[str, float]] = None
+    description: Optional[str] = None
+    judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or ""
+    organization_id: str = os.getenv("JUDGMENT_ORG_ID") or ""
+
+    @classmethod
+    @dont_throw
+    def get(
+        cls,
+        name: str,
+        judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
+        organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
+    ):
+        scorer_config = fetch_prompt_scorer(name, judgment_api_key, organization_id)
+        if scorer_config["is_trace"] != issubclass(cls, TracePromptScorer):
+            raise JudgmentAPIError(
+                status_code=400,
+                detail=f"Scorer with name {name} is not a {cls.__name__}",
+                response=None,  # type: ignore
+            )
+        if issubclass(cls, TracePromptScorer):
+            score_type = APIScorerType.TRACE_PROMPT_SCORER
+        else:
+            score_type = APIScorerType.PROMPT_SCORER
+        return cls(
+            score_type=score_type,
+            name=name,
+            prompt=scorer_config["prompt"],
+            threshold=scorer_config["threshold"],
+            options=scorer_config.get("options"),
+            model=scorer_config.get("model"),
+            description=scorer_config.get("description"),
+            judgment_api_key=judgment_api_key,
+            organization_id=organization_id,
+        )
+
+    @classmethod
+    def create(
+        cls,
+        name: str,
+        prompt: str,
+        threshold: float = 0.5,
+        options: Optional[Dict[str, float]] = None,
+        model: str = JUDGMENT_DEFAULT_GPT_MODEL,
+        description: Optional[str] = None,
+        judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
+        organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
+    ):
+        if not scorer_exists(name, judgment_api_key, organization_id):
+            if issubclass(cls, TracePromptScorer):
+                is_trace = True
+                score_type = APIScorerType.TRACE_PROMPT_SCORER
+            else:
+                is_trace = False
+                score_type = APIScorerType.PROMPT_SCORER
+            push_prompt_scorer(
+                name,
+                prompt,
+                threshold,
+                options,
+                model,
+                description,
+                judgment_api_key,
+                organization_id,
+                is_trace,
+            )
+            judgeval_logger.info(f"Successfully created PromptScorer: {name}")
+            return cls(
+                score_type=score_type,
+                name=name,
+                prompt=prompt,
+                threshold=threshold,
+                options=options,
+                model=model,
+                description=description,
+                judgment_api_key=judgment_api_key,
+                organization_id=organization_id,
+            )
+        else:
+            raise JudgmentAPIError(
+                status_code=400,
+                detail=f"Scorer with name {name} already exists. Either use the existing scorer with the get() method or use a new name.",
+                response=None,  # type: ignore
+            )
+
+    # Setter functions. Each setter function pushes the scorer to the DB.
+    def set_threshold(self, threshold: float):
+        """
+        Updates the threshold of the scorer.
+        """
+        self.threshold = threshold
+        self.push_prompt_scorer()
+
+    def set_prompt(self, prompt: str):
+        """
+        Updates the prompt with the new prompt.
+
+        Sample prompt:
+        "Did the chatbot answer the user's question in a kind way?"
+        """
+        self.prompt = prompt
+        self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated prompt for {self.name}")
+
+    def set_model(self, model: str):
+        """
+        Updates the model of the scorer.
+        """
+        self.model = model
+        self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated model for {self.name}")
+
+    def set_options(self, options: Optional[Dict[str, float]]):
+        """
+        Updates the options of the scorer.
+        """
+        self.options = options
+        self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated options for {self.name}")
+
+    def set_description(self, description: Optional[str]):
+        """
+        Updates the description of the scorer.
+        """
+        self.description = description
+        self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated description for {self.name}")
+
+    def append_to_prompt(self, prompt_addition: str):
+        """
+        Appends a string to the prompt.
+        """
+        self.prompt += prompt_addition
+        self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully appended to prompt for {self.name}")
+
+    # Getters
+    def get_threshold(self) -> float:
+        """
+        Returns the threshold of the scorer.
+        """
+        return self.threshold
+
+    def get_prompt(self) -> str:
+        """
+        Returns the prompt of the scorer.
+        """
+        return self.prompt
+
+    def get_model(self) -> str:
+        """
+        Returns the model of the scorer.
+        """
+        return self.model
+
+    def get_options(self) -> Dict[str, float] | None:
+        """
+        Returns the options of the scorer.
+        """
+        return copy(self.options) if self.options is not None else None
+
+    def get_description(self) -> str | None:
+        """
+        Returns the description of the scorer.
+        """
+        return self.description
+
+    def get_name(self) -> str:
+        """
+        Returns the name of the scorer.
+        """
+        return self.name
+
+    def get_config(self) -> dict:
+        """
+        Returns a dictionary with all the fields in the scorer.
+        """
+        return {
+            "name": self.name,
+            "model": self.model,
+            "prompt": self.prompt,
+            "threshold": self.threshold,
+            "options": self.options,
+            "description": self.description,
+        }
+
+    def push_prompt_scorer(self):
+        """
+        Pushes the scorer to the DB.
+        """
+        push_prompt_scorer(
+            self.name,
+            self.prompt,
+            self.threshold,
+            self.options,
+            self.model,
+            self.description,
+            self.judgment_api_key,
+            self.organization_id,
+            isinstance(self, TracePromptScorer),
+        )
+
+    def __str__(self):
+        return f"PromptScorer(name={self.name}, model={self.model}, prompt={self.prompt}, threshold={self.threshold}, options={self.options}, description={self.description})"
+
+    def model_dump(self, *args, **kwargs) -> Dict[str, Any]:
+        base = super().model_dump(*args, **kwargs)
+        base_fields = set(APIScorerConfig.model_fields.keys())
+        all_fields = set(self.__class__.model_fields.keys())
+
+        extra_fields = all_fields - base_fields - {"kwargs"}
+
+        base["kwargs"] = {
+            k: getattr(self, k) for k in extra_fields if getattr(self, k) is not None
+        }
+        return base
+
+
+class PromptScorer(BasePromptScorer, ExampleAPIScorerConfig):
+    pass
+
+
+class TracePromptScorer(BasePromptScorer, TraceAPIScorerConfig):
+    pass