judgeval 0.0.1__py3-none-any.whl

judgeval/run_evaluation.py

@@ -0,0 +1,355 @@
+import asyncio
+import requests
+from typing import List, Dict
+from datetime import datetime
+from rich import print as rprint
+
+from judgeval.data import (
+    Example, 
+    ScorerData, 
+    ScoringResult
+)
+from judgeval.scorers import (
+    CustomScorer, 
+    JudgmentScorer,
+    ClassifierScorer
+)
+from judgeval.scorers.score import a_execute_scoring
+
+from judgeval.constants import (
+    ROOT_API,
+    JUDGMENT_EVAL_API_URL,
+    JUDGMENT_EVAL_LOG_API_URL,
+)
+from judgeval.common.exceptions import JudgmentAPIError
+from judgeval.evaluation_run import EvaluationRun
+from judgeval.common.logger import (
+    enable_logging, 
+    debug, 
+    info, 
+    error, 
+    example_logging_context
+)
+
+
+def execute_api_eval(evaluation_run: EvaluationRun) -> List[Dict]:
+    """
+    Executes an evaluation of a list of `Example`s using one or more `JudgmentScorer`s via the Judgment API.
+
+    Args:
+        evaluation_run (EvaluationRun): The evaluation run object containing the examples, scorers, and metadata
+
+    Returns:
+        List[Dict]: The results of the evaluation. Each result is a dictionary containing the fields of a `ScoringResult`
+                    object. 
+    """
+    
+    try:
+        # submit API request to execute evals
+        payload = evaluation_run.model_dump(warnings=False)
+        response = requests.post(JUDGMENT_EVAL_API_URL, json=payload)
+        response_data = response.json()
+    except Exception as e:
+        error(f"Error: {e}")
+        details = response.json().get("detail", "No details provided")
+        raise JudgmentAPIError("An error occurred while executing the Judgment API request: " + details)
+    # Check if the response status code is not 2XX
+    # Add check for the duplicate eval run name
+    if not response.ok:
+        error_message = response_data.get('detail', 'An unknown error occurred.')
+        error(f"Error: {error_message=}")
+        raise JudgmentAPIError(error_message)
+    return response_data
+
+
+def merge_results(api_results: List[ScoringResult], local_results: List[ScoringResult]) -> List[ScoringResult]:
+    """
+    When executing scorers that come from both the Judgment API and custom scorers, we're left with
+    results for each type of scorer. This function merges the results from the API and local evaluations,
+    grouped by example. In particular, we merge the `scorers_data` field of each `ScoringResult` object.
+
+    Args:
+        api_results (List[ScoringResult]): The `ScoringResult`s from the API evaluation
+        local_results (List[ScoringResult]): The `ScoringResult`s from the local evaluation
+
+    Returns:
+        List[ScoringResult]: The merged `ScoringResult`s (updated `scorers_data` field)
+    """
+    # No merge required
+    if not local_results and api_results:
+        return api_results
+    if not api_results and local_results:
+        return local_results
+
+    if len(api_results) != len(local_results):
+        # Results should be of same length because each ScoringResult is a 1-1 mapping to an Example
+        raise ValueError(f"The number of API and local results do not match: {len(api_results)} vs {len(local_results)}")
+    
+    # Each ScoringResult in api and local have all the same fields besides `scorers_data`
+    for api_result, local_result in zip(api_results, local_results):
+        if api_result.input != local_result.input:
+            raise ValueError("The API and local results are not aligned.")
+        if api_result.actual_output != local_result.actual_output:
+            raise ValueError("The API and local results are not aligned.")
+        if api_result.expected_output != local_result.expected_output:
+            raise ValueError("The API and local results are not aligned.")
+        if api_result.context != local_result.context:
+            raise ValueError("The API and local results are not aligned.")
+        if api_result.retrieval_context != local_result.retrieval_context:
+            raise ValueError("The API and local results are not aligned.")
+        
+        # Merge ScorerData from the API and local scorers together
+        api_scorer_data = api_result.scorers_data
+        local_scorer_data = local_result.scorers_data
+        if api_scorer_data is None and local_scorer_data is not None:
+            api_result.scorers_data = local_scorer_data
+
+        if api_scorer_data is not None and local_scorer_data is not None:
+            api_result.scorers_data = api_scorer_data + local_scorer_data
+    
+    return api_results
+
+
+def check_missing_scorer_data(results: List[ScoringResult]) -> List[ScoringResult]:
+    """
+    Checks if any `ScoringResult` objects are missing `scorers_data`.
+
+    If any are missing, logs an error and returns the results.
+    """
+    for i, result in enumerate(results):
+        if not result.scorers_data:
+            error(
+                f"Scorer data is missing for example {i}. "
+                "This is usually caused when the example does not contain "
+                "the fields required by the scorer. "
+                "Check that your example contains the fields required by the scorers. "
+                "TODO add docs link here for reference."
+            )
+    return results
+
+def check_eval_run_name_exists(eval_name: str, project_name: str, judgment_api_key: str) -> None:
+    """
+    Checks if an evaluation run name already exists for a given project.
+
+    Args:
+        eval_name (str): Name of the evaluation run
+        project_name (str): Name of the project
+        judgment_api_key (str): API key for authentication
+
+    Raises:
+        ValueError: If the evaluation run name already exists
+        JudgmentAPIError: If there's an API error during the check
+    """
+    try:
+        response = requests.post(
+            f"{ROOT_API}/eval-run-name-exists/",
+            json={
+                "eval_name": eval_name,
+                "project_name": project_name,
+                "judgment_api_key": judgment_api_key,
+            }
+        )
+        
+        if response.status_code == 409:
+            error(f"Evaluation run name '{eval_name}' already exists for this project")
+            raise ValueError(f"Evaluation run name '{eval_name}' already exists for this project")
+        
+        if not response.ok:
+            response_data = response.json()
+            error_message = response_data.get('detail', 'An unknown error occurred.')
+            error(f"Error checking eval run name: {error_message}")
+            raise JudgmentAPIError(error_message)
+            
+    except requests.exceptions.RequestException as e:
+        error(f"Failed to check if eval run name exists: {str(e)}")
+        raise JudgmentAPIError(f"Failed to check if eval run name exists: {str(e)}")
+
+def log_evaluation_results(merged_results: List[ScoringResult], evaluation_run: EvaluationRun) -> None:
+    """
+    Logs evaluation results to the Judgment API database.
+
+    Args:
+        merged_results (List[ScoringResult]): The results to log
+        evaluation_run (EvaluationRun): The evaluation run containing project info and API key
+
+    Raises:
+        JudgmentAPIError: If there's an API error during logging
+        ValueError: If there's a validation error with the results
+    """
+    try:
+        res = requests.post(
+            JUDGMENT_EVAL_LOG_API_URL,
+            json={
+                "results": [result.to_dict() for result in merged_results],
+                "judgment_api_key": evaluation_run.judgment_api_key,
+                "project_name": evaluation_run.project_name,
+                "eval_name": evaluation_run.eval_name,
+            }
+        )
+        
+        if not res.ok:
+            response_data = res.json()
+            error_message = response_data.get('detail', 'An unknown error occurred.')
+            error(f"Error {res.status_code}: {error_message}")
+            raise JudgmentAPIError(error_message)
+        
+        if "ui_results_url" in res.json():
+            rprint(f"\n🔍 You can view your evaluation results here: [rgb(106,0,255)]{res.json()['ui_results_url']}[/]\n")
+            
+    except requests.exceptions.RequestException as e:
+        error(f"Request failed while saving evaluation results to DB: {str(e)}")
+        raise JudgmentAPIError(f"Request failed while saving evaluation results to DB: {str(e)}")
+    except Exception as e:
+        error(f"Failed to save evaluation results to DB: {str(e)}")
+        raise ValueError(f"Failed to save evaluation results to DB: {str(e)}")
+
+def run_eval(evaluation_run: EvaluationRun, override: bool = False) -> List[ScoringResult]:
+    """
+    Executes an evaluation of `Example`s using one or more `Scorer`s
+
+    Args:
+        evaluation_run (EvaluationRun): Stores example and evaluation together for running
+    
+        Args: 
+            project_name (str): The name of the project the evaluation results belong to
+            eval_name (str): The name of the evaluation run
+            examples (List[Example]): The examples to evaluate
+            scorers (List[Union[JudgmentScorer, CustomScorer]]): A list of scorers to use for evaluation
+            model (str): The model used as a judge when using LLM as a Judge
+            aggregator (Optional[str]): The aggregator to use for evaluation if using Mixture of Judges
+            metadata (Optional[Dict[str, Any]]): Additional metadata to include for this evaluation run, e.g. comments, dataset name, purpose, etc.
+            judgment_api_key (Optional[str]): The API key for running evaluations on the Judgment API
+            log_results (bool): Whether to log the results to the Judgment API
+
+
+    Returns:
+        List[ScoringResult]: The results of the evaluation. Each result is a dictionary containing the fields of a `ScoringResult` object.
+    """
+    
+    # Call endpoint to check to see if eval run name exists (if we DON'T want to override and DO want to log results)
+    if not override and evaluation_run.log_results:
+        check_eval_run_name_exists(
+            evaluation_run.eval_name,
+            evaluation_run.project_name,
+            evaluation_run.judgment_api_key
+        )
+    
+    # Set example IDs if not already set
+    debug("Initializing examples with IDs and timestamps")
+    for idx, example in enumerate(evaluation_run.examples):
+        if example.example_id is None:
+            example.example_id = idx
+            debug(f"Set example ID {idx} for input: {example.input[:50]}...")
+        example.timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        with example_logging_context(example.timestamp, example.example_id):
+            debug(f"Initialized example {example.example_id}")
+            debug(f"Input: {example.input}")
+            debug(f"Actual output: {example.actual_output}")
+            if example.expected_output:
+                debug(f"Expected output: {example.expected_output}")
+            if example.context:
+                debug(f"Context: {example.context}")
+            if example.retrieval_context:
+                debug(f"Retrieval context: {example.retrieval_context}")
+    
+    debug(f"Starting evaluation run with {len(evaluation_run.examples)} examples")
+    
+    # Group JudgmentScorers and CustomScorers, then evaluate them in parallel
+    debug("Grouping scorers by type")
+    judgment_scorers: List[JudgmentScorer] = []
+    custom_scorers: List[CustomScorer] = []
+    for scorer in evaluation_run.scorers:
+        if isinstance(scorer, (JudgmentScorer, ClassifierScorer)):
+            judgment_scorers.append(scorer)
+            debug(f"Added judgment scorer: {type(scorer).__name__}")
+        else:
+            custom_scorers.append(scorer)
+            debug(f"Added custom scorer: {type(scorer).__name__}")
+    
+    debug(f"Found {len(judgment_scorers)} judgment scorers and {len(custom_scorers)} custom scorers")
+    
+    api_results: List[ScoringResult] = []
+    local_results: List[ScoringResult] = []
+    
+    # Execute evaluation using Judgment API
+    if judgment_scorers:
+        info("Starting API evaluation")
+        debug(f"Creating API evaluation run with {len(judgment_scorers)} scorers")
+        try:  # execute an EvaluationRun with just JudgmentScorers
+            api_evaluation_run: EvaluationRun = EvaluationRun(
+                eval_name=evaluation_run.eval_name,
+                project_name=evaluation_run.project_name,
+                examples=evaluation_run.examples,
+                scorers=judgment_scorers,
+                model=evaluation_run.model,
+                aggregator=evaluation_run.aggregator,
+                metadata=evaluation_run.metadata,
+                judgment_api_key=evaluation_run.judgment_api_key,
+                log_results=evaluation_run.log_results
+            )
+            debug("Sending request to Judgment API")    
+            response_data: List[Dict] = execute_api_eval(api_evaluation_run)  # ScoringResults
+            info(f"Received {len(response_data['results'])} results from API")
+        except JudgmentAPIError as e:
+            error(f"An error occurred while executing the Judgment API request: {str(e)}")
+            raise JudgmentAPIError(f"An error occurred while executing the Judgment API request: {str(e)}")
+        except ValueError as e:
+            raise ValueError(f"Please check your EvaluationRun object, one or more fields are invalid: {str(e)}")
+        
+        # Convert the response data to `ScoringResult` objects
+        debug("Processing API results")
+        for idx, result in enumerate(response_data["results"]):  
+            with example_logging_context(evaluation_run.examples[idx].timestamp, evaluation_run.examples[idx].example_id):
+                for scorer in judgment_scorers:
+                    debug(f"Processing API result for example {idx} and scorer {scorer.score_type}")
+                # filter for key-value pairs that are used to initialize ScoringResult
+                # there may be some stuff in here that doesn't belong in ScoringResult
+                # TODO: come back and refactor this to have ScoringResult take in **kwargs
+                filtered_result = {k: v for k, v in result.items() if k in ScoringResult.__annotations__}
+                
+                # Convert scorers_data dicts to ScorerData objects
+                if "scorers_data" in filtered_result and filtered_result["scorers_data"]:
+                    filtered_result["scorers_data"] = [
+                        ScorerData(**scorer_dict) 
+                        for scorer_dict in filtered_result["scorers_data"]
+                    ]
+                
+                api_results.append(ScoringResult(**filtered_result))
+
+    # Run local evals
+    if custom_scorers:  # List[CustomScorer]
+        info("Starting local evaluation")
+        for example in evaluation_run.examples:
+            with example_logging_context(example.timestamp, example.example_id):
+                debug(f"Processing example {example.example_id}: {example.input}")
+        
+        results: List[ScoringResult] = asyncio.run(
+            a_execute_scoring(
+                evaluation_run.examples,
+                custom_scorers,
+                model=evaluation_run.model,
+                ignore_errors=True,
+                skip_on_missing_params=True,
+                show_indicator=True,
+                _use_bar_indicator=True,
+                throttle_value=0,
+                max_concurrent=100,
+            )
+        )
+        local_results = results
+        info(f"Local evaluation complete with {len(local_results)} results")
+
+    # Aggregate the ScorerData from the API and local evaluations
+    debug("Merging API and local results")
+    merged_results: List[ScoringResult] = merge_results(api_results, local_results)
+    merged_results = check_missing_scorer_data(merged_results)
+
+    info(f"Successfully merged {len(merged_results)} results")
+
+    if evaluation_run.log_results:
+        log_evaluation_results(merged_results, evaluation_run)
+
+    for i, result in enumerate(merged_results):
+        if not result.scorers_data:  # none of the scorers could be executed on this example
+            info(f"None of the scorers could be executed on example {i}. This is usually because the Example is missing the fields needed by the scorers. Try checking that the Example has the necessary fields for your scorers.")
+    return merged_results

judgeval/scorers/__init__.py

@@ -0,0 +1,30 @@
+from judgeval.scorers.base_scorer import JudgmentScorer
+from judgeval.scorers.custom_scorer import CustomScorer
+from judgeval.scorers.prompt_scorer import PromptScorer, ClassifierScorer
+from judgeval.scorers.judgeval_scorers import (
+    ToolCorrectnessScorer,
+    JSONCorrectnessScorer,
+    SummarizationScorer,
+    HallucinationScorer,
+    FaithfulnessScorer,
+    ContextualRelevancyScorer,
+    ContextualPrecisionScorer,
+    ContextualRecallScorer,
+    AnswerRelevancyScorer,
+)
+
+__all__ = [
+    "JudgmentScorer",
+    "CustomScorer",
+    "PromptScorer",
+    "ClassifierScorer",
+    "ToolCorrectnessScorer",
+    "JSONCorrectnessScorer",
+    "SummarizationScorer",
+    "HallucinationScorer",
+    "FaithfulnessScorer",
+    "ContextualRelevancyScorer",
+    "ContextualPrecisionScorer",
+    "ContextualRecallScorer",
+    "AnswerRelevancyScorer",
+]

judgeval/scorers/base_scorer.py

@@ -0,0 +1,51 @@
+"""
+Judgment Scorer class.
+
+Scores `Example`s using ready-made Judgment evaluators.
+"""
+
+from pydantic import BaseModel, field_validator
+from judgeval.common.logger import debug, info, warning, error
+
+from judgeval.constants import APIScorer
+
+
+class JudgmentScorer(BaseModel):
+    """
+    Class for ready-made, "out-of-the-box" scorer that uses Judgment evaluators to score `Example`s.
+
+    Args:
+        score_type (APIScorer): The Judgment metric to use for scoring `Example`s
+        threshold (float): A value between 0 and 1 that determines the scoring threshold
+    """
+    threshold: float
+    score_type: APIScorer
+
+    @field_validator('threshold')
+    def validate_threshold(cls, v):
+        """
+        Validates that the threshold is between 0 and 1 inclusive.
+        """
+        if not 0 <= v <= 1:
+            error(f"Threshold must be between 0 and 1, got: {v}")
+            raise ValueError(f"Threshold must be between 0 and 1, got: {v}")
+        return v
+
+    @field_validator('score_type')
+    def convert_to_enum_value(cls, v):
+        """
+        Validates that the `score_type` is a valid `JudgmentMetric` enum value.
+        Converts string values to `JudgmentMetric` enum values.
+        """
+        debug(f"Attempting to convert score_type value: {v}")
+        if isinstance(v, APIScorer):
+            info(f"Using existing JudgmentMetric: {v.value}")
+            return v.value
+        elif isinstance(v, str):
+            debug(f"Converting string value to JudgmentMetric enum: {v}")
+            return APIScorer[v.upper()].value
+        error(f"Invalid score_type value: {v}")
+        raise ValueError(f"Invalid value for score_type: {v}")
+    
+    def __str__(self):
+        return f"JudgmentScorer(score_type={self.score_type}, threshold={self.threshold})"

judgeval/scorers/custom_scorer.py

@@ -0,0 +1,134 @@
+"""
+Custom Scorer class
+
+Enables client to create custom scorers that do not fall under any of the ready-made Judgment scorers.
+To create a custom scorer, extend this class and implement the `score_example`, `a_score_example`, and `success_check` methods.
+"""
+
+from typing import Optional, Dict, Union, List
+from abc import abstractmethod
+
+from judgeval.common.logger import debug, info, warning, error
+from judgeval.judges import judgevalJudge
+from judgeval.judges.utils import create_judge
+
+
+class CustomScorer:
+    """
+    If you want to create a scorer that does not fall under any of the ready-made Judgment scorers,
+    you can create a custom scorer by extending this class. This is best used for special use cases
+    where none of Judgment's scorers are suitable.
+    """
+    score_type: str  # name of your new scorer
+    threshold: float  # The threshold to pass a test while using this scorer as a scorer
+    score: Optional[float] = None  # The float score of the scorer run on the test case
+    score_breakdown: Dict = None
+    reason: Optional[str] = None  # The reason for the score when evaluating the test case
+    success: Optional[bool] = None  # Whether the test case passed or failed
+    evaluation_model: Optional[str] = None  # The model used to evaluate the test case
+    strict_mode: bool = False  # Whether to run the scorer in strict mode
+    async_mode: bool = True  # Whether to run the scorer in async mode
+    verbose_mode: bool = True  # Whether to run the scorer in verbose mode
+    include_reason: bool = False  # Whether to include the reason in the output
+    error: Optional[str] = None  # The error message if the scorer failed
+    evaluation_cost: Optional[float] = None  # The cost of running the scorer
+    verbose_logs: Optional[str] = None  # The verbose logs of the scorer
+    additional_metadata: Optional[Dict] = None  # Additional metadata for the scorer
+
+    def __init__(
+        self, 
+        score_type: str, 
+        threshold: float, 
+        score: Optional[float] = None, 
+        score_breakdown: Optional[Dict] = None, 
+        reason: Optional[str] = None, 
+        success: Optional[bool] = None, 
+        evaluation_model: Optional[str] = None, 
+        strict_mode: bool = False, 
+        async_mode: bool = True, 
+        verbose_mode: bool = True, 
+        include_reason: bool = False, 
+        error: Optional[str] = None, 
+        evaluation_cost: Optional[float] = None, 
+        verbose_logs: Optional[str] = None, 
+        additional_metadata: Optional[Dict] = None
+        ):
+            debug(f"Initializing CustomScorer with score_type={score_type}, threshold={threshold}")
+            if strict_mode:
+                warning("Strict mode enabled - scoring will be more rigorous")
+            info(f"CustomScorer initialized with evaluation_model: {evaluation_model}")
+            self.score_type = score_type
+            self.threshold = threshold
+            self.score = score
+            self.score_breakdown = score_breakdown
+            self.reason = reason
+            self.success = success
+            self.evaluation_model = evaluation_model
+            self.strict_mode = strict_mode
+            self.async_mode = async_mode
+            self.verbose_mode = verbose_mode
+            self.include_reason = include_reason
+            self.error = error
+            self.evaluation_cost = evaluation_cost
+            self.verbose_logs = verbose_logs
+            self.additional_metadata = additional_metadata
+
+    def _add_model(self, model: Optional[Union[str, List[str], judgevalJudge]] = None):
+        """
+        Adds the evaluation model to the CustomScorer instance 
+
+        This method is used at eval time
+        """
+        self.model, self.using_native_model = create_judge(model)
+        self.evaluation_model = self.model.get_model_name()
+
+    @abstractmethod
+    def score_example(self, example, *args, **kwargs) -> float:
+        """
+        Measures the score on a single example
+        """
+        warning("Attempting to call unimplemented score_example method")
+        error("score_example method not implemented")
+        raise NotImplementedError("You must implement the `score` method in your custom scorer")
+
+    @abstractmethod
+    async def a_score_example(self, example, *args, **kwargs) -> float:
+        """
+        Asynchronously measures the score on a single example
+        """
+        warning("Attempting to call unimplemented a_score_example method")
+        error("a_score_example method not implemented")
+        raise NotImplementedError("You must implement the `a_score` method in your custom scorer") 
+    
+    @abstractmethod
+    def _success_check(self) -> bool:
+        """
+        For unit testing, determines whether the test case passes or fails
+        """
+        warning("Attempting to call unimplemented success_check method")
+        error("success_check method not implemented")
+        raise NotImplementedError("You must implement the `passes` method in your custom scorer")
+
+    def __str__(self):
+        debug("Converting CustomScorer instance to string representation")
+        if self.error:
+            warning(f"CustomScorer contains error: {self.error}")
+        info(f"CustomScorer status - success: {self.success}, score: {self.score}")
+        attributes = {
+            "score_type": self.score_type,
+            "threshold": self.threshold,
+            "score": self.score,
+            "score_breakdown": self.score_breakdown,
+            "reason": self.reason,
+            "success": self.success,
+            "evaluation_model": self.evaluation_model,
+            "strict_mode": self.strict_mode,
+            "async_mode": self.async_mode,
+            "verbose_mode": self.verbose_mode,
+            "include_reason": self.include_reason,
+            "error": self.error,
+            "evaluation_cost": self.evaluation_cost,
+            "verbose_logs": self.verbose_logs,
+            "additional_metadata": self.additional_metadata,
+        }
+        return f"CustomScorer({attributes})"

judgeval/scorers/judgeval_scorers/__init__.py

@@ -0,0 +1,21 @@
+from judgeval.scorers.judgeval_scorers.tool_correctness import ToolCorrectnessScorer
+from judgeval.scorers.judgeval_scorers.json_correctness import JSONCorrectnessScorer
+from judgeval.scorers.judgeval_scorers.summarization import SummarizationScorer
+from judgeval.scorers.judgeval_scorers.hallucination import HallucinationScorer
+from judgeval.scorers.judgeval_scorers.faithfulness import FaithfulnessScorer
+from judgeval.scorers.judgeval_scorers.contextual_relevancy import ContextualRelevancyScorer
+from judgeval.scorers.judgeval_scorers.contextual_precision import ContextualPrecisionScorer
+from judgeval.scorers.judgeval_scorers.contextual_recall import ContextualRecallScorer
+from judgeval.scorers.judgeval_scorers.answer_relevancy import AnswerRelevancyScorer
+
+__all__ = [
+    "ToolCorrectnessScorer",
+    "JSONCorrectnessScorer",
+    "SummarizationScorer",
+    "HallucinationScorer",
+    "FaithfulnessScorer",
+    "ContextualRelevancyScorer",
+    "ContextualPrecisionScorer",
+    "ContextualRecallScorer",
+    "AnswerRelevancyScorer",
+]

judgeval/scorers/judgeval_scorers/answer_relevancy.py

@@ -0,0 +1,19 @@
+"""
+`judgeval` answer relevancy scorer
+
+TODO add link to docs page for this scorer
+
+"""
+
+# Internal imports
+from judgeval.scorers.base_scorer import JudgmentScorer
+from judgeval.constants import APIScorer
+
+
+class AnswerRelevancyScorer(JudgmentScorer):
+    def __init__(self, threshold: float):
+        super().__init__(threshold=threshold, score_type=APIScorer.ANSWER_RELEVANCY)
+
+    @property
+    def __name__(self):
+        return "Answer Relevancy"

judgeval/scorers/judgeval_scorers/contextual_precision.py

@@ -0,0 +1,19 @@
+"""
+`judgeval` contextual precision scorer
+
+TODO add link to docs page for this scorer
+
+"""
+
+# Internal imports
+from judgeval.scorers.base_scorer import JudgmentScorer
+from judgeval.constants import APIScorer
+
+
+class ContextualPrecisionScorer(JudgmentScorer):
+    def __init__(self, threshold: float):
+        super().__init__(threshold=threshold, score_type=APIScorer.CONTEXTUAL_PRECISION)
+
+    @property
+    def __name__(self):
+        return "Contextual Precision"

judgeval/scorers/judgeval_scorers/contextual_recall.py

@@ -0,0 +1,19 @@
+"""
+`judgeval` contextual recall scorer
+
+TODO add link to docs page for this scorer
+
+"""
+
+# Internal imports
+from judgeval.scorers.base_scorer import JudgmentScorer
+from judgeval.constants import APIScorer
+
+
+class ContextualRecallScorer(JudgmentScorer):
+    def __init__(self, threshold: float):
+        super().__init__(threshold=threshold, score_type=APIScorer.CONTEXTUAL_RECALL)
+
+    @property
+    def __name__(self):
+        return "Contextual Recall"

judgeval/scorers/judgeval_scorers/contextual_relevancy.py

@@ -0,0 +1,22 @@
+"""
+`judgeval` contextual relevancy scorer
+
+TODO add link to docs page for this scorer
+
+"""
+
+# Internal imports
+from judgeval.scorers.base_scorer import JudgmentScorer
+from judgeval.constants import APIScorer
+
+
+class ContextualRelevancyScorer(JudgmentScorer):
+    """
+    Scorer that checks if the output of a model is relevant to the retrieval context
+    """
+    def __init__(self, threshold: float):
+        super().__init__(threshold=threshold, score_type=APIScorer.CONTEXTUAL_RELEVANCY)
+
+    @property
+    def __name__(self):
+        return "Contextual Relevancy"