judgeval 0.0.32__py3-none-any.whl → 0.0.33__py3-none-any.whl

judgeval/scorers/judgeval_scorers/local_implementations/__init__.py

@@ -1,27 +0,0 @@
-from judgeval.scorers.judgeval_scorers.local_implementations.answer_relevancy.answer_relevancy_scorer import AnswerRelevancyScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.contextual_precision.contextual_precision_scorer import ContextualPrecisionScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.contextual_recall.contextual_recall_scorer import ContextualRecallScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.contextual_relevancy.contextual_relevancy_scorer import ContextualRelevancyScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.faithfulness.faithfulness_scorer import FaithfulnessScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.json_correctness.json_correctness_scorer import JsonCorrectnessScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.execution_order.execution_order import ExecutionOrderScorer 
-from judgeval.scorers.judgeval_scorers.local_implementations.hallucination.hallucination_scorer import HallucinationScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.summarization.summarization_scorer import SummarizationScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.answer_correctness.answer_correctness_scorer import AnswerCorrectnessScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.comparison.comparison_scorer import ComparisonScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.instruction_adherence.instruction_adherence import InstructionAdherenceScorer
-
-__all__ = [
-    "AnswerCorrectnessScorer",
-    "AnswerRelevancyScorer",
-    "ComparisonScorer",
-    "ContextualPrecisionScorer",
-    "ContextualRecallScorer",
-    "ContextualRelevancyScorer",
-    "FaithfulnessScorer",
-    "JsonCorrectnessScorer",
-    "ExecutionOrderScorer",
-    "HallucinationScorer",
-    "SummarizationScorer",
-    "InstructionAdherenceScorer",
-]

judgeval/scorers/judgeval_scorers/local_implementations/answer_correctness/__init__.py

@@ -1,4 +0,0 @@
-from judgeval.scorers.judgeval_scorers.local_implementations.answer_correctness.answer_correctness_scorer import AnswerCorrectnessScorer
-
-
-__all__ = ["AnswerCorrectnessScorer"]

judgeval/scorers/judgeval_scorers/local_implementations/answer_correctness/answer_correctness_scorer.py

@@ -1,276 +0,0 @@
-from typing import Optional, List, Union, Tuple
-
-from judgeval.constants import APIScorer
-from judgeval.judges import JudgevalJudge
-from judgeval.judges.utils import create_judge
-from judgeval.data import Example, ExampleParams
-from judgeval.scorers import JudgevalScorer
-from judgeval.scorers.utils import (
-    get_or_create_event_loop,
-    parse_response_json,
-    scorer_progress_meter,
-    create_verbose_logs,
-    check_example_params,
-)
-from .prompts import (
-    ACVerdict,
-    AnswerCorrectnessTemplate,
-    Statements,
-    Verdicts,
-    Reason,
-)
-
-
-required_params = [
-    ExampleParams.INPUT,
-    ExampleParams.ACTUAL_OUTPUT,
-    ExampleParams.EXPECTED_OUTPUT,
-]
-
-
-class AnswerCorrectnessScorer(JudgevalScorer):
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        model: Optional[Union[str, JudgevalJudge]] = None,
-        include_reason: bool = True,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False
-    ):
-        super().__init__(
-            score_type=APIScorer.ANSWER_CORRECTNESS,
-            threshold=1 if strict_mode else threshold,
-            evaluation_model=None,
-            include_reason=include_reason,
-            async_mode=async_mode,
-            strict_mode=strict_mode,
-            verbose_mode=verbose_mode
-        )
-        self.model, self.using_native_model = create_judge(model)
-        self.evaluation_model = self.model.get_model_name()
-
-    async def _a_get_statements(self, expected_output: str) -> List[str]:
-        prompt = AnswerCorrectnessTemplate.deduce_statements(
-            expected_output=expected_output,
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return data["statements"]
-        else:
-            try:
-                res: Statements = await self.model.a_generate(
-                    prompt, schema=Statements
-                )
-                return res.statements
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return data["statements"]
-
-    def _get_statements(self, expected_output: str) -> List[str]:
-        prompt = AnswerCorrectnessTemplate.deduce_statements(
-            expected_output=expected_output,
-        )
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return data["statements"]
-        else:
-            try:
-                res: Statements = self.model.generate(
-                    prompt, schema=Statements
-                )
-                return res.statements
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return data["statements"]
-
-    async def _a_get_verdicts(self, actual_output: str) -> List[ACVerdict]:
-        if len(self.statements) == 0:
-            return []
-
-        prompt = AnswerCorrectnessTemplate.generate_verdicts(
-            actual_output=actual_output,
-            statements=self.statements,
-        )
-
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return [ACVerdict(**item) for item in data["verdicts"]]
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(prompt, schema=Verdicts)
-                return [item for item in res.verdicts]
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return [ACVerdict(**item) for item in data["verdicts"]]
-
-    def _get_verdicts(self, actual_output: str) -> List[ACVerdict]:
-        if len(self.statements) == 0:
-            return []
-
-        prompt = AnswerCorrectnessTemplate.generate_verdicts(
-            actual_output=actual_output,
-            statements=self.statements,
-        )
-
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return [ACVerdict(**item) for item in data["verdicts"]]
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                return [item for item in res.verdicts]
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return [ACVerdict(**item) for item in data["verdicts"]]
-
-    async def _a_get_reason(self) -> str:
-        if self.include_reason is False:
-            return None
-
-        incorrect_statements: List[Tuple[str, str]] = []
-        for idx, verdict in enumerate(self.verdicts):
-            if verdict.verdict.strip().lower() == "no":
-                incorrect_statements.append((self.statements[idx], verdict.reason))
-
-        prompt = AnswerCorrectnessTemplate.generate_reason(
-            incorrect_statements=incorrect_statements,
-            score=format(self.score, ".2f"),
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = await self.model.a_generate(
-                    prompt=prompt, schema=Reason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    def _get_reason(self) -> str:
-        if self.include_reason is False:
-            return None
-
-        incorrect_statements: List[Tuple[str, str]] = []
-        for idx, verdict in enumerate(self.verdicts):
-            if verdict.verdict.strip().lower() == "no":
-                incorrect_statements.append((self.statements[idx], verdict.reason))
-
-        prompt = AnswerCorrectnessTemplate.generate_reason(
-            incorrect_statements=incorrect_statements,
-            score=format(self.score, ".2f"),
-        )
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = self.model.generate(
-                    prompt=prompt, schema=Reason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    def _compute_score(self) -> float:
-        number_of_verdicts = len(self.verdicts)
-        if number_of_verdicts == 0:
-            return 1
-
-        correct_count = 0
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() == "yes":
-                correct_count += 1
-
-        score = correct_count / number_of_verdicts
-        return 0 if self.strict_mode and score < self.threshold else score
-
-    def score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(self, display_meter=_show_indicator):
-            try:
-                if self.async_mode:
-                    loop = get_or_create_event_loop()
-                    loop.run_until_complete(
-                        self.a_score_example(example, _show_indicator=False)
-                    )
-                else:
-                    self.statements = self._get_statements(example.expected_output)
-                    self.verdicts = self._get_verdicts(example.actual_output)
-                    self.score = self._compute_score()
-                    self.reason = self._get_reason()
-                    self.success = self.score >= self.threshold
-                    self.verbose_logs = create_verbose_logs(
-                        self,
-                        steps=[
-                            f"Statements:\n{self.statements}",
-                            f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
-                            f"Score: {self.score}\nReason: {self.reason}",
-                        ],
-                    )
-                return self.score
-            except Exception as e:
-                print(f"Error in score_example for AnswerCorrectnessScorer: {e}")
-                raise
-    
-    async def a_score_example(
-        self, 
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(self, async_mode=True, display_meter=_show_indicator):
-            try:
-                self.statements: List[str] = await self._a_get_statements(example.expected_output)
-                self.verdicts: List[ACVerdict] = await self._a_get_verdicts(example.actual_output)
-                self.score = self._compute_score()
-                self.reason = await self._a_get_reason()
-                self.success = self.score >= self.threshold
-                self.verbose_logs = create_verbose_logs(
-                    self,
-                    steps=[
-                        f"Statements:\n{self.statements}",
-                        f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
-                        f"Score: {self.score}\nReason: {self.reason}",
-                    ],
-                )
-                return self.score
-            except Exception as e:
-                print(f"Error in a_score_example for AnswerCorrectnessScorer: {e}")
-                raise
-
-    def _success_check(self) -> bool:
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score >= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Answer Correctness"

judgeval/scorers/judgeval_scorers/local_implementations/answer_correctness/prompts.py

@@ -1,169 +0,0 @@
-"""
-Util prompts for AnswerCorrectnessScorer
-"""
-
-from typing import List, Tuple
-from pydantic import BaseModel
-
-
-# BaseModels to enforce formatting in LLM JSON response
-class Statements(BaseModel):
-    statements: List[str]
-
-
-class ACVerdict(BaseModel):
-    verdict: str
-    reason: str
-
-
-class Verdicts(BaseModel):
-    verdicts: List[ACVerdict]
-
-
-class Reason(BaseModel):
-    reason: str
-
-
-class AnswerCorrectnessTemplate:
-    @staticmethod
-    def deduce_statements(expected_output):
-        return f"""You will be presented with a piece of text. Your task is to break down the text and generate a list of statements contained within the text. Single words and ambiguous phrases should be considered statements.
-
-===== START OF EXAMPLES =====
-Example 1:
-Example text: The weather is sunny today. Temperature is 75 degrees. Don't forget your sunscreen!
-
-Output:
-{{
-    "statements": ["The weather is sunny today", "Temperature is 75 degrees", "Don't forget your sunscreen!"]
-}}
-
-Example 2:
-Example text: I love pizza. It has cheese and tomato sauce and the crust is crispy.
-
-Output:
-{{
-    "statements": ["I love pizza", "It has cheese and tomato sauce", "The crust is crispy"]
-}}
-===== END OF EXAMPLES =====
-
-        
-**
-IMPORTANT: Please return your answer in valid JSON format, with the "statements" key mapping to a list of strings. No words or explanation is needed.
-**
-
-==== START OF INPUT ====
-Text:
-{expected_output}
-==== END OF INPUT ====
-
-==== YOUR ANSWER ====
-JSON:
-"""
-
-    @staticmethod
-    def generate_verdicts(statements, actual_output):
-        return f"""You will be provided with:\n
-- a list of statements from a text that we will refer to as expected output
-- a text that we will refer to as actual output\n
-
-Your task is to determine whether each statement from the expected output is correct/consistent with the actual output text.
-More specifically, you should generate a JSON object with the key "verdicts". "verdicts" will map to a list of nested JSON objects with two keys: `verdict` and `reason`.
-The "reason" key should provide an explanation for your choice, regardless of which verdict you select. Try providing quotes from the text(s) to justify your answer where possible.
-The "verdict" key be EXACTLY EITHER "yes" or "no". You should select "yes" if the statement is correct/consistent based on the actual output and "no" otherwise.
-
-==== OUTPUT FORMATTING ====
-IMPORTANT: Please make sure to only return in JSON format, with the "verdicts" key mapping to a list of JSON objects. Each JSON object should contain keys "verdict" (one of "yes" or "no") and "reason" (str). 
-
-==== START OF EXAMPLES ====
-Example input 1: What's the capital of France?
-Example expected output statements 1: ["Paris is the capital of France", "It is located in northern France", "The city has a population of over 2 million"]
-Example actual output 1: "Paris is the capital city of France. It is situated in the northern part of the country and has over 2 million residents."
-Example JSON 1:
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes",
-            "reason": "The actual output directly states 'Paris is the capital city of France', which matches the statement"
-        }},
-        {{
-            "verdict": "yes", 
-            "reason": "The actual output confirms this by saying it is 'situated in the northern part of the country'"
-        }},
-        {{
-            "verdict": "yes",
-            "reason": "The actual output mentions the city 'has over 2 million residents', matching the population statement"
-        }}
-    ]
-}}
-
-Example input 2: What is the largest planet in our solar system?
-Example expected output statements 2: ["Jupiter is the largest planet", "It is a gas giant", "Jupiter has 79 known moons", "The Great Red Spot is a storm on Jupiter"]
-Example actual output 2: "Jupiter is the biggest planet in the solar system. It is made mostly of gas. The planet has many moons orbiting it."
-Example JSON 2:
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes",
-            "reason": "The actual output confirms 'Jupiter is the biggest planet', which is equivalent to it being the largest"
-        }},
-        {{
-            "verdict": "yes",
-            "reason": "The actual output states it is 'made mostly of gas', indicating it is a gas giant"
-        }},
-        {{
-            "verdict": "no",
-            "reason": "While the actual output mentions Jupiter has 'many moons', it does not specify the exact number of 79 known moons"
-        }},
-        {{
-            "verdict": "no",
-            "reason": "The actual output makes no mention of the Great Red Spot or any storms on Jupiter"
-        }}
-    ]
-}}
-==== END OF EXAMPLES ====
-
-** LASTLY **
-Since you are tasked to choose a verdict for each statement, the number of "verdicts" SHOULD BE EXACTLY EQUAL to the number of "statements".
-
-
-==== YOUR TURN =====
-
-Statements:
-{statements}
-
-Actual output:
-{actual_output}
-
-JSON:
-"""
-
-    @staticmethod
-    def generate_reason(incorrect_statements: List[Tuple[str, str]], score: float):
-        incorrect_statements = "\n".join([f"statement: {statement}\nreason: {reason}\n------" for statement, reason in incorrect_statements])
-        return f"""==== TASK INSTRUCTIONS ====\nYou will provided with two inputs: an answer correctness score and a list of inconsistent/incorrect statements made in a model's output (with the reason why it's irrelevant). Your task is to provide a CLEAR and CONCISE reason for the answer correctness score. 
-For context, there were a list of statements generated from an expected output. The model's actual output was then compared to the expected output, and we collected a list of claims made in the expected output that were either incorrect or inconsistent with the actual output. 
-The score represents how well the model's output matches the expected output.
-You should explain why the score is not higher, but also include why its current score is fair.
-The incorrect statements represent parts of the model output that are incorrect or inconsistent with the expected output. The incorrect statement will be paired with the reason why it's incorrect.
-If there are no incorrect statements, instead respond with a positive remark with an upbeat encouraging tone (but don't overblow the kind attitude).
-
-
-==== FORMATTING YOUR ANSWER ====
-IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-Example JSON:
-{{
-    "reason": "The score is <answer_relevancy_score> because <your_reason>."
-}}
-
-==== YOUR TURN ====
----- ANSWER CORRECTNESS SCORE ----
-{score}
-
----- INCORRECT STATEMENTS ----
-{incorrect_statements}
-
----- YOUR RESPONSE ----
-JSON:
-"""
-

judgeval/scorers/judgeval_scorers/local_implementations/answer_relevancy/__init__.py

@@ -1,4 +0,0 @@
-from judgeval.scorers.judgeval_scorers.local_implementations.answer_relevancy.answer_relevancy_scorer import AnswerRelevancyScorer
-
-
-__all__ = ["AnswerRelevancyScorer"]