judgeval 0.0.3__py3-none-any.whl → 0.0.5__py3-none-any.whl

judgeval/scorers/judgeval_scorers/{tool_correctness.py → api_scorers/tool_correctness.py}

@@ -6,11 +6,11 @@ TODO add link to docs page for this scorer
 """
 
 # Internal imports
-from judgeval.scorers.base_scorer import JudgmentScorer
+from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
 
 
-class ToolCorrectnessScorer(JudgmentScorer):
+class ToolCorrectnessScorer(APIJudgmentScorer):
     def __init__(self, threshold: float):
         super().__init__(threshold=threshold, score_type=APIScorer.TOOL_CORRECTNESS)
 

judgeval/scorers/judgeval_scorers/local_implementations/__init__.py

@@ -0,0 +1,24 @@
+from judgeval.scorers.judgeval_scorers.local_implementations.answer_relevancy.answer_relevancy_scorer import AnswerRelevancyScorer
+from judgeval.scorers.judgeval_scorers.local_implementations.contextual_precision.contextual_precision_scorer import ContextualPrecisionScorer
+from judgeval.scorers.judgeval_scorers.local_implementations.contextual_recall.contextual_recall_scorer import ContextualRecallScorer
+from judgeval.scorers.judgeval_scorers.local_implementations.contextual_relevancy.contextual_relevancy_scorer import ContextualRelevancyScorer
+from judgeval.scorers.judgeval_scorers.local_implementations.faithfulness.faithfulness_scorer import FaithfulnessScorer
+from judgeval.scorers.judgeval_scorers.local_implementations.json_correctness.json_correctness_scorer import JsonCorrectnessScorer
+from judgeval.scorers.judgeval_scorers.local_implementations.tool_correctness.tool_correctness_scorer import ToolCorrectnessScorer 
+from judgeval.scorers.judgeval_scorers.local_implementations.hallucination.hallucination_scorer import HallucinationScorer
+from judgeval.scorers.judgeval_scorers.local_implementations.summarization.summarization_scorer import SummarizationScorer
+from judgeval.scorers.judgeval_scorers.local_implementations.answer_correctness.answer_correctness_scorer import AnswerCorrectnessScorer
+
+
+__all__ = [
+    "AnswerCorrectnessScorer",
+    "AnswerRelevancyScorer",
+    "ContextualPrecisionScorer",
+    "ContextualRecallScorer",
+    "ContextualRelevancyScorer",
+    "FaithfulnessScorer",
+    "JsonCorrectnessScorer",
+    "ToolCorrectnessScorer",
+    "HallucinationScorer",
+    "SummarizationScorer",
+]

judgeval/scorers/judgeval_scorers/local_implementations/answer_correctness/__init__.py

@@ -0,0 +1,4 @@
+from judgeval.scorers.judgeval_scorers.local_implementations.answer_correctness.answer_correctness_scorer import AnswerCorrectnessScorer
+
+
+__all__ = ["AnswerCorrectnessScorer"]

judgeval/scorers/judgeval_scorers/local_implementations/answer_correctness/answer_correctness_scorer.py

@@ -0,0 +1,272 @@
+from typing import Optional, List, Union, Tuple
+from pydantic import BaseModel
+
+from judgeval.judges import JudgevalJudge
+from judgeval.judges.utils import create_judge
+from judgeval.data import Example, ExampleParams
+from judgeval.scorers import JudgevalScorer
+from judgeval.scorers.utils import (
+    get_or_create_event_loop,
+    parse_response_json,
+    scorer_progress_meter,
+    create_verbose_logs,
+    check_example_params,
+)
+from .prompts import (
+    ACVerdict,
+    AnswerCorrectnessTemplate,
+    Statements,
+    Verdicts,
+    Reason,
+)
+
+
+required_params = [
+    ExampleParams.INPUT,
+    ExampleParams.ACTUAL_OUTPUT,
+    ExampleParams.EXPECTED_OUTPUT,
+]
+
+
+class AnswerCorrectnessScorer(JudgevalScorer):
+    def __init__(
+        self,
+        threshold: float = 0.5,
+        model: Optional[Union[str, JudgevalJudge]] = None,
+        include_reason: bool = True,
+        async_mode: bool = True,
+        strict_mode: bool = False,
+        verbose_mode: bool = False
+    ):
+        self.threshold = 1 if strict_mode else threshold
+        self.include_reason = include_reason
+        self.model, self.using_native_model = create_judge(model)
+        self.evaluation_model = self.model.get_model_name()
+        self.async_mode = async_mode
+        self.strict_mode = strict_mode
+        self.verbose_mode = verbose_mode
+
+    async def _a_get_statements(self, expected_output: str) -> List[str]:
+        prompt = AnswerCorrectnessTemplate.deduce_statements(
+            expected_output=expected_output,
+        )
+        if self.using_native_model:
+            res = await self.model.a_generate(prompt)
+            data = parse_response_json(res, self)
+            return data["statements"]
+        else:
+            try:
+                res: Statements = await self.model.a_generate(
+                    prompt, schema=Statements
+                )
+                return res.statements
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = parse_response_json(res, self)
+                return data["statements"]
+
+    def _get_statements(self, expected_output: str) -> List[str]:
+        prompt = AnswerCorrectnessTemplate.deduce_statements(
+            expected_output=expected_output,
+        )
+        if self.using_native_model:
+            res = self.model.generate(prompt)
+            data = parse_response_json(res, self)
+            return data["statements"]
+        else:
+            try:
+                res: Statements = self.model.generate(
+                    prompt, schema=Statements
+                )
+                return res.statements
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = parse_response_json(res, self)
+                return data["statements"]
+
+    async def _a_get_verdicts(self, actual_output: str) -> List[ACVerdict]:
+        if len(self.statements) == 0:
+            return []
+
+        prompt = AnswerCorrectnessTemplate.generate_verdicts(
+            actual_output=actual_output,
+            statements=self.statements,
+        )
+
+        if self.using_native_model:
+            res = await self.model.a_generate(prompt)
+            data = parse_response_json(res, self)
+            return [ACVerdict(**item) for item in data["verdicts"]]
+        else:
+            try:
+                res: Verdicts = await self.model.a_generate(prompt, schema=Verdicts)
+                return [item for item in res.verdicts]
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = parse_response_json(res, self)
+                return [ACVerdict(**item) for item in data["verdicts"]]
+
+    def _get_verdicts(self, actual_output: str) -> List[ACVerdict]:
+        if len(self.statements) == 0:
+            return []
+
+        prompt = AnswerCorrectnessTemplate.generate_verdicts(
+            actual_output=actual_output,
+            statements=self.statements,
+        )
+
+        if self.using_native_model:
+            res = self.model.generate(prompt)
+            data = parse_response_json(res, self)
+            return [ACVerdict(**item) for item in data["verdicts"]]
+        else:
+            try:
+                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
+                return [item for item in res.verdicts]
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = parse_response_json(res, self)
+                return [ACVerdict(**item) for item in data["verdicts"]]
+
+    async def _a_get_reason(self) -> str:
+        if self.include_reason is False:
+            return None
+
+        incorrect_statements: List[Tuple[str, str]] = []
+        for idx, verdict in enumerate(self.verdicts):
+            if verdict.verdict.strip().lower() == "no":
+                incorrect_statements.append((self.statements[idx], verdict.reason))
+
+        prompt = AnswerCorrectnessTemplate.generate_reason(
+            incorrect_statements=incorrect_statements,
+            score=format(self.score, ".2f"),
+        )
+        if self.using_native_model:
+            res = await self.model.a_generate(prompt)
+            data = parse_response_json(res, self)
+            return data["reason"]
+        else:
+            try:
+                res: Reason = await self.model.a_generate(
+                    prompt=prompt, schema=Reason
+                )
+                return res.reason
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = parse_response_json(res, self)
+                return data["reason"]
+
+    def _get_reason(self) -> str:
+        if self.include_reason is False:
+            return None
+
+        incorrect_statements: List[Tuple[str, str]] = []
+        for idx, verdict in enumerate(self.verdicts):
+            if verdict.verdict.strip().lower() == "no":
+                incorrect_statements.append((self.statements[idx], verdict.reason))
+
+        prompt = AnswerCorrectnessTemplate.generate_reason(
+            incorrect_statements=incorrect_statements,
+            score=format(self.score, ".2f"),
+        )
+        if self.using_native_model:
+            res = self.model.generate(prompt)
+            data = parse_response_json(res, self)
+            return data["reason"]
+        else:
+            try:
+                res: Reason = self.model.generate(
+                    prompt=prompt, schema=Reason
+                )
+                return res.reason
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = parse_response_json(res, self)
+                return data["reason"]
+
+    def _compute_score(self) -> float:
+        number_of_verdicts = len(self.verdicts)
+        if number_of_verdicts == 0:
+            return 1
+
+        correct_count = 0
+        for verdict in self.verdicts:
+            if verdict.verdict.strip().lower() == "yes":
+                correct_count += 1
+
+        score = correct_count / number_of_verdicts
+        return 0 if self.strict_mode and score < self.threshold else score
+
+    def score_example(
+        self,
+        example: Example,
+        _show_indicator: bool = True,
+    ) -> float:
+        check_example_params(example, required_params, self)
+
+        with scorer_progress_meter(self, display_meter=_show_indicator):
+            try:
+                if self.async_mode:
+                    loop = get_or_create_event_loop()
+                    loop.run_until_complete(
+                        self.a_score_example(example, _show_indicator=False)
+                    )
+                else:
+                    self.statements = self._get_statements(example.expected_output)
+                    self.verdicts = self._get_verdicts(example.actual_output)
+                    self.score = self._compute_score()
+                    self.reason = self._get_reason()
+                    self.success = self.score >= self.threshold
+                    self.verbose_logs = create_verbose_logs(
+                        self,
+                        steps=[
+                            f"Statements:\n{self.statements}",
+                            f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
+                            f"Score: {self.score}\nReason: {self.reason}",
+                        ],
+                    )
+                return self.score
+            except Exception as e:
+                print(f"Error in score_example for AnswerCorrectnessScorer: {e}")
+                raise
+    
+    async def a_score_example(
+        self, 
+        example: Example,
+        _show_indicator: bool = True,
+    ) -> float:
+        check_example_params(example, required_params, self)
+
+        with scorer_progress_meter(self, async_mode=True, display_meter=_show_indicator):
+            try:
+                self.statements: List[str] = await self._a_get_statements(example.expected_output)
+                self.verdicts: List[ACVerdict] = await self._a_get_verdicts(example.actual_output)
+                self.score = self._compute_score()
+                self.reason = await self._a_get_reason()
+                self.success = self.score >= self.threshold
+                self.verbose_logs = create_verbose_logs(
+                    self,
+                    steps=[
+                        f"Statements:\n{self.statements}",
+                        f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
+                        f"Score: {self.score}\nReason: {self.reason}",
+                    ],
+                )
+                return self.score
+            except Exception as e:
+                print(f"Error in a_score_example for AnswerCorrectnessScorer: {e}")
+                raise
+
+    def _success_check(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Answer Correctness"

judgeval/scorers/judgeval_scorers/local_implementations/answer_correctness/prompts.py

@@ -0,0 +1,169 @@
+"""
+Util prompts for AnswerCorrectnessScorer
+"""
+
+from typing import List, Optional, Tuple
+from pydantic import BaseModel, Field
+
+
+# BaseModels to enforce formatting in LLM JSON response
+class Statements(BaseModel):
+    statements: List[str]
+
+
+class ACVerdict(BaseModel):
+    verdict: str
+    reason: str
+
+
+class Verdicts(BaseModel):
+    verdicts: List[ACVerdict]
+
+
+class Reason(BaseModel):
+    reason: str
+
+
+class AnswerCorrectnessTemplate:
+    @staticmethod
+    def deduce_statements(expected_output):
+        return f"""You will be presented with a piece of text. Your task is to break down the text and generate a list of statements contained within the text. Single words and ambiguous phrases should be considered statements.
+
+===== START OF EXAMPLES =====
+Example 1:
+Example text: The weather is sunny today. Temperature is 75 degrees. Don't forget your sunscreen!
+
+Output:
+{{
+    "statements": ["The weather is sunny today", "Temperature is 75 degrees", "Don't forget your sunscreen!"]
+}}
+
+Example 2:
+Example text: I love pizza. It has cheese and tomato sauce and the crust is crispy.
+
+Output:
+{{
+    "statements": ["I love pizza", "It has cheese and tomato sauce", "The crust is crispy"]
+}}
+===== END OF EXAMPLES =====
+
+        
+**
+IMPORTANT: Please return your answer in valid JSON format, with the "statements" key mapping to a list of strings. No words or explanation is needed.
+**
+
+==== START OF INPUT ====
+Text:
+{expected_output}
+==== END OF INPUT ====
+
+==== YOUR ANSWER ====
+JSON:
+"""
+
+    @staticmethod
+    def generate_verdicts(statements, actual_output):
+        return f"""You will be provided with:\n
+- a list of statements from a text that we will refer to as expected output
+- a text that we will refer to as actual output\n
+
+Your task is to determine whether each statement from the expected output is correct/consistent with the actual output text.
+More specifically, you should generate a JSON object with the key "verdicts". "verdicts" will map to a list of nested JSON objects with two keys: `verdict` and `reason`.
+The "reason" key should provide an explanation for your choice, regardless of which verdict you select. Try providing quotes from the text(s) to justify your answer where possible.
+The "verdict" key be EXACTLY EITHER "yes" or "no". You should select "yes" if the statement is correct/consistent based on the actual output and "no" otherwise.
+
+==== OUTPUT FORMATTING ====
+IMPORTANT: Please make sure to only return in JSON format, with the "verdicts" key mapping to a list of JSON objects. Each JSON object should contain keys "verdict" (one of "yes" or "no") and "reason" (str). 
+
+==== START OF EXAMPLES ====
+Example input 1: What's the capital of France?
+Example expected output statements 1: ["Paris is the capital of France", "It is located in northern France", "The city has a population of over 2 million"]
+Example actual output 1: "Paris is the capital city of France. It is situated in the northern part of the country and has over 2 million residents."
+Example JSON 1:
+{{
+    "verdicts": [
+        {{
+            "verdict": "yes",
+            "reason": "The actual output directly states 'Paris is the capital city of France', which matches the statement"
+        }},
+        {{
+            "verdict": "yes", 
+            "reason": "The actual output confirms this by saying it is 'situated in the northern part of the country'"
+        }},
+        {{
+            "verdict": "yes",
+            "reason": "The actual output mentions the city 'has over 2 million residents', matching the population statement"
+        }}
+    ]
+}}
+
+Example input 2: What is the largest planet in our solar system?
+Example expected output statements 2: ["Jupiter is the largest planet", "It is a gas giant", "Jupiter has 79 known moons", "The Great Red Spot is a storm on Jupiter"]
+Example actual output 2: "Jupiter is the biggest planet in the solar system. It is made mostly of gas. The planet has many moons orbiting it."
+Example JSON 2:
+{{
+    "verdicts": [
+        {{
+            "verdict": "yes",
+            "reason": "The actual output confirms 'Jupiter is the biggest planet', which is equivalent to it being the largest"
+        }},
+        {{
+            "verdict": "yes",
+            "reason": "The actual output states it is 'made mostly of gas', indicating it is a gas giant"
+        }},
+        {{
+            "verdict": "no",
+            "reason": "While the actual output mentions Jupiter has 'many moons', it does not specify the exact number of 79 known moons"
+        }},
+        {{
+            "verdict": "no",
+            "reason": "The actual output makes no mention of the Great Red Spot or any storms on Jupiter"
+        }}
+    ]
+}}
+==== END OF EXAMPLES ====
+
+** LASTLY **
+Since you are tasked to choose a verdict for each statement, the number of "verdicts" SHOULD BE EXACTLY EQUAL to the number of "statements".
+
+
+==== YOUR TURN =====
+
+Statements:
+{statements}
+
+Actual output:
+{actual_output}
+
+JSON:
+"""
+
+    @staticmethod
+    def generate_reason(incorrect_statements: List[Tuple[str, str]], score: float):
+        incorrect_statements = "\n".join([f"statement: {statement}\nreason: {reason}\n------" for statement, reason in incorrect_statements])
+        return f"""==== TASK INSTRUCTIONS ====\nYou will provided with two inputs: an answer correctness score and a list of inconsistent/incorrect statements made in a model's output (with the reason why it's irrelevant). Your task is to provide a CLEAR and CONCISE reason for the answer correctness score. 
+For context, there were a list of statements generated from an expected output. The model's actual output was then compared to the expected output, and we collected a list of claims made in the expected output that were either incorrect or inconsistent with the actual output. 
+The score represents how well the model's output matches the expected output.
+You should explain why the score is not higher, but also include why its current score is fair.
+The incorrect statements represent parts of the model output that are incorrect or inconsistent with the expected output. The incorrect statement will be paired with the reason why it's incorrect.
+If there are no incorrect statements, instead respond with a positive remark with an upbeat encouraging tone (but don't overblow the kind attitude).
+
+
+==== FORMATTING YOUR ANSWER ====
+IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+Example JSON:
+{{
+    "reason": "The score is <answer_relevancy_score> because <your_reason>."
+}}
+
+==== YOUR TURN ====
+---- ANSWER CORRECTNESS SCORE ----
+{score}
+
+---- INCORRECT STATEMENTS ----
+{incorrect_statements}
+
+---- YOUR RESPONSE ----
+JSON:
+"""
+

judgeval/scorers/judgeval_scorers/local_implementations/answer_relevancy/__init__.py

@@ -0,0 +1,4 @@
+from judgeval.scorers.judgeval_scorers.local_implementations.answer_relevancy.answer_relevancy_scorer import AnswerRelevancyScorer
+
+
+__all__ = ["AnswerRelevancyScorer"]