judgeval 0.0.11__py3-none-any.whl → 0.22.2__py3-none-any.whl

judgeval/scorers/judgeval_scorers/api_scorers/json_correctness.py

@@ -1,32 +0,0 @@
-"""
-`judgeval` JSON correctness scorer
-
-TODO add link to docs page for this scorer
-
-"""
-
-
-# External imports
-from pydantic import BaseModel, Field
-# Internal imports
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
-
-
-class JSONCorrectnessScorer(APIJudgmentScorer):
-    json_schema: BaseModel = Field(None, exclude=True)
-    
-    def __init__(self, threshold: float, json_schema: BaseModel):
-        super().__init__(threshold=threshold, score_type=APIScorer.JSON_CORRECTNESS)
-        object.__setattr__(self, 'json_schema', json_schema)
-    
-    def to_dict(self):
-        base_dict = super().to_dict()  # Get the parent class's dictionary
-        base_dict["kwargs"] = {
-            "json_schema": self.json_schema.model_json_schema()
-        }
-        return base_dict
-
-    @property
-    def __name__(self):
-        return "JSON Correctness"

judgeval/scorers/judgeval_scorers/api_scorers/summarization.py

@@ -1,20 +0,0 @@
-"""
-`judgeval` summarization scorer
-
-TODO add link to docs page for this scorer
-
-"""
-
-# Internal imports
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
-
-
-class SummarizationScorer(APIJudgmentScorer):
-    def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.SUMMARIZATION)
-
-    @property
-    def __name__(self):
-        return "Summarization"
-    

judgeval/scorers/judgeval_scorers/api_scorers/tool_correctness.py

@@ -1,19 +0,0 @@
-"""
-`judgeval` tool correctness scorer
-
-TODO add link to docs page for this scorer
-
-"""
-
-# Internal imports
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
-
-
-class ToolCorrectnessScorer(APIJudgmentScorer):
-    def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.TOOL_CORRECTNESS)
-
-    @property
-    def __name__(self):
-        return "Tool Correctness"

judgeval/scorers/judgeval_scorers/classifiers/__init__.py

@@ -1,3 +0,0 @@
-from .text2sql import Text2SQLScorer
-
-__all__ = ["Text2SQLScorer"]

judgeval/scorers/judgeval_scorers/classifiers/text2sql/__init__.py

@@ -1,3 +0,0 @@
-from .text2sql_scorer import Text2SQLScorer
-
-__all__ = ["Text2SQLScorer"]

judgeval/scorers/judgeval_scorers/classifiers/text2sql/text2sql_scorer.py

@@ -1,54 +0,0 @@
-"""
-ClassifierScorer implementation for basic Text-to-SQL evaluation.
-
-Takes a natural language query, a corresponding LLM-generated SQL query, and a table schema + (optional) metadata.
-Determines if the LLM-generated SQL query is valid and works for the natural language query.
-"""
-from judgeval.scorers import ClassifierScorer
-
-Text2SQLScorer = ClassifierScorer(
-    "Text to SQL",
-    slug="text2sql-1010101010",
-    threshold=1.0,
-    conversation=[{
-        "role": "system",
-        "content": """You will be given a natural language query, a corresponding LLM-generated SQL query, and a table schema + (optional) metadata.
-
-** TASK INSTRUCTIONS **
-Your task is to decide whether the LLM generated SQL query properly filters for what the natural language query is asking, based on the table schema + (optional) metadata. 
-Additionally, you should check if the SQL query is valid based on the table schema (checking for syntax errors, false column names, etc.)
-
-** TIPS **
-- Look for correct references to the table schema for column names, table names, etc.
-- Check that the SQL query can be executed; make sure JOINs, GROUP BYs, ORDER BYs, etc. are valid with respect to the table schema.
-- Check that aggregation functions (COUNT, SUM, AVG, etc.) are used appropriately with GROUP BY clauses
-- Verify that WHERE conditions use the correct operators and data types for comparisons
-- Ensure LIMIT and OFFSET clauses make sense for the query's purpose
-- Check that JOINs use the correct keys and maintain referential integrity
-- Verify that ORDER BY clauses use valid column names and sort directions
-- Check for proper handling of NULL values where relevant
-- Ensure subqueries are properly constructed and correlated when needed
-- EVEN IF THE QUERY IS VALID, IF IT DOESN'T WORK FOR THE NATURAL LANGUAGE QUERY, YOU SHOULD CHOOSE "N" AS THE ANSWER.
-
-** FORMATTING YOUR ANSWER **
-If the SQL query is valid and works for the natural language query, choose option "Y" and otherwise "N". Provide a justification for your decision; if you choose "N", explain what about the LLM-generated SQL query is incorrect, or explain why it doesn't address the natural language query. 
-IF YOUR JUSTIFICATION SHOWS THAT THE SQL QUERY IS VALID AND WORKS FOR THE NATURAL LANGUAGE QUERY, YOU SHOULD CHOOSE "Y" AS THE ANSWER. 
-IF THE SQL QUERY IS INVALID, YOU SHOULD CHOOSE "N" AS THE ANSWER. 
-
-** YOUR TURN **
-Natural language query:
-{{input}}
-
-LLM generated SQL query:
-{{actual_output}}
-
-Table schema:
-{{context}}
-        """
-    }],
-    options={
-        "Y": 1.0, 
-        "N": 0.0
-    }
-)
-

judgeval/scorers/judgeval_scorers/local_implementations/__init__.py

@@ -1,24 +0,0 @@
-from judgeval.scorers.judgeval_scorers.local_implementations.answer_relevancy.answer_relevancy_scorer import AnswerRelevancyScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.contextual_precision.contextual_precision_scorer import ContextualPrecisionScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.contextual_recall.contextual_recall_scorer import ContextualRecallScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.contextual_relevancy.contextual_relevancy_scorer import ContextualRelevancyScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.faithfulness.faithfulness_scorer import FaithfulnessScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.json_correctness.json_correctness_scorer import JsonCorrectnessScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.tool_correctness.tool_correctness_scorer import ToolCorrectnessScorer 
-from judgeval.scorers.judgeval_scorers.local_implementations.hallucination.hallucination_scorer import HallucinationScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.summarization.summarization_scorer import SummarizationScorer
-from judgeval.scorers.judgeval_scorers.local_implementations.answer_correctness.answer_correctness_scorer import AnswerCorrectnessScorer
-
-
-__all__ = [
-    "AnswerCorrectnessScorer",
-    "AnswerRelevancyScorer",
-    "ContextualPrecisionScorer",
-    "ContextualRecallScorer",
-    "ContextualRelevancyScorer",
-    "FaithfulnessScorer",
-    "JsonCorrectnessScorer",
-    "ToolCorrectnessScorer",
-    "HallucinationScorer",
-    "SummarizationScorer",
-]

judgeval/scorers/judgeval_scorers/local_implementations/answer_correctness/__init__.py

@@ -1,4 +0,0 @@
-from judgeval.scorers.judgeval_scorers.local_implementations.answer_correctness.answer_correctness_scorer import AnswerCorrectnessScorer
-
-
-__all__ = ["AnswerCorrectnessScorer"]

judgeval/scorers/judgeval_scorers/local_implementations/answer_correctness/answer_correctness_scorer.py

@@ -1,277 +0,0 @@
-from typing import Optional, List, Union, Tuple
-from pydantic import BaseModel
-
-from judgeval.constants import APIScorer
-from judgeval.judges import JudgevalJudge
-from judgeval.judges.utils import create_judge
-from judgeval.data import Example, ExampleParams
-from judgeval.scorers import JudgevalScorer
-from judgeval.scorers.utils import (
-    get_or_create_event_loop,
-    parse_response_json,
-    scorer_progress_meter,
-    create_verbose_logs,
-    check_example_params,
-)
-from .prompts import (
-    ACVerdict,
-    AnswerCorrectnessTemplate,
-    Statements,
-    Verdicts,
-    Reason,
-)
-
-
-required_params = [
-    ExampleParams.INPUT,
-    ExampleParams.ACTUAL_OUTPUT,
-    ExampleParams.EXPECTED_OUTPUT,
-]
-
-
-class AnswerCorrectnessScorer(JudgevalScorer):
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        model: Optional[Union[str, JudgevalJudge]] = None,
-        include_reason: bool = True,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False
-    ):
-        super().__init__(
-            score_type=APIScorer.ANSWER_CORRECTNESS,
-            threshold=1 if strict_mode else threshold,
-            evaluation_model=None,
-            include_reason=include_reason,
-            async_mode=async_mode,
-            strict_mode=strict_mode,
-            verbose_mode=verbose_mode
-        )
-        self.model, self.using_native_model = create_judge(model)
-        self.evaluation_model = self.model.get_model_name()
-
-    async def _a_get_statements(self, expected_output: str) -> List[str]:
-        prompt = AnswerCorrectnessTemplate.deduce_statements(
-            expected_output=expected_output,
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return data["statements"]
-        else:
-            try:
-                res: Statements = await self.model.a_generate(
-                    prompt, schema=Statements
-                )
-                return res.statements
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return data["statements"]
-
-    def _get_statements(self, expected_output: str) -> List[str]:
-        prompt = AnswerCorrectnessTemplate.deduce_statements(
-            expected_output=expected_output,
-        )
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return data["statements"]
-        else:
-            try:
-                res: Statements = self.model.generate(
-                    prompt, schema=Statements
-                )
-                return res.statements
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return data["statements"]
-
-    async def _a_get_verdicts(self, actual_output: str) -> List[ACVerdict]:
-        if len(self.statements) == 0:
-            return []
-
-        prompt = AnswerCorrectnessTemplate.generate_verdicts(
-            actual_output=actual_output,
-            statements=self.statements,
-        )
-
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return [ACVerdict(**item) for item in data["verdicts"]]
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(prompt, schema=Verdicts)
-                return [item for item in res.verdicts]
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return [ACVerdict(**item) for item in data["verdicts"]]
-
-    def _get_verdicts(self, actual_output: str) -> List[ACVerdict]:
-        if len(self.statements) == 0:
-            return []
-
-        prompt = AnswerCorrectnessTemplate.generate_verdicts(
-            actual_output=actual_output,
-            statements=self.statements,
-        )
-
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return [ACVerdict(**item) for item in data["verdicts"]]
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                return [item for item in res.verdicts]
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return [ACVerdict(**item) for item in data["verdicts"]]
-
-    async def _a_get_reason(self) -> str:
-        if self.include_reason is False:
-            return None
-
-        incorrect_statements: List[Tuple[str, str]] = []
-        for idx, verdict in enumerate(self.verdicts):
-            if verdict.verdict.strip().lower() == "no":
-                incorrect_statements.append((self.statements[idx], verdict.reason))
-
-        prompt = AnswerCorrectnessTemplate.generate_reason(
-            incorrect_statements=incorrect_statements,
-            score=format(self.score, ".2f"),
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = await self.model.a_generate(
-                    prompt=prompt, schema=Reason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    def _get_reason(self) -> str:
-        if self.include_reason is False:
-            return None
-
-        incorrect_statements: List[Tuple[str, str]] = []
-        for idx, verdict in enumerate(self.verdicts):
-            if verdict.verdict.strip().lower() == "no":
-                incorrect_statements.append((self.statements[idx], verdict.reason))
-
-        prompt = AnswerCorrectnessTemplate.generate_reason(
-            incorrect_statements=incorrect_statements,
-            score=format(self.score, ".2f"),
-        )
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = self.model.generate(
-                    prompt=prompt, schema=Reason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    def _compute_score(self) -> float:
-        number_of_verdicts = len(self.verdicts)
-        if number_of_verdicts == 0:
-            return 1
-
-        correct_count = 0
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() == "yes":
-                correct_count += 1
-
-        score = correct_count / number_of_verdicts
-        return 0 if self.strict_mode and score < self.threshold else score
-
-    def score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(self, display_meter=_show_indicator):
-            try:
-                if self.async_mode:
-                    loop = get_or_create_event_loop()
-                    loop.run_until_complete(
-                        self.a_score_example(example, _show_indicator=False)
-                    )
-                else:
-                    self.statements = self._get_statements(example.expected_output)
-                    self.verdicts = self._get_verdicts(example.actual_output)
-                    self.score = self._compute_score()
-                    self.reason = self._get_reason()
-                    self.success = self.score >= self.threshold
-                    self.verbose_logs = create_verbose_logs(
-                        self,
-                        steps=[
-                            f"Statements:\n{self.statements}",
-                            f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
-                            f"Score: {self.score}\nReason: {self.reason}",
-                        ],
-                    )
-                return self.score
-            except Exception as e:
-                print(f"Error in score_example for AnswerCorrectnessScorer: {e}")
-                raise
-    
-    async def a_score_example(
-        self, 
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(self, async_mode=True, display_meter=_show_indicator):
-            try:
-                self.statements: List[str] = await self._a_get_statements(example.expected_output)
-                self.verdicts: List[ACVerdict] = await self._a_get_verdicts(example.actual_output)
-                self.score = self._compute_score()
-                self.reason = await self._a_get_reason()
-                self.success = self.score >= self.threshold
-                self.verbose_logs = create_verbose_logs(
-                    self,
-                    steps=[
-                        f"Statements:\n{self.statements}",
-                        f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
-                        f"Score: {self.score}\nReason: {self.reason}",
-                    ],
-                )
-                return self.score
-            except Exception as e:
-                print(f"Error in a_score_example for AnswerCorrectnessScorer: {e}")
-                raise
-
-    def _success_check(self) -> bool:
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score >= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Answer Correctness"

judgeval/scorers/judgeval_scorers/local_implementations/answer_correctness/prompts.py

@@ -1,169 +0,0 @@
-"""
-Util prompts for AnswerCorrectnessScorer
-"""
-
-from typing import List, Optional, Tuple
-from pydantic import BaseModel, Field
-
-
-# BaseModels to enforce formatting in LLM JSON response
-class Statements(BaseModel):
-    statements: List[str]
-
-
-class ACVerdict(BaseModel):
-    verdict: str
-    reason: str
-
-
-class Verdicts(BaseModel):
-    verdicts: List[ACVerdict]
-
-
-class Reason(BaseModel):
-    reason: str
-
-
-class AnswerCorrectnessTemplate:
-    @staticmethod
-    def deduce_statements(expected_output):
-        return f"""You will be presented with a piece of text. Your task is to break down the text and generate a list of statements contained within the text. Single words and ambiguous phrases should be considered statements.
-
-===== START OF EXAMPLES =====
-Example 1:
-Example text: The weather is sunny today. Temperature is 75 degrees. Don't forget your sunscreen!
-
-Output:
-{{
-    "statements": ["The weather is sunny today", "Temperature is 75 degrees", "Don't forget your sunscreen!"]
-}}
-
-Example 2:
-Example text: I love pizza. It has cheese and tomato sauce and the crust is crispy.
-
-Output:
-{{
-    "statements": ["I love pizza", "It has cheese and tomato sauce", "The crust is crispy"]
-}}
-===== END OF EXAMPLES =====
-
-        
-**
-IMPORTANT: Please return your answer in valid JSON format, with the "statements" key mapping to a list of strings. No words or explanation is needed.
-**
-
-==== START OF INPUT ====
-Text:
-{expected_output}
-==== END OF INPUT ====
-
-==== YOUR ANSWER ====
-JSON:
-"""
-
-    @staticmethod
-    def generate_verdicts(statements, actual_output):
-        return f"""You will be provided with:\n
-- a list of statements from a text that we will refer to as expected output
-- a text that we will refer to as actual output\n
-
-Your task is to determine whether each statement from the expected output is correct/consistent with the actual output text.
-More specifically, you should generate a JSON object with the key "verdicts". "verdicts" will map to a list of nested JSON objects with two keys: `verdict` and `reason`.
-The "reason" key should provide an explanation for your choice, regardless of which verdict you select. Try providing quotes from the text(s) to justify your answer where possible.
-The "verdict" key be EXACTLY EITHER "yes" or "no". You should select "yes" if the statement is correct/consistent based on the actual output and "no" otherwise.
-
-==== OUTPUT FORMATTING ====
-IMPORTANT: Please make sure to only return in JSON format, with the "verdicts" key mapping to a list of JSON objects. Each JSON object should contain keys "verdict" (one of "yes" or "no") and "reason" (str). 
-
-==== START OF EXAMPLES ====
-Example input 1: What's the capital of France?
-Example expected output statements 1: ["Paris is the capital of France", "It is located in northern France", "The city has a population of over 2 million"]
-Example actual output 1: "Paris is the capital city of France. It is situated in the northern part of the country and has over 2 million residents."
-Example JSON 1:
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes",
-            "reason": "The actual output directly states 'Paris is the capital city of France', which matches the statement"
-        }},
-        {{
-            "verdict": "yes", 
-            "reason": "The actual output confirms this by saying it is 'situated in the northern part of the country'"
-        }},
-        {{
-            "verdict": "yes",
-            "reason": "The actual output mentions the city 'has over 2 million residents', matching the population statement"
-        }}
-    ]
-}}
-
-Example input 2: What is the largest planet in our solar system?
-Example expected output statements 2: ["Jupiter is the largest planet", "It is a gas giant", "Jupiter has 79 known moons", "The Great Red Spot is a storm on Jupiter"]
-Example actual output 2: "Jupiter is the biggest planet in the solar system. It is made mostly of gas. The planet has many moons orbiting it."
-Example JSON 2:
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes",
-            "reason": "The actual output confirms 'Jupiter is the biggest planet', which is equivalent to it being the largest"
-        }},
-        {{
-            "verdict": "yes",
-            "reason": "The actual output states it is 'made mostly of gas', indicating it is a gas giant"
-        }},
-        {{
-            "verdict": "no",
-            "reason": "While the actual output mentions Jupiter has 'many moons', it does not specify the exact number of 79 known moons"
-        }},
-        {{
-            "verdict": "no",
-            "reason": "The actual output makes no mention of the Great Red Spot or any storms on Jupiter"
-        }}
-    ]
-}}
-==== END OF EXAMPLES ====
-
-** LASTLY **
-Since you are tasked to choose a verdict for each statement, the number of "verdicts" SHOULD BE EXACTLY EQUAL to the number of "statements".
-
-
-==== YOUR TURN =====
-
-Statements:
-{statements}
-
-Actual output:
-{actual_output}
-
-JSON:
-"""
-
-    @staticmethod
-    def generate_reason(incorrect_statements: List[Tuple[str, str]], score: float):
-        incorrect_statements = "\n".join([f"statement: {statement}\nreason: {reason}\n------" for statement, reason in incorrect_statements])
-        return f"""==== TASK INSTRUCTIONS ====\nYou will provided with two inputs: an answer correctness score and a list of inconsistent/incorrect statements made in a model's output (with the reason why it's irrelevant). Your task is to provide a CLEAR and CONCISE reason for the answer correctness score. 
-For context, there were a list of statements generated from an expected output. The model's actual output was then compared to the expected output, and we collected a list of claims made in the expected output that were either incorrect or inconsistent with the actual output. 
-The score represents how well the model's output matches the expected output.
-You should explain why the score is not higher, but also include why its current score is fair.
-The incorrect statements represent parts of the model output that are incorrect or inconsistent with the expected output. The incorrect statement will be paired with the reason why it's incorrect.
-If there are no incorrect statements, instead respond with a positive remark with an upbeat encouraging tone (but don't overblow the kind attitude).
-
-
-==== FORMATTING YOUR ANSWER ====
-IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-Example JSON:
-{{
-    "reason": "The score is <answer_relevancy_score> because <your_reason>."
-}}
-
-==== YOUR TURN ====
----- ANSWER CORRECTNESS SCORE ----
-{score}
-
----- INCORRECT STATEMENTS ----
-{incorrect_statements}
-
----- YOUR RESPONSE ----
-JSON:
-"""
-

judgeval/scorers/judgeval_scorers/local_implementations/answer_relevancy/__init__.py

@@ -1,4 +0,0 @@
-from judgeval.scorers.judgeval_scorers.local_implementations.answer_relevancy.answer_relevancy_scorer import AnswerRelevancyScorer
-
-
-__all__ = ["AnswerRelevancyScorer"]