judgeval 0.0.31__py3-none-any.whl → 0.0.34__py3-none-any.whl

judgeval/scorers/judgeval_scorers/local_implementations/hallucination/hallucination_scorer.py

@@ -1,264 +0,0 @@
-"""
-Metric that evaluates hallucinations in model outputs
-
-The hallucination metric determines whether your LLM generates factually correct information by comparing 
-the actual_output to the provided context.
-
-If you're looking to evaluate hallucination for a RAG system, refer to the faithfulness metric instead.
-
-The HallucinationMetric uses an LLM to determine, for each context in contexts, whether there are any 
-contradictions to the actual_output.
-
-Although extremely similar to the FaithfulnessMetric, the HallucinationMetric is calculated differently 
-since it uses contexts as the source of truth instead. Since contexts is the ideal segment of your 
-knowledge base relevant to a specific input, the degree of hallucination can be measured by the degree 
-of which the contexts is disagreed upon.
-
-Faithfulness is measuring the number of statements in output that agree with contexts.
-Hallucination is measuring the fraction of contexts that agree with output (do not contradict == agree)
-"""
-
-from typing import Optional, Union, List
-
-from judgeval.constants import APIScorer
-from judgeval.scorers.utils import (
-    get_or_create_event_loop,
-    scorer_progress_meter,
-    create_verbose_logs,
-    parse_response_json,
-    check_example_params,
-)
-from judgeval.scorers import JudgevalScorer
-from judgeval.judges import JudgevalJudge
-from judgeval.judges.utils import create_judge
-from judgeval.data import Example, ExampleParams
-from judgeval.scorers.judgeval_scorers.local_implementations.hallucination.prompts import *
-
-
-required_params = [
-    ExampleParams.INPUT,
-    ExampleParams.ACTUAL_OUTPUT,
-    ExampleParams.CONTEXT,
-]
-
-
-class HallucinationScorer(JudgevalScorer):
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        model: Optional[Union[str, JudgevalJudge]] = None,
-        include_reason: bool = True,
-        async_mode: bool = False,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-    ):
-        super().__init__(
-            score_type=APIScorer.HALLUCINATION,
-            threshold=1 if strict_mode else threshold,
-            evaluation_model=None,
-            include_reason=include_reason,
-            async_mode=async_mode,
-            strict_mode=strict_mode,
-            verbose_mode=verbose_mode
-        )
-        self.model, self.using_native_model = create_judge(model)
-        self.evaluation_model = self.model.get_model_name()
-
-    def score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(self, display_meter=_show_indicator):
-            if self.async_mode:
-                loop = get_or_create_event_loop()
-                loop.run_until_complete(
-                    self.a_score_example(example, _show_indicator=False)
-                )
-            else:
-                self.verdicts: List[HallucinationVerdict] = (
-                    self._generate_verdicts(
-                        example.actual_output, example.context
-                    )
-                )
-                self.score = self._calculate_score()
-                self.reason = self._generate_reason()
-                self.success = self.score <= self.threshold
-                self.verbose_logs = create_verbose_logs(
-                    self,
-                    steps=[
-                        f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
-                        f"Score: {self.score}\nReason: {self.reason}",
-                    ],
-                )
-
-                return self.score
-
-    async def a_score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(
-            self, async_mode=True, display_meter=_show_indicator
-        ):
-            self.verdicts: List[HallucinationVerdict] = (
-                await self._a_generate_verdicts(
-                    example.actual_output, example.context
-                )
-            )
-            self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason()
-            self.success = self.score <= self.threshold
-            self.verbose_logs = create_verbose_logs(
-                self,
-                steps=[
-                    f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
-                    f"Score: {self.score}\nReason: {self.reason}",
-                ],
-            )
-
-            return self.score
-
-    async def _a_generate_reason(self):
-        if self.include_reason is False:
-            return None
-
-        contradictions = []
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() == "no":
-                contradictions.append(verdict.reason)
-
-        prompt: dict = HallucinationTemplate.generate_reason(
-            contradictions=contradictions,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = await self.model.a_generate(prompt, schema=Reason)
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    def _generate_reason(self):
-        if self.include_reason is False:
-            return None
-
-        factual_alignments = []
-        contradictions = []
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() == "no":
-                contradictions.append(verdict.reason)
-
-        prompt: dict = HallucinationTemplate.generate_reason(
-            factual_alignments=factual_alignments,
-            contradictions=contradictions,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = self.model.generate(prompt, schema=Reason)
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    async def _a_generate_verdicts(
-        self, actual_output: str, contexts: List[str]
-    ) -> List[HallucinationVerdict]:
-        verdicts: List[HallucinationVerdict] = []
-        prompt = HallucinationTemplate.generate_verdicts(
-            actual_output=actual_output, contexts=contexts
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            verdicts = [
-                HallucinationVerdict(**item) for item in data["verdicts"]
-            ]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                verdicts = [
-                    HallucinationVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def _generate_verdicts(
-        self, actual_output: str, contexts: List[str]
-    ) -> List[HallucinationVerdict]:
-        verdicts: List[HallucinationVerdict] = []
-        prompt = HallucinationTemplate.generate_verdicts(
-            actual_output=actual_output, contexts=contexts
-        )
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            verdicts = [
-                HallucinationVerdict(**item) for item in data["verdicts"]
-            ]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                verdicts = [
-                    HallucinationVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def _calculate_score(self) -> float:
-        number_of_verdicts = len(self.verdicts)
-        if number_of_verdicts == 0:
-            return 0
-
-        hallucination_count = 0
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() == "no":
-                hallucination_count += 1
-
-        score = hallucination_count / number_of_verdicts
-        return 1 if self.strict_mode and score > self.threshold else score
-
-    def _success_check(self) -> bool:
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score <= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Hallucination"

judgeval/scorers/judgeval_scorers/local_implementations/hallucination/prompts.py

@@ -1,104 +0,0 @@
-from typing import List
-from pydantic import BaseModel
-
-
-class HallucinationVerdict(BaseModel):
-    verdict: str
-    reason: str
-
-
-class Verdicts(BaseModel):
-    verdicts: List[HallucinationVerdict]
-
-
-class Reason(BaseModel):
-    reason: str
-
-
-class HallucinationTemplate:
-    @staticmethod
-    def generate_verdicts(actual_output, contexts):
-        return f"""==== TASK INSTRUCTIONS ====
-You will be provided with an `actual output` (the response of an LLM to a particular query) and `contexts` (ground truth contextual information from a knowledge base).
-Your task is to take each context in contexts and determine whether the `actual output` factually agrees with the context.
-
-Additional notes:
-You should NOT use any prior knowledge you have in your decision making process; take each context at face value. 
-Since you will determine a verdict for EACH context, the number of 'verdicts' is EXACTLY EQUAL TO the number of contexts. 
-You should be lenient in your judgment when the actual output lacks detail with respect to the context segment; you should ONLY provide a 'no' answer if the context contradicts the actual output.
-
-==== FORMATTING INSTRUCTIONS ====
-You should return a JSON object with a key 'verdicts', which is a list of JSON objects. Each JSON object corresponds to a context in `contexts`, and should have 2 fields: 'verdict' and 'reason'. 
-The 'verdict' key should be EXACTLY one of 'yes' or 'no', representing whether the `actual output` factually agrees with the context segment. 
-The 'reason' is the justification for the verdict. If your verdict is 'no', try to provide a correction in the reason. 
-
-==== EXAMPLE ====
-Example contexts: ["Einstein won the Nobel Prize for his discovery of the photoelectric effect.", "Einstein won the Nobel Prize in 1968."]
-Example actual output: "Einstein won the Nobel Prize in 1969 for his discovery of the photoelectric effect."
-
-Example:
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes",
-            "reason": "The actual output agrees with the provided context which states that Einstein won the Nobel Prize for his discovery of the photoelectric effect."
-        }},
-        {{
-            "verdict": "no",
-            "reason": "The actual output contradicts the provided context which states that Einstein won the Nobel Prize in 1968, not 1969."
-        }}
-    ]  
-}}
-
-==== YOUR TURN ====
-Contexts:
-{contexts}
-
-Actual Output:
-{actual_output}
-
-JSON:
-"""
-
-    @staticmethod
-    def generate_reason(contradictions, score):
-        return f"""==== TASK INSTRUCTIONS ====
-An LLM has been provided with a list of `contexts` (ground truth contextual information from a knowledge base) and `actual output` (the response of an LLM to a particular query). 
-You will be provided with a list of `contradictions`, which are factual discrepancies between the context segments and the actual output. 
-Additionally, you will be provided with a hallucination score, which is a float (0 - 1, where 0 is the best score) indicating the fraction of context segments that contradict the actual output.
-
-Your task is to provide a CLEAR and CONCISE reason for the hallucination score. 
-If the hallucination score is 0 (no contradictions), you should instead respond with a positive remark with an upbeat encouraging tone (but don't overblow the kind attitude).
-        
-==== FORMATTING INSTRUCTIONS ====
-Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-Example JSON:
-{{
-    "reason": "The score is <hallucination_score> because <your_reason>."
-}}
-
-==== EXAMPLE ====
-Example Contradictions:
-[
-    "The actual output claims Einstein won the Nobel Prize in 1969, which contradicts the context stating he won it in 1968.",
-    "The actual output states Einstein was a chemist, but the context indicates he was a physicist.",
-    "The actual output claims Einstein was born in Switzerland, while the context states he was born in Germany."
-]
-
-Example Hallucination Score: 
-0.75
-
-Example Response:
-{{
-    "reason": "The score is 0.75 because the actual output made multiple factual errors: incorrectly stating Einstein's Nobel Prize year (1969 vs 1968), his profession (chemist vs physicist), and birthplace (Switzerland vs Germany)."
-}}
-
-==== YOUR TURN ====
-Contradictions:
-{contradictions}
-
-Hallucination Score:
-{score}
-
-JSON:
-"""

judgeval/scorers/judgeval_scorers/local_implementations/instruction_adherence/instruction_adherence.py

@@ -1,232 +0,0 @@
-from typing import Optional, List, Union, Tuple
-from pydantic import BaseModel
-
-from judgeval.constants import APIScorer
-from judgeval.scorers.utils import (get_or_create_event_loop,
-                                    scorer_progress_meter,
-                                    create_verbose_logs,
-                                    parse_response_json,
-                                    check_example_params
-                                    )
-from judgeval.scorers import JudgevalScorer
-from judgeval.judges import JudgevalJudge
-from judgeval.judges.utils import create_judge
-from judgeval.data import Example, ExampleParams
-from judgeval.scorers.judgeval_scorers.local_implementations.instruction_adherence.prompt import (
-    InstructionAdherenceTemplate,
-)
-required_params = [
-    ExampleParams.INPUT,
-    ExampleParams.ACTUAL_OUTPUT,
-]
-
-class Instructions(BaseModel):
-    instructions: List[str]
-
-class Verdict(BaseModel):
-    instruction: str
-    score: float
-    reason: str
-
-class ListOfVerdicts(BaseModel):
-    verdicts: List[Verdict]
-
-class InstructionAdherenceScorer(JudgevalScorer):
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        model: Optional[Union[str, JudgevalJudge]] = None,
-        include_reason: bool = True,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-    ):
-        super().__init__(
-            score_type=APIScorer.INSTRUCTION_ADHERENCE,
-            threshold=1 if strict_mode else threshold,
-            evaluation_model=None,
-            include_reason=include_reason,
-            async_mode=async_mode,
-            strict_mode=strict_mode,
-            verbose_mode=verbose_mode
-        )
-        self.model, self.using_native_model = create_judge(model)
-        self.evaluation_model = self.model.get_model_name()
-
-    def score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(self, display_meter=_show_indicator):
-            try:
-                if self.async_mode:
-                    loop = get_or_create_event_loop()
-                    loop.run_until_complete(
-                        self.a_score_example(example, _show_indicator=False)
-                    )
-                else:
-                    self.instructions: List[str] = self._get_instructions(example.input)
-                    self.verdicts: List[Verdict] = (
-                        self._get_verdicts(self.instructions, example.actual_output)
-                    )
-                    self.score = self._compute_score()
-                    self.reason = str(self.verdicts)
-                    self.success = self.score >= self.threshold
-                    self.verbose_logs = create_verbose_logs(
-                        self,
-                        steps=[
-                            f"Instructions:\n{self.instructions}",
-                            f"Score: {self.score}\nReason: {self.reason}",
-                        ],
-                    )
-                    return self.score
-            except Exception as e:
-                raise
-
-    async def a_score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-        try:
-            with scorer_progress_meter(
-                self, async_mode=True, display_meter=_show_indicator
-            ):
-                self.instructions: List[str] = await self._a_get_instructions(example.input)
-                self.verdicts: List[Verdict] = (
-                    await self._a_get_verdicts(self.instructions, example.actual_output)
-                )
-                self.score = self._compute_score()
-                self.reason = str(self.verdicts)
-                self.success = self.score >= self.threshold
-                self.verbose_logs = create_verbose_logs(
-                    self,
-                    steps=[
-                        f"Instructions:\n{self.instructions}",
-                        f"Score: {self.score}\nReason: {self.reason}",
-                    ],
-                )
-                return self.score
-        except Exception as e:
-            raise e
-
-
-    async def _a_get_verdicts(
-        self, instructions: List[str], actual_output: str
-    ) -> List[Verdict]:
-        if len(instructions) == 0:
-            return []
-
-        prompt = InstructionAdherenceTemplate.generate_verdicts(
-            instructions=instructions,
-            actual_output=actual_output,
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return [
-                Verdict(**item) for item in data["verdicts"]
-            ]
-        else:
-            try:
-                res: List[Verdict] = await self.model.a_generate(
-                    prompt, schema=List[Verdict]
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return [
-                    Verdict(**item) for item in data["verdicts"]
-                ]
-
-    def _get_verdicts(self, instructions: List[str], actual_output: str) -> List[Verdict]:
-        if len(instructions) == 0:
-            return []
-
-        prompt = InstructionAdherenceTemplate.generate_verdicts(
-            instructions=instructions,
-            actual_output=actual_output,
-        )
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return [Verdict(**item) for item in data["verdicts"]]
-        else:
-            try:
-                res: List[Verdict] = self.model.generate(prompt, schema=List[Verdict])
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return [
-                    Verdict(**item) for item in data["verdicts"]
-                ]
-
-    async def _a_get_instructions(
-        self,
-        input: str,
-    ) -> List[str]:
-        prompt = InstructionAdherenceTemplate.get_instructions(
-            input=input,
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return data["instructions"]
-        else:
-            try:
-                res: List[str] = await self.model.a_generate(
-                    prompt, schema=List[str]
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return data["instructions"]
-
-    def _get_instructions(
-        self,
-        input: str,
-    ) -> List[str]:
-        prompt = InstructionAdherenceTemplate.get_instructions(
-            input=input,
-        )
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return data["instructions"]
-        else:
-            try:
-                res: List[str] = self.model.generate(prompt, schema=List[str])
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return data["instructions"]
-
-    def _compute_score(self):
-        if len(self.verdicts) == 0:
-            return 1
-        score = 0
-        for verdict in self.verdicts:
-            score += verdict.score
-        return score / len(self.verdicts)
-
-    def success_check(self) -> bool:
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score >= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Instruction Adherence"

judgeval/scorers/judgeval_scorers/local_implementations/instruction_adherence/prompt.py

@@ -1,102 +0,0 @@
-"""
-Util prompts for InstructionAdherenceScorer
-"""
-
-from typing import List, Optional, Tuple
-from pydantic import BaseModel, Field
-
-
-class InstructionAdherenceTemplate:
-    @staticmethod
-    def get_instructions(input):
-        return f"""You will be presented with a piece of text. Your task is to break down the text and generate a list of the instructions contained within the text.
-
-===== START OF EXAMPLES =====
-Example 1:
-Example text: Hello my name is John Doe. I like cars. Write two poems about the weather and create a joke. Also what is 5 + 5?
-
-Output:
-{{
-    "instructions": ["Write two poem about the weather", "Create a joke", "What is 5 + 5?"]
-}}
-===== END OF EXAMPLES =====
-
-        
-**
-IMPORTANT: Please return your answer in valid JSON format, with the "instructions" key mapping to a list of strings. No words or explanation is needed.
-**
-
-==== START OF INPUT ====
-Text:
-{input}
-==== END OF INPUT ====
-
-==== YOUR ANSWER ====
-JSON:
-"""
-
-    @staticmethod
-    def generate_verdicts(instructions, actual_output):
-        return f"""
-        You will be presented with a list of instructions and a piece of text. For each instruction, determine if the instruction was completed in the text. There are 3 categories: either completed, partially completed, or not completed. The scores for these will be 1, 0.5, and 0 respectively.
-        Go through each instruction and provide score for each instruction as well as the reasoning for that score.
-
-        ==== FORMATTING YOUR ANSWER ====
-        Please return your answer in JSON format, with a list of JSON objects with keys "instruction", "score", and "reason". No words or explanation beyond the output JSON is needed.
-
-
-        ===== START OF EXAMPLES =====
-        Example 1:
-        instructions: ["Write two poems about the weather", "Create a joke", "What is 5 + 5?"]
-        output: Poem 1: The Sun's Embrace
-        The sun climbs high, a golden flame,
-        It whispers warmth, it calls my name.
-        The sky, a canvas, blue and clear,
-        A perfect day for cars, my dear.
-
-        The asphalt hums beneath the wheels,
-        A symphony of speed it feels.
-        The weather smiles, no clouds in sight,
-        A driver's joy, pure delight.
-
-        Poem 2: The Storm's Dance
-        A sunlit meadow, alive with whispers of wind, where daisies dance and hope begins again. Each petal holds a promise—bright, unbruised— a symphony of light that cannot be refused.
-
-        Joke
-        Why dont cars ever get cold in the winter?
-        Because they have radiators!
-
-        Math Answer
-        5 + 5 = 10
-        
-        YOUR JSON OUTPUT:
-        {{
-            [
-                {{
-                    "instruction": "Write two poem about the weather",
-                    "score": 0.5,
-                    "reason": "The output contained one poem about the weather, but the other poem was not about the weather."
-                }},
-                {{
-                    "instruction": "Create a joke",
-                    "score": 1,
-                    "reason": "There was a joke created in the output."
-                }},
-                {{
-                    "instruction": "What is 5 + 5?",
-                    "score": 1,
-                    "reason": "The answer to the math question was provided in the output."
-                }}
-            ]
-        }}
-        ===== END OF EXAMPLES =====
-        
-        ==== START OF INPUT ====
-        instructions: {instructions}
-        output: {actual_output}
-        ==== END OF INPUT ====
-
-        ==== YOUR ANSWER ====
-        JSON:
-        """
-