judgeval 0.0.2__py3-none-any.whl → 0.0.4__py3-none-any.whl

judgeval/scorers/judgeval_scorers/local_implementations/hallucination/hallucination_scorer.py

@@ -0,0 +1,258 @@
+"""
+Metric that evaluates hallucinations in model outputs
+
+The hallucination metric determines whether your LLM generates factually correct information by comparing 
+the actual_output to the provided context.
+
+If you're looking to evaluate hallucination for a RAG system, refer to the faithfulness metric instead.
+
+The HallucinationMetric uses an LLM to determine, for each context in contexts, whether there are any 
+contradictions to the actual_output.
+
+Although extremely similar to the FaithfulnessMetric, the HallucinationMetric is calculated differently 
+since it uses contexts as the source of truth instead. Since contexts is the ideal segment of your 
+knowledge base relevant to a specific input, the degree of hallucination can be measured by the degree 
+of which the contexts is disagreed upon.
+
+Faithfulness is measuring the number of statements in output that agree with contexts.
+Hallucination is measuring the fraction of contexts that agree with output (do not contradict == agree)
+"""
+
+from typing import Optional, Union, List
+
+from judgeval.scorers.utils import (get_or_create_event_loop,
+                                    scorer_progress_meter,
+                                    create_verbose_logs,
+                                    parse_response_json,
+                                    check_example_params,
+                                    )
+from judgeval.scorers import JudgevalScorer
+from judgeval.judges import JudgevalJudge
+from judgeval.judges.utils import create_judge
+from judgeval.data import Example, ExampleParams
+from judgeval.scorers.judgeval_scorers.local_implementations.hallucination.prompts import *
+
+
+required_params = [
+    ExampleParams.INPUT,
+    ExampleParams.ACTUAL_OUTPUT,
+    ExampleParams.CONTEXT,
+]
+
+
+class HallucinationScorer(JudgevalScorer):
+    def __init__(
+        self,
+        threshold: float = 0.5,
+        model: Optional[Union[str, JudgevalJudge]] = None,
+        include_reason: bool = True,
+        async_mode: bool = False,
+        strict_mode: bool = False,
+        verbose_mode: bool = False,
+    ):
+        self.threshold = 1 if strict_mode else threshold
+        self.model, self.using_native_model = create_judge(model)
+        self.evaluation_model = self.model.get_model_name()
+        self.include_reason = include_reason
+        self.async_mode = async_mode
+        self.strict_mode = strict_mode
+        self.verbose_mode = verbose_mode
+
+    def score_example(
+        self,
+        example: Example,
+        _show_indicator: bool = True,
+    ) -> float:
+        check_example_params(example, required_params, self)
+
+        with scorer_progress_meter(self, display_meter=_show_indicator):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                loop.run_until_complete(
+                    self.a_score_example(example, _show_indicator=False)
+                )
+            else:
+                self.verdicts: List[HallucinationVerdict] = (
+                    self._generate_verdicts(
+                        example.actual_output, example.context
+                    )
+                )
+                self.score = self._calculate_score()
+                self.reason = self._generate_reason()
+                self.success = self.score <= self.threshold
+                self.verbose_logs = create_verbose_logs(
+                    self,
+                    steps=[
+                        f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
+                        f"Score: {self.score}\nReason: {self.reason}",
+                    ],
+                )
+
+                return self.score
+
+    async def a_score_example(
+        self,
+        example: Example,
+        _show_indicator: bool = True,
+    ) -> float:
+        check_example_params(example, required_params, self)
+
+        with scorer_progress_meter(
+            self, async_mode=True, display_meter=_show_indicator
+        ):
+            self.verdicts: List[HallucinationVerdict] = (
+                await self._a_generate_verdicts(
+                    example.actual_output, example.context
+                )
+            )
+            self.score = self._calculate_score()
+            self.reason = await self._a_generate_reason()
+            self.success = self.score <= self.threshold
+            self.verbose_logs = create_verbose_logs(
+                self,
+                steps=[
+                    f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
+                    f"Score: {self.score}\nReason: {self.reason}",
+                ],
+            )
+
+            return self.score
+
+    async def _a_generate_reason(self):
+        if self.include_reason is False:
+            return None
+
+        contradictions = []
+        for verdict in self.verdicts:
+            if verdict.verdict.strip().lower() == "no":
+                contradictions.append(verdict.reason)
+
+        prompt: dict = HallucinationTemplate.generate_reason(
+            contradictions=contradictions,
+            score=format(self.score, ".2f"),
+        )
+
+        if self.using_native_model:
+            res = await self.model.a_generate(prompt)
+            data = parse_response_json(res, self)
+            return data["reason"]
+        else:
+            try:
+                res: Reason = await self.model.a_generate(prompt, schema=Reason)
+                return res.reason
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = parse_response_json(res, self)
+                return data["reason"]
+
+    def _generate_reason(self):
+        if self.include_reason is False:
+            return None
+
+        factual_alignments = []
+        contradictions = []
+        for verdict in self.verdicts:
+            if verdict.verdict.strip().lower() == "no":
+                contradictions.append(verdict.reason)
+
+        prompt: dict = HallucinationTemplate.generate_reason(
+            factual_alignments=factual_alignments,
+            contradictions=contradictions,
+            score=format(self.score, ".2f"),
+        )
+
+        if self.using_native_model:
+            res = self.model.generate(prompt)
+            data = parse_response_json(res, self)
+            return data["reason"]
+        else:
+            try:
+                res: Reason = self.model.generate(prompt, schema=Reason)
+                return res.reason
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = parse_response_json(res, self)
+                return data["reason"]
+
+    async def _a_generate_verdicts(
+        self, actual_output: str, contexts: List[str]
+    ) -> List[HallucinationVerdict]:
+        verdicts: List[HallucinationVerdict] = []
+        prompt = HallucinationTemplate.generate_verdicts(
+            actual_output=actual_output, contexts=contexts
+        )
+        if self.using_native_model:
+            res = await self.model.a_generate(prompt)
+            data = parse_response_json(res, self)
+            verdicts = [
+                HallucinationVerdict(**item) for item in data["verdicts"]
+            ]
+            return verdicts
+        else:
+            try:
+                res: Verdicts = await self.model.a_generate(
+                    prompt, schema=Verdicts
+                )
+                verdicts = [item for item in res.verdicts]
+                return verdicts
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = parse_response_json(res, self)
+                verdicts = [
+                    HallucinationVerdict(**item) for item in data["verdicts"]
+                ]
+                return verdicts
+
+    def _generate_verdicts(
+        self, actual_output: str, contexts: List[str]
+    ) -> List[HallucinationVerdict]:
+        verdicts: List[HallucinationVerdict] = []
+        prompt = HallucinationTemplate.generate_verdicts(
+            actual_output=actual_output, contexts=contexts
+        )
+        if self.using_native_model:
+            res = self.model.generate(prompt)
+            data = parse_response_json(res, self)
+            verdicts = [
+                HallucinationVerdict(**item) for item in data["verdicts"]
+            ]
+            return verdicts
+        else:
+            try:
+                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
+                verdicts = [item for item in res.verdicts]
+                return verdicts
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = parse_response_json(res, self)
+                verdicts = [
+                    HallucinationVerdict(**item) for item in data["verdicts"]
+                ]
+                return verdicts
+
+    def _calculate_score(self) -> float:
+        number_of_verdicts = len(self.verdicts)
+        if number_of_verdicts == 0:
+            return 0
+
+        hallucination_count = 0
+        for verdict in self.verdicts:
+            if verdict.verdict.strip().lower() == "no":
+                hallucination_count += 1
+
+        score = hallucination_count / number_of_verdicts
+        return 1 if self.strict_mode and score > self.threshold else score
+
+    def _success_check(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score <= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Hallucination"

judgeval/scorers/judgeval_scorers/local_implementations/hallucination/prompts.py

@@ -0,0 +1,104 @@
+from typing import List, Optional
+from pydantic import BaseModel, Field
+
+
+class HallucinationVerdict(BaseModel):
+    verdict: str
+    reason: str
+
+
+class Verdicts(BaseModel):
+    verdicts: List[HallucinationVerdict]
+
+
+class Reason(BaseModel):
+    reason: str
+
+
+class HallucinationTemplate:
+    @staticmethod
+    def generate_verdicts(actual_output, contexts):
+        return f"""==== TASK INSTRUCTIONS ====
+You will be provided with an `actual output` (the response of an LLM to a particular query) and `contexts` (ground truth contextual information from a knowledge base).
+Your task is to take each context in contexts and determine whether the `actual output` factually agrees with the context.
+
+Additional notes:
+You should NOT use any prior knowledge you have in your decision making process; take each context at face value. 
+Since you will determine a verdict for EACH context, the number of 'verdicts' is EXACTLY EQUAL TO the number of contexts. 
+You should be lenient in your judgment when the actual output lacks detail with respect to the context segment; you should ONLY provide a 'no' answer if the context contradicts the actual output.
+
+==== FORMATTING INSTRUCTIONS ====
+You should return a JSON object with a key 'verdicts', which is a list of JSON objects. Each JSON object corresponds to a context in `contexts`, and should have 2 fields: 'verdict' and 'reason'. 
+The 'verdict' key should be EXACTLY one of 'yes' or 'no', representing whether the `actual output` factually agrees with the context segment. 
+The 'reason' is the justification for the verdict. If your verdict is 'no', try to provide a correction in the reason. 
+
+==== EXAMPLE ====
+Example contexts: ["Einstein won the Nobel Prize for his discovery of the photoelectric effect.", "Einstein won the Nobel Prize in 1968."]
+Example actual output: "Einstein won the Nobel Prize in 1969 for his discovery of the photoelectric effect."
+
+Example:
+{{
+    "verdicts": [
+        {{
+            "verdict": "yes",
+            "reason": "The actual output agrees with the provided context which states that Einstein won the Nobel Prize for his discovery of the photoelectric effect."
+        }},
+        {{
+            "verdict": "no",
+            "reason": "The actual output contradicts the provided context which states that Einstein won the Nobel Prize in 1968, not 1969."
+        }}
+    ]  
+}}
+
+==== YOUR TURN ====
+Contexts:
+{contexts}
+
+Actual Output:
+{actual_output}
+
+JSON:
+"""
+
+    @staticmethod
+    def generate_reason(contradictions, score):
+        return f"""==== TASK INSTRUCTIONS ====
+An LLM has been provided with a list of `contexts` (ground truth contextual information from a knowledge base) and `actual output` (the response of an LLM to a particular query). 
+You will be provided with a list of `contradictions`, which are factual discrepancies between the context segments and the actual output. 
+Additionally, you will be provided with a hallucination score, which is a float (0 - 1, where 0 is the best score) indicating the fraction of context segments that contradict the actual output.
+
+Your task is to provide a CLEAR and CONCISE reason for the hallucination score. 
+If the hallucination score is 0 (no contradictions), you should instead respond with a positive remark with an upbeat encouraging tone (but don't overblow the kind attitude).
+        
+==== FORMATTING INSTRUCTIONS ====
+Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+Example JSON:
+{{
+    "reason": "The score is <hallucination_score> because <your_reason>."
+}}
+
+==== EXAMPLE ====
+Example Contradictions:
+[
+    "The actual output claims Einstein won the Nobel Prize in 1969, which contradicts the context stating he won it in 1968.",
+    "The actual output states Einstein was a chemist, but the context indicates he was a physicist.",
+    "The actual output claims Einstein was born in Switzerland, while the context states he was born in Germany."
+]
+
+Example Hallucination Score: 
+0.75
+
+Example Response:
+{{
+    "reason": "The score is 0.75 because the actual output made multiple factual errors: incorrectly stating Einstein's Nobel Prize year (1969 vs 1968), his profession (chemist vs physicist), and birthplace (Switzerland vs Germany)."
+}}
+
+==== YOUR TURN ====
+Contradictions:
+{contradictions}
+
+Hallucination Score:
+{score}
+
+JSON:
+"""

judgeval/scorers/judgeval_scorers/local_implementations/json_correctness/json_correctness_scorer.py

@@ -0,0 +1,127 @@
+from typing import List, Optional, Union, Any
+from pydantic import BaseModel, ValidationError, create_model
+
+from judgeval.judges import JudgevalJudge
+from judgeval.scorers.utils import (get_or_create_event_loop,
+                                    scorer_progress_meter,
+                                    create_verbose_logs,
+                                    parse_response_json,
+                                    check_example_params
+                                    )
+from judgeval.scorers import JudgevalScorer
+from judgeval.data import Example, ExampleParams
+
+
+required_params = [
+    ExampleParams.INPUT,
+    ExampleParams.ACTUAL_OUTPUT
+]
+
+
+class JsonCorrectnessScorer(JudgevalScorer):
+
+    def __init__(
+        self,
+        json_schema: Union[BaseModel, dict],
+        model: Optional[Union[str, JudgevalJudge]] = None,
+        threshold: float = 0.5,
+        async_mode: bool = True,
+        strict_mode: bool = False,
+        verbose_mode: bool = False,
+        user: Optional[str] = None
+    ):
+        self.score_type = "json_correctness"
+        self.model = model
+        self.threshold = threshold
+        self.async_mode = async_mode
+        self.strict_mode = strict_mode
+        self.verbose_mode = verbose_mode
+        self.user = user
+
+        if isinstance(json_schema, dict):
+            # Convert to BaseModel
+            fields = {
+                key: (str if prop["type"] == "string" else Any, ...)
+                for key, prop in json_schema["properties"].items()
+            }
+
+            # Dynamically create the model
+            DynamicModel = create_model(json_schema["title"], **fields)
+
+            self.json_schema = DynamicModel
+        else:
+            self.json_schema = json_schema
+
+    def score_example(self, example: Example, _show_indicator: bool = True) -> float:
+        check_example_params(example, required_params, self)
+        with scorer_progress_meter(
+            self,
+            async_mode=self.async_mode,
+            display_meter=_show_indicator,
+        ):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                loop.run_until_complete(
+                    self.a_measure(example, _show_indicator=False)
+                )
+            else:
+                valid_json = True
+                try:
+                    self.json_schema.model_validate_json(
+                        example.actual_output
+                    )
+                except ValidationError as e:
+                    valid_json = False
+
+                self.score = 1.0 if valid_json else 0
+                self.success = self.score >= self.threshold
+                self.verbose_logs = create_verbose_logs(
+                    self,
+                    steps=[
+                        f"LLM outputed Json:\n{example.actual_output}",
+                        f"Score: {self.score}",
+                    ],
+                )
+
+                return self.score
+
+    async def a_score_example(self, example: Example, _show_indicator: bool = True) -> float:
+        check_example_params(example, required_params, self)
+        with scorer_progress_meter(
+            self,
+            async_mode=self.async_mode,
+            display_meter=_show_indicator,
+        ):
+            valid_json = True
+            try:
+                self.json_schema.model_validate_json(
+                    example.actual_output
+                )
+            except ValidationError as e:
+                valid_json = False
+
+            self.score = 1.0 if valid_json else 0
+            self.success = self.score >= self.threshold
+            self.verbose_logs = create_verbose_logs(
+                self,
+                steps=[
+                    f"LLM outputed Json:\n{example.actual_output}",
+                    f"Score: {self.score}",
+                ],
+            )
+            return self.score
+
+    def _success_check(self):
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "JSON Correctness"
+    

judgeval/scorers/judgeval_scorers/local_implementations/summarization/__init__.py

@@ -0,0 +1,3 @@
+from .summarization_scorer import SummarizationScorer
+
+__all__ = ["SummarizationScorer"]