judgeval 0.0.11__py3-none-any.whl → 0.22.2__py3-none-any.whl

judgeval/scorers/judgeval_scorers/local_implementations/hallucination/hallucination_scorer.py

@@ -1,263 +0,0 @@
-"""
-Metric that evaluates hallucinations in model outputs
-
-The hallucination metric determines whether your LLM generates factually correct information by comparing 
-the actual_output to the provided context.
-
-If you're looking to evaluate hallucination for a RAG system, refer to the faithfulness metric instead.
-
-The HallucinationMetric uses an LLM to determine, for each context in contexts, whether there are any 
-contradictions to the actual_output.
-
-Although extremely similar to the FaithfulnessMetric, the HallucinationMetric is calculated differently 
-since it uses contexts as the source of truth instead. Since contexts is the ideal segment of your 
-knowledge base relevant to a specific input, the degree of hallucination can be measured by the degree 
-of which the contexts is disagreed upon.
-
-Faithfulness is measuring the number of statements in output that agree with contexts.
-Hallucination is measuring the fraction of contexts that agree with output (do not contradict == agree)
-"""
-
-from typing import Optional, Union, List
-
-from judgeval.constants import APIScorer
-from judgeval.scorers.utils import (get_or_create_event_loop,
-                                    scorer_progress_meter,
-                                    create_verbose_logs,
-                                    parse_response_json,
-                                    check_example_params,
-                                    )
-from judgeval.scorers import JudgevalScorer
-from judgeval.judges import JudgevalJudge
-from judgeval.judges.utils import create_judge
-from judgeval.data import Example, ExampleParams
-from judgeval.scorers.judgeval_scorers.local_implementations.hallucination.prompts import *
-
-
-required_params = [
-    ExampleParams.INPUT,
-    ExampleParams.ACTUAL_OUTPUT,
-    ExampleParams.CONTEXT,
-]
-
-
-class HallucinationScorer(JudgevalScorer):
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        model: Optional[Union[str, JudgevalJudge]] = None,
-        include_reason: bool = True,
-        async_mode: bool = False,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-    ):
-        super().__init__(
-            score_type=APIScorer.HALLUCINATION,
-            threshold=1 if strict_mode else threshold,
-            evaluation_model=None,
-            include_reason=include_reason,
-            async_mode=async_mode,
-            strict_mode=strict_mode,
-            verbose_mode=verbose_mode
-        )
-        self.model, self.using_native_model = create_judge(model)
-        self.evaluation_model = self.model.get_model_name()
-
-    def score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(self, display_meter=_show_indicator):
-            if self.async_mode:
-                loop = get_or_create_event_loop()
-                loop.run_until_complete(
-                    self.a_score_example(example, _show_indicator=False)
-                )
-            else:
-                self.verdicts: List[HallucinationVerdict] = (
-                    self._generate_verdicts(
-                        example.actual_output, example.context
-                    )
-                )
-                self.score = self._calculate_score()
-                self.reason = self._generate_reason()
-                self.success = self.score <= self.threshold
-                self.verbose_logs = create_verbose_logs(
-                    self,
-                    steps=[
-                        f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
-                        f"Score: {self.score}\nReason: {self.reason}",
-                    ],
-                )
-
-                return self.score
-
-    async def a_score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(
-            self, async_mode=True, display_meter=_show_indicator
-        ):
-            self.verdicts: List[HallucinationVerdict] = (
-                await self._a_generate_verdicts(
-                    example.actual_output, example.context
-                )
-            )
-            self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason()
-            self.success = self.score <= self.threshold
-            self.verbose_logs = create_verbose_logs(
-                self,
-                steps=[
-                    f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
-                    f"Score: {self.score}\nReason: {self.reason}",
-                ],
-            )
-
-            return self.score
-
-    async def _a_generate_reason(self):
-        if self.include_reason is False:
-            return None
-
-        contradictions = []
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() == "no":
-                contradictions.append(verdict.reason)
-
-        prompt: dict = HallucinationTemplate.generate_reason(
-            contradictions=contradictions,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = await self.model.a_generate(prompt, schema=Reason)
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    def _generate_reason(self):
-        if self.include_reason is False:
-            return None
-
-        factual_alignments = []
-        contradictions = []
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() == "no":
-                contradictions.append(verdict.reason)
-
-        prompt: dict = HallucinationTemplate.generate_reason(
-            factual_alignments=factual_alignments,
-            contradictions=contradictions,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = self.model.generate(prompt, schema=Reason)
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    async def _a_generate_verdicts(
-        self, actual_output: str, contexts: List[str]
-    ) -> List[HallucinationVerdict]:
-        verdicts: List[HallucinationVerdict] = []
-        prompt = HallucinationTemplate.generate_verdicts(
-            actual_output=actual_output, contexts=contexts
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            verdicts = [
-                HallucinationVerdict(**item) for item in data["verdicts"]
-            ]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                verdicts = [
-                    HallucinationVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def _generate_verdicts(
-        self, actual_output: str, contexts: List[str]
-    ) -> List[HallucinationVerdict]:
-        verdicts: List[HallucinationVerdict] = []
-        prompt = HallucinationTemplate.generate_verdicts(
-            actual_output=actual_output, contexts=contexts
-        )
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            verdicts = [
-                HallucinationVerdict(**item) for item in data["verdicts"]
-            ]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                verdicts = [
-                    HallucinationVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def _calculate_score(self) -> float:
-        number_of_verdicts = len(self.verdicts)
-        if number_of_verdicts == 0:
-            return 0
-
-        hallucination_count = 0
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() == "no":
-                hallucination_count += 1
-
-        score = hallucination_count / number_of_verdicts
-        return 1 if self.strict_mode and score > self.threshold else score
-
-    def _success_check(self) -> bool:
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score <= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Hallucination"

judgeval/scorers/judgeval_scorers/local_implementations/hallucination/prompts.py

@@ -1,104 +0,0 @@
-from typing import List, Optional
-from pydantic import BaseModel, Field
-
-
-class HallucinationVerdict(BaseModel):
-    verdict: str
-    reason: str
-
-
-class Verdicts(BaseModel):
-    verdicts: List[HallucinationVerdict]
-
-
-class Reason(BaseModel):
-    reason: str
-
-
-class HallucinationTemplate:
-    @staticmethod
-    def generate_verdicts(actual_output, contexts):
-        return f"""==== TASK INSTRUCTIONS ====
-You will be provided with an `actual output` (the response of an LLM to a particular query) and `contexts` (ground truth contextual information from a knowledge base).
-Your task is to take each context in contexts and determine whether the `actual output` factually agrees with the context.
-
-Additional notes:
-You should NOT use any prior knowledge you have in your decision making process; take each context at face value. 
-Since you will determine a verdict for EACH context, the number of 'verdicts' is EXACTLY EQUAL TO the number of contexts. 
-You should be lenient in your judgment when the actual output lacks detail with respect to the context segment; you should ONLY provide a 'no' answer if the context contradicts the actual output.
-
-==== FORMATTING INSTRUCTIONS ====
-You should return a JSON object with a key 'verdicts', which is a list of JSON objects. Each JSON object corresponds to a context in `contexts`, and should have 2 fields: 'verdict' and 'reason'. 
-The 'verdict' key should be EXACTLY one of 'yes' or 'no', representing whether the `actual output` factually agrees with the context segment. 
-The 'reason' is the justification for the verdict. If your verdict is 'no', try to provide a correction in the reason. 
-
-==== EXAMPLE ====
-Example contexts: ["Einstein won the Nobel Prize for his discovery of the photoelectric effect.", "Einstein won the Nobel Prize in 1968."]
-Example actual output: "Einstein won the Nobel Prize in 1969 for his discovery of the photoelectric effect."
-
-Example:
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes",
-            "reason": "The actual output agrees with the provided context which states that Einstein won the Nobel Prize for his discovery of the photoelectric effect."
-        }},
-        {{
-            "verdict": "no",
-            "reason": "The actual output contradicts the provided context which states that Einstein won the Nobel Prize in 1968, not 1969."
-        }}
-    ]  
-}}
-
-==== YOUR TURN ====
-Contexts:
-{contexts}
-
-Actual Output:
-{actual_output}
-
-JSON:
-"""
-
-    @staticmethod
-    def generate_reason(contradictions, score):
-        return f"""==== TASK INSTRUCTIONS ====
-An LLM has been provided with a list of `contexts` (ground truth contextual information from a knowledge base) and `actual output` (the response of an LLM to a particular query). 
-You will be provided with a list of `contradictions`, which are factual discrepancies between the context segments and the actual output. 
-Additionally, you will be provided with a hallucination score, which is a float (0 - 1, where 0 is the best score) indicating the fraction of context segments that contradict the actual output.
-
-Your task is to provide a CLEAR and CONCISE reason for the hallucination score. 
-If the hallucination score is 0 (no contradictions), you should instead respond with a positive remark with an upbeat encouraging tone (but don't overblow the kind attitude).
-        
-==== FORMATTING INSTRUCTIONS ====
-Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-Example JSON:
-{{
-    "reason": "The score is <hallucination_score> because <your_reason>."
-}}
-
-==== EXAMPLE ====
-Example Contradictions:
-[
-    "The actual output claims Einstein won the Nobel Prize in 1969, which contradicts the context stating he won it in 1968.",
-    "The actual output states Einstein was a chemist, but the context indicates he was a physicist.",
-    "The actual output claims Einstein was born in Switzerland, while the context states he was born in Germany."
-]
-
-Example Hallucination Score: 
-0.75
-
-Example Response:
-{{
-    "reason": "The score is 0.75 because the actual output made multiple factual errors: incorrectly stating Einstein's Nobel Prize year (1969 vs 1968), his profession (chemist vs physicist), and birthplace (Switzerland vs Germany)."
-}}
-
-==== YOUR TURN ====
-Contradictions:
-{contradictions}
-
-Hallucination Score:
-{score}
-
-JSON:
-"""

judgeval/scorers/judgeval_scorers/local_implementations/json_correctness/__init__.py

@@ -1,5 +0,0 @@
-from judgeval.scorers.judgeval_scorers.local_implementations.json_correctness.json_correctness_scorer import JsonCorrectnessScorer
-
-__all__ = [
-    "JsonCorrectnessScorer",
-]

judgeval/scorers/judgeval_scorers/local_implementations/json_correctness/json_correctness_scorer.py

@@ -1,134 +0,0 @@
-from typing import List, Optional, Union, Any
-from pydantic import BaseModel, ValidationError, create_model
-
-from judgeval.constants import APIScorer
-from judgeval.judges import JudgevalJudge
-from judgeval.judges.utils import create_judge
-from judgeval.scorers.utils import (get_or_create_event_loop,
-                                    scorer_progress_meter,
-                                    create_verbose_logs,
-                                    parse_response_json,
-                                    check_example_params
-                                    )
-from judgeval.scorers import JudgevalScorer
-from judgeval.data import Example, ExampleParams
-
-
-required_params = [
-    ExampleParams.INPUT,
-    ExampleParams.ACTUAL_OUTPUT
-]
-
-
-class JsonCorrectnessScorer(JudgevalScorer):
-
-    def __init__(
-        self,
-        json_schema: Union[BaseModel, dict],
-        model: Optional[Union[str, JudgevalJudge]] = None,
-        threshold: float = 0.5,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-        user: Optional[str] = None
-    ):
-        super().__init__(
-            score_type=APIScorer.JSON_CORRECTNESS,
-            threshold=1 if strict_mode else threshold,
-            evaluation_model=None,
-            include_reason=False,
-            async_mode=async_mode,
-            strict_mode=strict_mode,
-            verbose_mode=verbose_mode
-        )
-        self.user = user
-        self.model, self.using_native_model = create_judge(model)
-        self.evaluation_model = self.model.get_model_name()
-
-        if isinstance(json_schema, dict):
-            # Convert to BaseModel
-            fields = {
-                key: (str if prop["type"] == "string" else Any, ...)
-                for key, prop in json_schema["properties"].items()
-            }
-
-            # Dynamically create the model
-            DynamicModel = create_model(json_schema["title"], **fields)
-
-            self.json_schema = DynamicModel
-        else:
-            self.json_schema = json_schema
-
-    def score_example(self, example: Example, _show_indicator: bool = True) -> float:
-        check_example_params(example, required_params, self)
-        with scorer_progress_meter(
-            self,
-            async_mode=self.async_mode,
-            display_meter=_show_indicator,
-        ):
-            if self.async_mode:
-                loop = get_or_create_event_loop()
-                loop.run_until_complete(
-                    self.a_measure(example, _show_indicator=False)
-                )
-            else:
-                valid_json = True
-                try:
-                    self.json_schema.model_validate_json(
-                        example.actual_output
-                    )
-                except ValidationError as e:
-                    valid_json = False
-
-                self.score = 1.0 if valid_json else 0
-                self.success = self.score >= self.threshold
-                self.verbose_logs = create_verbose_logs(
-                    self,
-                    steps=[
-                        f"LLM outputed Json:\n{example.actual_output}",
-                        f"Score: {self.score}",
-                    ],
-                )
-
-                return self.score
-
-    async def a_score_example(self, example: Example, _show_indicator: bool = True) -> float:
-        check_example_params(example, required_params, self)
-        with scorer_progress_meter(
-            self,
-            async_mode=self.async_mode,
-            display_meter=_show_indicator,
-        ):
-            valid_json = True
-            try:
-                self.json_schema.model_validate_json(
-                    example.actual_output
-                )
-            except ValidationError as e:
-                valid_json = False
-
-            self.score = 1.0 if valid_json else 0
-            self.success = self.score >= self.threshold
-            self.verbose_logs = create_verbose_logs(
-                self,
-                steps=[
-                    f"LLM outputed Json:\n{example.actual_output}",
-                    f"Score: {self.score}",
-                ],
-            )
-            return self.score
-
-    def _success_check(self):
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score >= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "JSON Correctness"
-    

judgeval/scorers/judgeval_scorers/local_implementations/summarization/__init__.py

@@ -1,3 +0,0 @@
-from .summarization_scorer import SummarizationScorer
-
-__all__ = ["SummarizationScorer"]