judgeval 0.0.3__py3-none-any.whl → 0.0.4__py3-none-any.whl

judgeval/scorers/judgeval_scorers/local_implementations/faithfulness/faithfulness_scorer.py

@@ -0,0 +1,318 @@
+"""
+Code for the local implementation of the Faithfulness metric.
+"""
+from typing import List, Optional, Union
+
+
+from judgeval.data import (
+    Example, 
+    ExampleParams
+)
+from judgeval.scorers import JudgevalScorer
+from judgeval.scorers.utils import (
+    get_or_create_event_loop, 
+    check_example_params
+)
+from judgeval.judges.utils import create_judge
+from judgeval.judges import JudgevalJudge
+from judgeval.scorers.utils import (
+    scorer_progress_meter, 
+    create_verbose_logs, 
+    parse_response_json
+)
+from judgeval.scorers.judgeval_scorers.local_implementations.faithfulness.prompts import (
+    FaithfulnessTemplate,
+    FaithfulnessVerdict,
+    Verdicts,
+    Reason,
+    Claims,
+)
+
+
+required_params = [
+    ExampleParams.INPUT,
+    ExampleParams.ACTUAL_OUTPUT,
+    ExampleParams.RETRIEVAL_CONTEXT,
+]
+
+
+class FaithfulnessScorer(JudgevalScorer):
+    def __init__(
+        self,
+        threshold: float = 0.5,
+        model: Optional[Union[str, JudgevalJudge]] = None,
+        include_reason: bool = True,
+        async_mode: bool = True,
+        strict_mode: bool = False,
+        verbose_mode: bool = False,
+        user: Optional[str] = None
+    ):
+        self.threshold = 1 if strict_mode else threshold
+        self.model, self.using_native_model = create_judge(model)
+        self.using_native_model = True  # NOTE: SETTING THIS FOR LITELLM and TOGETHER usage
+        self.evaluation_model = self.model.get_model_name()
+        self.include_reason = include_reason
+        self.async_mode = async_mode
+        self.strict_mode = strict_mode
+        self.verbose_mode = verbose_mode
+
+    def score_example(
+        self,
+        example: Example,
+        all_claims: bool = False,
+        _show_indicator: bool = True,
+    ) -> float:
+        check_example_params(example, required_params, self)
+        
+        with scorer_progress_meter(self, display_meter=_show_indicator):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                loop.run_until_complete(
+                    self.a_score_example(
+                        example, 
+                        all_claims=all_claims, 
+                        _show_indicator=False
+                    )
+                )
+            else:
+                self.claims = self._generate_claims(example.actual_output, all_claims=all_claims)
+                if self.additional_metadata is None:
+                    self.additional_metadata = {}
+                self.additional_metadata["claims"] = self.claims  # Add claims generated to metadata
+
+                self.verdicts = self._generate_verdicts(example.retrieval_context)
+                self.additional_metadata["verdicts"] = [v.model_dump() for v in self.verdicts]  # Add verdicts generated to metadata
+                
+                self.score = self._calculate_score()
+                self.reason = self._generate_reason()
+                self.success = self.score >= self.threshold
+                self.verbose_logs = create_verbose_logs(
+                    self,
+                    steps=[
+                        f"Claims:\n{self.claims}",
+                        f"Verdicts:\n{self.verdicts}",
+                        f"Score: {self.score}\nReason: {self.reason}",
+                    ],
+                )
+
+                return self.score
+
+    async def a_score_example(
+        self,
+        example: Example,
+        _show_indicator: bool = True
+    ) -> float:
+        check_example_params(example, required_params, self)
+
+        with scorer_progress_meter(
+            self, async_mode=True, display_meter=_show_indicator
+        ):
+            self.claims = await self._a_generate_claims(example.actual_output)
+
+            if self.additional_metadata is None:
+                self.additional_metadata = {}
+            self.additional_metadata["claims"] = self.claims
+
+            self.verdicts = await self._a_generate_verdicts(example.retrieval_context)  
+            self.additional_metadata["verdicts"] = [v.model_dump() for v in self.verdicts]  # Add verdicts generated to metadata
+
+            self.score = self._calculate_score()
+            self.reason = await self._a_generate_reason()
+            self.success = self.score >= self.threshold
+            self.verbose_logs = create_verbose_logs(
+                self,
+                steps=[
+                    f"Claims:\n{self.claims}",
+                    f"Verdicts:\n{self.verdicts}",
+                    f"Score: {self.score}\nReason: {self.reason}",
+                ],
+            )
+
+            return self.score
+
+    async def _a_generate_reason(self) -> str:
+        if self.include_reason is False:
+            return None
+
+        contradictions = []
+        for verdict in self.verdicts:
+            if verdict.verdict.strip().lower() == "no":
+                contradictions.append(verdict.model_dump())
+
+        prompt: dict = FaithfulnessTemplate.justify_reason(
+            contradictions=contradictions,
+            score=format(self.score, ".2f"),
+        )
+        if self.using_native_model:
+            res = await self.model.a_generate(prompt)
+            data = parse_response_json(res, self)
+            return data["reason"]
+        else:
+            try:
+                res: Reason = await self.model.a_generate(prompt, schema=Reason)
+                return res.reason
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = parse_response_json(res, self)
+                return data["reason"]
+
+    def _generate_reason(self) -> str:
+        if self.include_reason is False:
+            return None
+
+        contradictions = []
+        for verdict in self.verdicts:
+            if verdict.verdict.strip().lower() == "no":
+                contradictions.append(verdict.reason)
+
+        prompt: dict = FaithfulnessTemplate.justify_reason(
+            contradictions=contradictions,
+            score=format(self.score, ".2f"),
+        )
+
+        if self.using_native_model:
+            res = self.model.generate(prompt)
+            data = parse_response_json(res, self)
+            return data["reason"]
+        else:
+            try:
+                res: Reason = self.model.generate(prompt, schema=Reason)
+                return res.reason
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = parse_response_json(res, self)
+                return data["reason"]
+
+    async def _a_generate_verdicts(self, retrieval_context: str) -> List[FaithfulnessVerdict]:
+        if len(self.claims) == 0:
+            return []
+
+        verdicts: List[FaithfulnessVerdict] = []
+
+        claims = [
+            claim["claim"] for claim in self.claims
+        ]  # We only need the claims, not the quotes involved
+
+        prompt = FaithfulnessTemplate.create_verdicts(
+            claims=claims,
+            retrieval_context=retrieval_context,
+        )
+        if self.using_native_model:
+            res = await self.model.a_generate(prompt)
+            data = parse_response_json(res, self)
+            verdicts = [
+                FaithfulnessVerdict(**item) for item in data["verdicts"]
+            ]
+            return verdicts
+        else:
+            try:
+                res: Verdicts = await self.model.generate(
+                    prompt, schema=Verdicts
+                )
+                verdicts = [item for item in res.verdicts]
+                return verdicts
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = parse_response_json(res, self)
+                verdicts = [
+                    FaithfulnessVerdict(**item) for item in data["verdicts"]
+                ]
+                return verdicts
+
+    def _generate_verdicts(self, retrieval_context: str) -> List[FaithfulnessVerdict]:
+        if len(self.claims) == 0:
+            return []
+
+        verdicts: List[FaithfulnessVerdict] = []
+
+        claims = [
+            claim["claim"] for claim in self.claims
+        ]  # We only need the claims, not the quotes involved
+
+        prompt = FaithfulnessTemplate.create_verdicts(
+            claims=claims,
+            retrieval_context=retrieval_context,
+        )
+        if self.using_native_model:
+            res = self.model.generate(prompt)
+            data = parse_response_json(res, self)
+            verdicts = [
+                FaithfulnessVerdict(**item) for item in data["verdicts"]
+            ]
+            return verdicts
+        else:
+            try:
+                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
+                verdicts = [item for item in res.verdicts]
+                return verdicts
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = parse_response_json(res, self)
+                verdicts = [
+                    FaithfulnessVerdict(**item) for item in data["verdicts"]
+                ]
+                return verdicts
+
+    async def _a_generate_claims(self, actual_output: str) -> List[str]:
+        prompt = FaithfulnessTemplate.find_claims(text=actual_output)
+        if self.using_native_model:
+            res = await self.model.a_generate(prompt)
+            data = parse_response_json(res, self)
+            return data["claims"]
+        else:
+            try:
+                res: Claims = await self.model.a_generate(prompt, schema=Claims)
+                return res.claims
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = parse_response_json(res, self)
+                return data["claims"]
+
+    def _generate_claims(self, actual_output: str, all_claims: bool = False) -> List[str]:
+        prompt = FaithfulnessTemplate.find_claims(text=actual_output)
+        if self.using_native_model:
+            res = self.model.generate(prompt)
+            data = parse_response_json(res, self)
+            return data["claims"]
+        else:
+            try:
+                res: Claims = self.model.generate(prompt, schema=Claims)
+                return res.claims
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = parse_response_json(res, self)
+                return data["claims"]
+
+    def _calculate_score(self) -> float:
+        number_of_verdicts = len(self.verdicts)
+        if number_of_verdicts == 0:
+            return 1
+
+        faithfulness_count = 0
+        for verdict in self.verdicts:
+            if verdict.verdict.strip().lower() != "no":
+                faithfulness_count += 1
+
+        score = faithfulness_count / number_of_verdicts
+        return 0 if self.strict_mode and score < self.threshold else score
+
+    def _success_check(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    def get_claims(self):
+        return self.claims
+    
+    def get_verdicts(self):
+        return self.verdicts
+
+    @property
+    def __name__(self):
+        return "Faithfulness"
+    

judgeval/scorers/judgeval_scorers/local_implementations/faithfulness/prompts.py

@@ -0,0 +1,265 @@
+from typing import List, Optional
+from pydantic import BaseModel, Field
+
+
+class FaithfulnessVerdict(BaseModel):
+    verdict: str
+    reason: Optional[str] = Field(default=None)
+
+
+class Verdicts(BaseModel):
+    verdicts: List[FaithfulnessVerdict]
+
+
+class Truths(BaseModel):
+    truths: List[str]
+
+
+class Claims(BaseModel):
+    claims: List[str]
+
+
+class Reason(BaseModel):
+    reason: str
+
+
+class FaithfulnessTemplate:
+    @staticmethod
+    def find_claims(text):
+        return f"""==== TASK INSTRUCTIONS ====
+You will be provided with a passage of text. Based on the text, your task is to generate a comprehensive list of ALL CLAIMS that can be inferred from the text.  
+For every claim that you derive from the text, provide the source of the claim via quoting the original text. Please try to extract EVERY CLAIM that is in the original text; priortize generating the most claims rather than being concise. 
+You should NOT include any prior knowledge, and take the text at face value when extracting claims.
+
+==== FORMATTING YOUR ANSWER ====
+Please return your answer in JSON format, with the "claims" key as a list of JSON objects with keys "claim" and "quote". No words or explanation beyond the output JSON is needed.
+
+==== EXAMPLES ====
+
+---- START OF EXAMPLE 1 ----
+Example Text: 
+"Einstein won the nobel prize in 1968 for his discovery of the photoelectric effect."
+
+Example JSON: 
+{{
+    "claims": [
+        {{
+             "claim": "Einstein won the nobel prize for his discovery of the photoelectric effect.",
+             "quote": "Einstein won the nobel prize in 1968 for his discovery of the photoelectric effect."
+        }},
+        {{
+             "claim": "Einstein won the nobel prize in 1968.",
+             "quote": "Einstein won the nobel prize in 1968 for his discovery of the photoelectric effect."
+        }}
+    ]  
+}}
+---- END OF EXAMPLE 1 ----
+
+---- START OF EXAMPLE 2 ----
+Example Text: "The Wright brothers successfully flew the first powered airplane on December 17, 1903, in Kitty Hawk, North Carolina."
+
+{{
+    "claims": [
+        {{
+            "claim": "The Wright brothers flew the first powered airplane.",
+            "quote": "The Wright brothers successfully flew the first powered airplane on December 17, 1903, in Kitty Hawk, North Carolina."
+        }},
+        {{
+            "claim": "The Wright brothers made their flight in Kitty Hawk, North Carolina.",
+            "quote": "The Wright brothers successfully flew the first powered airplane on December 17, 1903, in Kitty Hawk, North Carolina."
+        }},
+        {{
+            "claim": "The first powered airplane flight occurred on December 17, 1903.",
+            "quote": "The Wright brothers successfully flew the first powered airplane on December 17, 1903, in Kitty Hawk, North Carolina."
+        }}
+    ]
+}}
+---- END OF EXAMPLE 2 ----
+
+---- START OF EXAMPLE 3 ----
+Example Text:
+"The Great Wall of China was built over many centuries by different Chinese dynasties. Construction began more than 2,000 years ago during the Warring States period. The most famous sections were built during the Ming Dynasty. The wall stretches for thousands of miles across northern China and was primarily built for military defense."
+
+Example JSON:
+{{
+    "claims": [
+        {{
+            "claim": "The Great Wall of China was built by multiple Chinese dynasties",
+            "quote": "The Great Wall of China was built over many centuries by different Chinese dynasties."
+        }},
+        {{
+            "claim": "Construction of the Great Wall began over 2,000 years ago",
+            "quote": "Construction began more than 2,000 years ago during the Warring States period."
+        }},
+        {{
+            "claim": "Construction started during the Warring States period",
+            "quote": "Construction began more than 2,000 years ago during the Warring States period."
+        }},
+        {{
+            "claim": "The most well-known parts of the wall were constructed during the Ming Dynasty",
+            "quote": "The most famous sections were built during the Ming Dynasty."
+        }},
+        {{
+            "claim": "The Great Wall extends for thousands of miles",
+            "quote": "The wall stretches for thousands of miles across northern China"
+        }},
+        {{
+            "claim": "The wall is located in northern China",
+            "quote": "The wall stretches for thousands of miles across northern China"
+        }},
+        {{
+            "claim": "The Great Wall was constructed for defensive military purposes",
+            "quote": "was primarily built for military defense."
+        }}
+    ]
+}}
+---- END OF EXAMPLE 3 ----
+
+==== END OF EXAMPLES ====
+
+==== YOUR TURN ====
+
+Example Text:
+{text}
+
+JSON:
+"""
+
+    @staticmethod
+    def create_verdicts(claims, retrieval_context):
+        return f"""==== TASK INSTRUCTIONS ====
+You will be provided with a list of claims from an LLM's output text, accompanied by the retrieval documents that the LLM used to generate the output. 
+Your task is to determine whether EACH claim is factually consistent with the retrieval context ("yes", "no", or "idk"). 
+ONLY choose 'no' if the retrieval context DIRECTLY CONTRADICTS the claims. YOU SHOULD NEVER USE YOUR PRIOR KNOWLEDGE IN YOUR JUDGMENT.
+Claims made using vague, suggestive, or speculative language such as 'may have', 'possibility due to', do NOT count as a contradiction.
+Claims that are not justified by the retrieval context due to a lack of information MUST BE ANSWERED with 'idk'.
+
+==== FORMATTING YOUR ANSWER ====
+Please return your answer in JSON format, with the 'verdicts' key as a list of JSON objects. Each JSON object should have 2 fields: 'verdict' and 'reason'. 
+The 'verdict' key should be either 'yes', 'no', or 'idk', which states WHETHER THE CLAIM AGREES with the context. 
+The 'reason' key should be a string explaining why the claim is 'yes', 'no', or 'idk'. Make specific reference to the retrieval context to justify your verdict.
+
+==== EXAMPLES ====
+---- START OF EXAMPLE 1 ----
+Example retrieval contexts: "Einstein won the Nobel Prize for his discovery of the photoelectric effect. Einstein won the Nobel Prize in 1968. Einstein is a German Scientist."
+Example claims: ["Barack Obama is a caucasian male.", "Zurich is a city in London", "Einstein won the Nobel Prize for the discovery of the photoelectric effect which may have contributed to his fame.", "Einstein won the Nobel Prize in 1969 for his discovery of the photoelectric effect.", "Einstein was a Germen chef."]
+
+Example JSON:
+{{
+    "verdicts": [
+        {{
+            "verdict": "idk",
+            "reason": "The claim about Barack Obama's ethnicity cannot be verified from the given retrieval context as it contains no information about Barack Obama."
+        }},
+        {{
+            "verdict": "idk", 
+            "reason": "The claim about Zurich being a city in London cannot be verified from the given retrieval context as it contains no information about Zurich or London."
+        }},
+        {{
+            "verdict": "yes",
+            "reason": "The retrieval context confirms that Einstein won the Nobel Prize for discovering the photoelectric effect."
+        }},
+        {{
+            "verdict": "no",
+            "reason": "The actual output claims Einstein won the Nobel Prize in 1969, which is untrue as the retrieval context states it is 1968 instead."
+        }},
+        {{
+            "verdict": "no",
+            "reason": "The actual output claims Einstein is a Germen chef, which is not correct as the retrieval context states he was a German scientist instead."
+        }}
+    ]  
+}}
+---- END OF EXAMPLE 1 ----
+---- START OF EXAMPLE 2 ----
+Example retrieval contexts: "The Great Wall of China was built over many centuries by different Chinese dynasties. Construction began more than 2,000 years ago. The wall stretches for thousands of miles across China's northern borders. Most of the existing wall was built during the Ming Dynasty."
+Example claims: ["The Great Wall was built in a single year.", "The Great Wall may have taken centuries to complete.", "The Great Wall was built by the Romans.", "The Great Wall is located in China's northern region.", "The Great Wall is 100 meters long."]
+
+Example JSON:
+{{
+    "verdicts": [
+        {{
+            "verdict": "no",
+            "reason": "The claim that the Great Wall was built in a single year directly contradicts the retrieval context, which states it was built over many centuries."
+        }},
+        {{
+            "verdict": "yes",
+            "reason": "The retrieval context confirms that the Great Wall was built over many centuries by different Chinese dynasties."
+        }},
+        {{
+            "verdict": "no",
+            "reason": "The claim states the Romans built the wall, which contradicts the retrieval context that specifies it was built by Chinese dynasties."
+        }},
+        {{
+            "verdict": "yes",
+            "reason": "The retrieval context explicitly states that the wall stretches across China's northern borders."
+        }},
+        {{
+            "verdict": "no",
+            "reason": "The claim that the wall is 100 meters long contradicts the retrieval context which states it stretches for thousands of miles."
+        }}
+    ]
+}}
+---- END OF EXAMPLE 2 ----
+==== END OF EXAMPLES ====
+
+==== YOUR TURN ====
+Retrieval Contexts:
+{retrieval_context}
+
+Claims:
+{claims}
+
+JSON:
+"""
+
+    @staticmethod
+    def justify_reason(score, contradictions):
+        return f"""==== TASK INSTRUCTIONS ====
+You will be provided with a list of contradictions and a faithfulness score. 
+The list of contradictions will be references to statements made by a RAG generator that contradicted one or more document(s) from the retrieval context. 
+- To clarify, the LLM produced an `actual output` that contained claims that contradicted the `retrieval context` used to produce the output.
+The faithfulness score is a 0 - 1 float (1 is highest) indicating how factually consistent the RAG generator's output is to the retrieval context.
+
+Your task is to CLEARLY and CONCISELY summarize the contradictions to justify the score. 
+If there are no contradictions, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying). 
+Your reason MUST use information from the contradictions in your reason.
+
+
+==== FORMATTING YOUR ANSWER ====
+Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+
+Example JSON:
+{{
+    "reason": "The score is <faithfulness_score> because <your_reason>."
+}}
+
+==== EXAMPLE ====
+Example Contradictions:
+[
+    {{
+        "verdict": "no", 
+        "reason": "The output claims Marie Curie was born in Russia, but the context clearly states she was born in Warsaw, Poland."
+    }},
+    {{
+        "verdict": "no",
+        "reason": "The output states Marie Curie discovered uranium, but the context indicates she discovered radium and polonium."
+    }}
+]
+
+Example Faithfulness Score:
+0.60
+
+Example Response:
+{{
+    "reason": "The score is 0.60 because the output made two significant factual errors: incorrectly stating Marie Curie was born in Russia instead of Poland, and wrongly attributing the discovery of uranium to her instead of radium and polonium."
+}}
+
+==== YOUR TURN ====
+Faithfulness Score:
+{score}
+
+Contradictions:
+{contradictions}
+
+JSON:
+"""

judgeval/scorers/judgeval_scorers/local_implementations/hallucination/__init__.py

@@ -0,0 +1,3 @@
+from judgeval.scorers.judgeval_scorers.local_implementations.hallucination.hallucination_scorer import HallucinationScorer
+
+__all__ = ["HallucinationScorer"]