judgeval 0.0.32__py3-none-any.whl → 0.0.33__py3-none-any.whl

judgeval/scorers/judgeval_scorers/local_implementations/contextual_recall/contextual_recall_scorer.py

@@ -1,254 +0,0 @@
-from typing import Optional, List, Union
-
-from judgeval.constants import APIScorer
-from judgeval.scorers.utils import (
-    get_or_create_event_loop, 
-    parse_response_json, 
-    scorer_progress_meter, 
-    create_verbose_logs,
-    check_example_params
-)
-from judgeval.judges.utils import create_judge
-from judgeval.scorers import JudgevalScorer
-from judgeval.judges import JudgevalJudge
-from judgeval.judges.utils import create_judge
-from judgeval.data import Example, ExampleParams
-from judgeval.scorers.judgeval_scorers.local_implementations.contextual_recall.prompts import *
-
-required_params = [
-    ExampleParams.INPUT,
-    ExampleParams.ACTUAL_OUTPUT,
-    ExampleParams.EXPECTED_OUTPUT,
-    ExampleParams.RETRIEVAL_CONTEXT,
-]
-
-class ContextualRecallScorer(JudgevalScorer):
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        model: Optional[Union[str, JudgevalJudge]] = None,
-        include_reason: bool = True,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-        user: Optional[str] = None
-    ):
-        super().__init__(
-            score_type=APIScorer.CONTEXTUAL_RECALL,
-            threshold=1 if strict_mode else threshold,
-            evaluation_model=None,
-            include_reason=include_reason,
-            async_mode=async_mode,
-            strict_mode=strict_mode,
-            verbose_mode=verbose_mode
-        )
-        self.user = user
-        self.model, self.using_native_model = create_judge(model)
-        self.evaluation_model = self.model.get_model_name()
-
-    def score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(self, display_meter=_show_indicator):
-            if self.async_mode:
-                loop = get_or_create_event_loop()
-                loop.run_until_complete(
-                    self.a_score_example(example, _show_indicator=False)
-                )
-            else:
-                self.verdicts: List[ContextualRecallVerdict] = (
-                    self._generate_verdicts(
-                        example.expected_output, example.retrieval_context
-                    )
-                )
-                self.score = self._calculate_score()
-                self.reason = self._generate_reason(example.expected_output)
-                self.success = self.score >= self.threshold
-                self.verbose_logs = create_verbose_logs(
-                    self,
-                    steps=[
-                        f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
-                        f"Score: {self.score}\nReason: {self.reason}",
-                    ],
-                )
-                return self.score
-
-    async def a_score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(
-            self,
-            async_mode=True,
-            display_meter=_show_indicator,
-        ):
-            self.verdicts: List[ContextualRecallVerdict] = (
-                await self._a_generate_verdicts(
-                    example.expected_output, example.retrieval_context
-                )
-            )
-            self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(
-                example.expected_output
-            )
-            self.success = self.score >= self.threshold
-            self.verbose_logs = create_verbose_logs(
-                self,
-                steps=[
-                    f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
-                    f"Score: {self.score}\nReason: {self.reason}",
-                ],
-            )
-            return self.score
-
-    async def _a_generate_reason(self, expected_output: str):
-        if self.include_reason is False:
-            return None
-
-        supportive_reasons = []
-        unsupportive_reasons = []
-        for idx, verdict in enumerate(self.verdicts):
-            if verdict.verdict.lower() == "yes":
-                supportive_reasons.append(f"Sentence {idx + 1}: {verdict.reason}")
-            else:
-                unsupportive_reasons.append(f"Sentence {idx + 1}: {verdict.reason}")
-
-        prompt = ContextualRecallTemplate.generate_reason(
-            expected_output=expected_output,
-            supportive_reasons=supportive_reasons,
-            unsupportive_reasons=unsupportive_reasons,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = await self.model.a_generate(prompt, schema=Reason)
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    def _generate_reason(self, expected_output: str):
-        if self.include_reason is False:
-            return None
-
-        supportive_reasons = []
-        unsupportive_reasons = []
-        for verdict in self.verdicts:
-            if verdict.verdict.lower() == "yes":
-                supportive_reasons.append(verdict.reason)
-            else:
-                unsupportive_reasons.append(verdict.reason)
-
-        prompt = ContextualRecallTemplate.generate_reason(
-            expected_output=expected_output,
-            supportive_reasons=supportive_reasons,
-            unsupportive_reasons=unsupportive_reasons,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = self.model.generate(prompt, schema=Reason)
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    def _calculate_score(self):
-        number_of_verdicts = len(self.verdicts)
-        if number_of_verdicts == 0:
-            return 0
-
-        justified_sentences = 0
-        for verdict in self.verdicts:
-            if verdict.verdict.lower() == "yes":
-                justified_sentences += 1
-
-        score = justified_sentences / number_of_verdicts
-        return 0 if self.strict_mode and score < self.threshold else score
-
-    async def _a_generate_verdicts(
-        self, expected_output: str, retrieval_context: List[str]
-    ) -> List[ContextualRecallVerdict]:
-        prompt = ContextualRecallTemplate.generate_verdicts(
-            expected_output=expected_output, retrieval_context=retrieval_context
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            verdicts = [
-                ContextualRecallVerdict(**item) for item in data["verdicts"]
-            ]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts: Verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                verdicts = [
-                    ContextualRecallVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def _generate_verdicts(
-        self, expected_output: str, retrieval_context: List[str]
-    ) -> List[ContextualRecallVerdict]:
-        prompt = ContextualRecallTemplate.generate_verdicts(
-            expected_output=expected_output, retrieval_context=retrieval_context
-        )
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            verdicts = [
-                ContextualRecallVerdict(**item) for item in data["verdicts"]
-            ]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts: Verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                verdicts = [
-                    ContextualRecallVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def _success_check(self) -> bool:
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score >= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Contextual Recall"

judgeval/scorers/judgeval_scorers/local_implementations/contextual_recall/prompts.py

@@ -1,142 +0,0 @@
-from typing import List
-from pydantic import BaseModel
-
-
-class ContextualRecallVerdict(BaseModel):
-    verdict: str
-    reason: str
-
-
-class Verdicts(BaseModel):
-    verdicts: List[ContextualRecallVerdict]
-
-
-class Reason(BaseModel):
-    reason: str
-
-
-class ContextualRecallTemplate:
-
-    @staticmethod
-    def generate_verdicts(expected_output, retrieval_context):
-        return f"""
-==== TASK INSTRUCTIONS ====
-You will be provided with an expected output and a retrieval context (list of retrieved documents). Your task is to take each sentence in the expected output and determine whether the sentence is ATTRIBUTABLE or RELEVANT to ANY PART of the retrieval context.
-
-==== FORMATTING YOUR ANSWER ====
-Please format your answer as a list of JSON objects, each with two keys: `verdict` and `reason`.
-The `verdict` key should STRICTLY be 'yes' or 'no'. You should answer 'yes' if the sentence can be attributed/is relevant to ANY PART(S) of the retrieval context. If not, you should answer 'no'.
-The `reason` key should provide a justification of your verdict. In the justification, you should aim to include references to the document(s) in the retrieval context (eg., 1st document, and 2nd document in the retrieval context) that is attributed/relevant to the expected output sentence. 
-Please also AIM TO CITE the specific part of the retrieval context to justify your verdict, but **be extremely concise! Cut short the quote with an ellipsis if possible**.
-
-Here's an example of formatting your answer:
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes",
-            "reason": "..."
-        }},
-        ...
-    ]  
-}}
-
-==== EXAMPLE ====
-Expected Output:
-The Earth's climate has warmed significantly over the past century. This warming is primarily caused by human activities like burning fossil fuels. Today's weather was sunny and warm.
-
-Retrieval Context:
-["Global temperatures have risen by approximately 1.1°C since pre-industrial times, with most of this increase occurring in the past 100 years.", 
-"Scientific consensus shows that greenhouse gas emissions from human activities, particularly the burning of coal, oil and gas, are the main driver of observed climate change."]
-
-Example Response:
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes",
-            "reason": "The 1st document directly confirms this, stating 'temperatures have risen by approximately 1.1°C...in the past 100 years'"
-        }},
-        {{
-            "verdict": "yes", 
-            "reason": "The 2nd document explicitly states that 'greenhouse gas emissions from human activities, particularly the burning of...fossil fuels' drive climate change"
-        }},
-        {{
-            "verdict": "no",
-            "reason": "Neither document contains information about today's specific weather conditions"
-        }}
-    ]
-}}
-
-Since your task is to generate a verdict for each sentence, the number of 'verdicts' SHOULD BE EXACTLY EQUAL to the number of sentences in of `expected output`.
-**
-
-==== YOUR TURN ====
-Expected Output:
-{expected_output}
-
-Retrieval Context:
-{retrieval_context}
-
-JSON:
-"""
-    @staticmethod
-    def generate_reason(
-        expected_output, supportive_reasons, unsupportive_reasons, score
-    ):
-        return f"""
-==== PROBLEM SETUP ====
-You will be provided with an expected output, a list of supportive reasons, a list of unsupportive reasons, and a contextual recall score. Let's break down each input component:
-- expected output: A text generated by a language model to answer a question/solve a task.
-- supportive reasons: A list of reasons why a specific sentence in the expected output can be attributed/is relevant to any part of the retrieval context (a list of documents retrieved in a RAG pipeline)
-- unsupportive reasons: A list of reasons why a specific sentence in the expected output cannot be attributed/is not relevant to any part of the retrieval context
-**NOTE**: The reasons are provided in the form of "Sentence <number>: <reason>", where <number> is the sentence number in the expected output.
-- contextual recall score: A score between 0 and 1 (closer to 1 the better) representing how much of the expected output can be attributed/is relevant to any part of the retrieval context. 
-The point of this score is to measure how well the retriever in a RAG pipeline operates, retrieving relevant documents that should back the expected output of the RAG generator.
-
-==== TASK INSTRUCTIONS ====
-Given these inputs, summarize a CONCISE and CLEAR reason for the value of the contextual recall score. Remember, the score is a measure of how well the retriever in a RAG pipeline operates, retrieving relevant documents that should back the expected output of the RAG generator.
-In your reason, you should reference the supportive/unsupportive reasons by their sentence number to justify the score. Make specific references to the retrieval context in your reason if applicable.
-
-==== FORMATTING YOUR ANSWER ====
-IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-Example JSON:
-{{
-    "reason": "The score is <contextual_recall_score> because <your_reason>."
-}}
-
-DO NOT mention 'supportive reasons' and 'unsupportive reasons' in your reason, these terms are just here for you to understand the broader scope of things.
-If the score is 1, keep it short and say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
-
-==== EXAMPLE ====
-Expected Output:
-The Earth's climate has warmed significantly over the past century. This warming is primarily caused by human activities like burning fossil fuels. Today's weather was sunny and warm.
-
-Supportive Reasons:
-Sentence 1: The first document confirms this by stating global temperatures have risen by 1.1°C in the past 100 years
-Sentence 2: The second document directly states that human activities and fossil fuel burning drive climate change
-
-Unsupportive Reasons:
-Sentence 3: Neither document contains information about today's specific weather conditions
-
-Contextual Recall Score:
-0.67
-
-Example Response:
-{{
-    "reason": "The score is 0.67 because while sentences 1 and 2 are well-supported by the retrieval context with specific temperature data and human activity impacts, sentence 3 about today's weather has no backing in the provided documents."
-}}
-
-==== YOUR TURN ====
-Contextual Recall Score:
-{score}
-
-Expected Output:
-{expected_output}
-
-Supportive Reasons:
-{supportive_reasons}
-
-Unsupportive Reasons:
-{unsupportive_reasons}
-
-JSON:
-"""

judgeval/scorers/judgeval_scorers/local_implementations/contextual_relevancy/__init__.py

@@ -1,3 +0,0 @@
-from judgeval.scorers.judgeval_scorers.local_implementations.contextual_relevancy.contextual_relevancy_scorer import ContextualRelevancyScorer
-
-__all__ = ["ContextualRelevancyScorer"]

judgeval/scorers/judgeval_scorers/local_implementations/contextual_relevancy/contextual_relevancy_scorer.py

@@ -1,245 +0,0 @@
-from typing import Optional, List, Union
-import asyncio
-
-from judgeval.constants import APIScorer
-from judgeval.scorers.utils import (get_or_create_event_loop,
-                                    scorer_progress_meter,
-                                    create_verbose_logs,
-                                    parse_response_json,
-                                    check_example_params
-                                    )
-from judgeval.scorers import JudgevalScorer
-from judgeval.judges import JudgevalJudge
-from judgeval.judges.utils import create_judge
-from judgeval.data import Example, ExampleParams
-from judgeval.scorers.judgeval_scorers.local_implementations.contextual_relevancy.prompts import *
-
-
-required_params = [
-    ExampleParams.INPUT,
-    ExampleParams.ACTUAL_OUTPUT,
-    ExampleParams.RETRIEVAL_CONTEXT,
-]
-
-
-class ContextualRelevancyScorer(JudgevalScorer):
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        model: Optional[Union[str, JudgevalJudge]] = None,
-        include_reason: bool = True,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-        user: Optional[str] = None
-    ):
-        super().__init__(
-            score_type=APIScorer.CONTEXTUAL_RELEVANCY,
-            threshold=1 if strict_mode else threshold,
-            evaluation_model=None,
-            include_reason=include_reason,
-            async_mode=async_mode,
-            strict_mode=strict_mode,
-            verbose_mode=verbose_mode
-        )
-        self.user = user
-        self.model, self.using_native_model = create_judge(model)
-        self.evaluation_model = self.model.get_model_name()
-
-    def score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-        
-        with scorer_progress_meter(self, display_meter=_show_indicator):
-            if self.async_mode:
-                loop = get_or_create_event_loop()
-                loop.run_until_complete(
-                    self.a_score_example(example, _show_indicator=False)
-                )
-            else:
-                self.verdicts_list: List[ContextualRelevancyVerdicts] = [
-                    (self._generate_verdicts(example.input, context))
-                    for context in example.retrieval_context
-                ]
-                self.score = self._calculate_score()
-                self.reason = self._generate_reason(example.input)
-                self.success = self.score >= self.threshold
-                self.verbose_logs = create_verbose_logs(
-                    self,
-                    steps=[
-                        f"Verdicts:\n{[v.model_dump() for v in self.verdicts_list]}",
-                        f"Score: {self.score}\nReason: {self.reason}",
-                    ],
-                )
-
-                return self.score
-
-    async def a_score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-        
-        with scorer_progress_meter(
-            self,
-            async_mode=True,
-            display_meter=_show_indicator,
-        ):
-            self.verdicts_list: List[ContextualRelevancyVerdicts] = (
-                await asyncio.gather(
-                    *[
-                        self._a_generate_verdicts(example.input, context)
-                        for context in example.retrieval_context
-                    ]
-                )
-            )
-            self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(example.input)
-            self.success = self.score >= self.threshold
-            self.verbose_logs = create_verbose_logs(
-                self,
-                steps=[
-                    f"Verdicts:\n{[v.model_dump() for v in self.verdicts_list]}",
-                    f"Score: {self.score}\nReason: {self.reason}",
-                ],
-            )
-            return self.score
-
-    async def _a_generate_reason(self, input: str):
-        if self.include_reason is False:
-            return None
-
-        irrelevancies = []
-        relevant_statements = []
-        for verdicts in self.verdicts_list:
-            for verdict in verdicts.verdicts:
-                if verdict.verdict.lower() == "no":
-                    irrelevancies.append(verdict.model_dump())
-                else:
-                    relevant_statements.append(verdict.model_dump())
-        prompt: dict = ContextualRelevancyTemplate.generate_reason(
-            input=input,
-            irrelevancies=irrelevancies,
-            relevant_statements=relevant_statements,
-            score=format(self.score, ".2f"),
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = await self.model.a_generate(prompt, schema=Reason)
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    def _generate_reason(self, input: str):
-        if self.include_reason is False:
-            return None
-
-        irrelevancies = []
-        relevant_statements = []
-        for verdicts in self.verdicts_list:
-            for verdict in verdicts.verdicts:
-                if verdict.verdict.lower() == "no":
-                    irrelevancies.append(verdict.reason)
-                else:
-                    relevant_statements.append(verdict.statement)
-
-        prompt: dict = ContextualRelevancyTemplate.generate_reason(
-            input=input,
-            irrelevancies=irrelevancies,
-            relevant_statements=relevant_statements,
-            score=format(self.score, ".2f"),
-        )
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = self.model.generate(prompt, schema=Reason)
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    def _calculate_score(self):
-        total_verdicts = 0
-        relevant_statements = 0
-        for verdicts in self.verdicts_list:
-            for verdict in verdicts.verdicts:
-                total_verdicts += 1
-                if verdict.verdict.lower() == "yes":
-                    relevant_statements += 1
-
-        if total_verdicts == 0:
-            return 0
-
-        score = relevant_statements / total_verdicts
-        return 0 if self.strict_mode and score < self.threshold else score
-
-    async def _a_generate_verdicts(
-        self, input: str, context: List[str]
-    ) -> ContextualRelevancyVerdicts:
-        prompt = ContextualRelevancyTemplate.generate_verdicts(
-            input=input, context=context
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return ContextualRelevancyVerdicts(**data)
-        else:
-            try:
-                res = await self.model.a_generate(
-                    prompt, schema=ContextualRelevancyVerdicts
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return ContextualRelevancyVerdicts(**data)
-
-    def _generate_verdicts(
-        self, input: str, context: str
-    ) -> ContextualRelevancyVerdicts:
-        prompt = ContextualRelevancyTemplate.generate_verdicts(
-            input=input, context=context
-        )
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return ContextualRelevancyVerdicts(**data)
-        else:
-            try:
-                res = self.model.generate(
-                    prompt, schema=ContextualRelevancyVerdicts
-                )
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return ContextualRelevancyVerdicts(**data)
-
-    def _success_check(self) -> bool:
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score >= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Contextual Relevancy"
-