judgeval 0.0.31__py3-none-any.whl → 0.0.33__py3-none-any.whl

judgeval/scorers/judgeval_scorers/local_implementations/contextual_precision/contextual_precision_scorer.py

@@ -1,264 +0,0 @@
-from typing import Optional, List, Union
-
-from judgeval.judges import JudgevalJudge
-from judgeval.judges.utils import create_judge
-from judgeval.data import Example, ExampleParams
-from judgeval.scorers import JudgevalScorer
-from judgeval.constants import APIScorer
-from judgeval.scorers.utils import (
-    get_or_create_event_loop,
-    parse_response_json,
-    scorer_progress_meter,
-    create_verbose_logs,
-    check_example_params,
-)
-from judgeval.scorers.judgeval_scorers.local_implementations.contextual_precision.prompts import *
-
-required_params = [
-    ExampleParams.INPUT,
-    ExampleParams.ACTUAL_OUTPUT,
-    ExampleParams.RETRIEVAL_CONTEXT,
-    ExampleParams.EXPECTED_OUTPUT,
-]
-
-class ContextualPrecisionScorer(JudgevalScorer):
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        model: Optional[Union[str, JudgevalJudge]] = None,
-        include_reason: bool = True,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-    ):
-        super().__init__(
-            score_type=APIScorer.CONTEXTUAL_PRECISION,
-            threshold=1 if strict_mode else threshold,
-            evaluation_model=None,
-            include_reason=include_reason,
-            async_mode=async_mode,
-            strict_mode=strict_mode,
-            verbose_mode=verbose_mode
-        )
-        self.model, self.using_native_model = create_judge(model)
-        self.evaluation_model = self.model.get_model_name()
-
-    def score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(self, display_meter=_show_indicator):
-            if self.async_mode:
-                loop = get_or_create_event_loop()
-                loop.run_until_complete(
-                    self.a_score_example(example, _show_indicator=False)
-                )
-            else:
-                self.verdicts: List[ContextualPrecisionVerdict] = (
-                    self._generate_verdicts(
-                        example.input,
-                        example.expected_output,
-                        example.retrieval_context,
-                    )
-                )
-                self.score = self._calculate_score()
-                self.reason = self._generate_reason(example.input)
-                self.success = self.score >= self.threshold
-                self.verbose_logs = create_verbose_logs(
-                    self,
-                    steps=[
-                        # Convert to dict for serialization purposes
-                        f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
-                        f"Score: {self.score}\nReason: {self.reason}",
-                    ],
-                )
-                return self.score
-
-    async def a_score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-        
-        with scorer_progress_meter(
-            self,
-            async_mode=True,
-            display_meter=_show_indicator,
-        ):
-            self.verdicts: List[ContextualPrecisionVerdict] = (
-                await self._a_generate_verdicts(
-                    example.input,
-                    example.expected_output,
-                    example.retrieval_context,
-                )
-            )
-            self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(example.input)
-            self.success = self.score >= self.threshold
-            self.verbose_logs = create_verbose_logs(
-                self,
-                steps=[
-                    # Convert to dict for serialization purposes
-                    f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
-                    f"Score: {self.score}\nReason: {self.reason}",
-                ],
-            )
-            return self.score
-
-    async def _a_generate_reason(self, input: str):
-        if self.include_reason is False:
-            return None
-
-        retrieval_contexts_verdicts = [
-            {"verdict": verdict.verdict, "reasons": verdict.reason}
-            for verdict in self.verdicts
-        ]
-        prompt = ContextualPrecisionTemplate.generate_reason(
-            input=input,
-            verdicts=retrieval_contexts_verdicts,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = await self.model.a_generate(prompt, schema=Reason)
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    def _generate_reason(self, input: str):
-        if self.include_reason is False:
-            return None
-
-        retrieval_contexts_verdicts = [
-            {"verdict": verdict.verdict, "reasons": verdict.reason}
-            for verdict in self.verdicts
-        ]
-        prompt = ContextualPrecisionTemplate.generate_reason(
-            input=input,
-            verdicts=retrieval_contexts_verdicts,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = self.model.generate(prompt, schema=Reason)
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    async def _a_generate_verdicts(
-        self, input: str, expected_output: str, retrieval_context: List[str]
-    ) -> List[ContextualPrecisionVerdict]:
-        prompt = ContextualPrecisionTemplate.generate_verdicts(
-            input=input,
-            expected_output=expected_output,
-            retrieval_context=retrieval_context,
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            verdicts = [
-                ContextualPrecisionVerdict(**item) for item in data["verdicts"]
-            ]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                verdicts = [
-                    ContextualPrecisionVerdict(**item)
-                    for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def _generate_verdicts(
-        self, input: str, expected_output: str, retrieval_context: List[str]
-    ) -> List[ContextualPrecisionVerdict]:
-        prompt = ContextualPrecisionTemplate.generate_verdicts(
-            input=input,
-            expected_output=expected_output,
-            retrieval_context=retrieval_context,
-        )
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            verdicts = [
-                ContextualPrecisionVerdict(**item) for item in data["verdicts"]
-            ]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                verdicts = [
-                    ContextualPrecisionVerdict(**item)
-                    for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def _calculate_score(self):
-        number_of_verdicts = len(self.verdicts)
-        if number_of_verdicts == 0:
-            return 0
-
-        # Convert verdicts to a binary list where 'yes' is 1 and others are 0
-        node_verdicts = [
-            1 if v.verdict.strip().lower() == "yes" else 0
-            for v in self.verdicts
-        ]
-
-        sum_weighted_precision_at_k = 0.0
-        relevant_nodes_count = 0
-        for k, is_relevant in enumerate(node_verdicts, start=1):
-            # If the item is relevant, update the counter and add the weighted precision at k to the sum
-            if is_relevant:
-                relevant_nodes_count += 1
-                precision_at_k = relevant_nodes_count / k
-                sum_weighted_precision_at_k += precision_at_k * is_relevant
-
-        if relevant_nodes_count == 0:
-            return 0
-        # Calculate weighted cumulative precision
-        score = sum_weighted_precision_at_k / relevant_nodes_count
-        return 0 if self.strict_mode and score < self.threshold else score
-
-    def _success_check(self) -> bool:
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score >= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Contextual Precision"

judgeval/scorers/judgeval_scorers/local_implementations/contextual_precision/prompts.py

@@ -1,106 +0,0 @@
-from typing import List
-from pydantic import BaseModel
-
-
-class ContextualPrecisionVerdict(BaseModel):
-    verdict: str
-    reason: str
-
-
-class Verdicts(BaseModel):
-    verdicts: List[ContextualPrecisionVerdict]
-
-
-class Reason(BaseModel):
-    reason: str
-
-
-class ContextualPrecisionTemplate:
-    @staticmethod
-    def generate_verdicts(input, expected_output, retrieval_context):
-        return f"""==== TASK INSTRUCTIONS ====\nGiven the input, expected output, and retrieval context, your task is to determine whether each document in the retrieval context was relevant to arrive at the expected output.
-You should reason through the documents in the retrieval context thoroughly, and then generate a list of JSON objects representing your decision.
-
-==== FORMAT INSTRUCTIONS ====\nIMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON. These JSON only contain the `verdict` key that outputs only 'yes' or 'no', and a `reason` key to justify the verdict. In your reason, aim to quote parts of the context to support your verdict.
-
-==== EXAMPLE ====
-Example Input: "What are the main symptoms of COVID-19?"
-Example Expected Output: "The main symptoms of COVID-19 include fever, cough, fatigue, and loss of taste or smell."
-Example Retrieval Context: ["Common COVID-19 symptoms include fever and dry cough", "Loss of taste and smell are distinctive COVID-19 symptoms", "The first COVID-19 case was reported in Wuhan", "My friend's birthday party was fun last weekend"]
-
-Example output JSON:
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes", 
-            "reason": "The text directly lists key COVID-19 symptoms including 'fever and dry cough' which are part of the main symptoms."
-        }},
-        {{
-            "verdict": "yes",
-            "reason": "The text mentions 'loss of taste and smell' which are distinctive symptoms of COVID-19 that should be included."
-        }},
-        {{
-            "verdict": "no",
-            "reason": "While related to COVID-19, the origin of the first case is not relevant to listing the main symptoms."
-        }},
-        {{
-            "verdict": "no",
-            "reason": "A personal anecdote about a birthday party has no relevance to COVID-19 symptoms."
-        }}
-    ]
-}}
-
-Your task is to generate a verdict for each document in the retrieval context, so the number of 'verdicts' SHOULD BE EXACTLY EQUAL to that of the retrievalcontexts.
-
-==== YOUR TURN ====
-Input:
-{input}
-
-Expected output:
-{expected_output}
-
-Retrieval Context:
-{retrieval_context}
-
-JSON:
-"""
-
-    @staticmethod
-    def generate_reason(input, verdicts, score):
-        return f"""==== TASK INSTRUCTIONS ====\nYou will be provided with an input, retrieval contexts, and a contextual precision score. Your task is to provide a CLEAR and CONCISE reason for the score. 
-You should explain why the score is not higher, but also the current score is reasonable. Here's a further breakdown of the task:
-
-1. input (str) is a task or question that the model attempted to solve
-2. retrieval contexts (list[dict]) is a list of JSON with the following keys:
-- `verdict` (str): either 'yes' or 'no', which represents whether the corresponding document in the retrieval context is relevant to the input.
-- `reason` (str): a reason for the verdict.
-3. The contextual precision score is a float between 0 and 1 and represents if the relevant documents are ranked higher than irrelevant ones in the retrieval context. 
-The ranking can be inferred by the order of the retrieval documents: retrieval contexts is given IN THE ORDER OF THE DOCUMENT RANKINGS.
-This implies that the score will be higher if the relevant documents are ranked higher (appears earlier in the list) than irrelevant ones.
-
-==== FORMAT INSTRUCTIONS ====\nIMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason for the contextual precision score.
-Example JSON:
-{{
-    "reason": "The score is <contextual_precision_score> because <your_reason>."
-}}
-
-
-DO NOT mention 'verdict' in your reason, but instead phrase it as irrelevant nodes. The term 'verdict' are just here for you to understand the broader scope of things.
-Also DO NOT mention there are `reason` fields in the retrieval contexts you are presented with, instead just use the information in the `reason` field.
-In your reason, you MUST USE the `reason`, QUOTES in the 'reason', and the node RANK (starting from 1, eg. first node) to explain why the 'no' verdicts should be ranked lower than the 'yes' verdicts.
-When addressing nodes, make it explicit that it is nodes in retrieval context.
-If the score is 1, keep it short and say something positive with an upbeat tone (but don't overdo it otherwise it gets annoying).
-
-==== YOUR TURN ====
-Contextual Precision Score:
-{score}
-
-Input:
-{input}
-
-Retrieval Contexts:
-{verdicts}
-
-JSON:
-"""
-

judgeval/scorers/judgeval_scorers/local_implementations/contextual_recall/__init__.py

@@ -1,3 +0,0 @@
-from .contextual_recall_scorer import ContextualRecallScorer
-
-__all__ = ["ContextualRecallScorer"]

judgeval/scorers/judgeval_scorers/local_implementations/contextual_recall/contextual_recall_scorer.py

@@ -1,254 +0,0 @@
-from typing import Optional, List, Union
-
-from judgeval.constants import APIScorer
-from judgeval.scorers.utils import (
-    get_or_create_event_loop, 
-    parse_response_json, 
-    scorer_progress_meter, 
-    create_verbose_logs,
-    check_example_params
-)
-from judgeval.judges.utils import create_judge
-from judgeval.scorers import JudgevalScorer
-from judgeval.judges import JudgevalJudge
-from judgeval.judges.utils import create_judge
-from judgeval.data import Example, ExampleParams
-from judgeval.scorers.judgeval_scorers.local_implementations.contextual_recall.prompts import *
-
-required_params = [
-    ExampleParams.INPUT,
-    ExampleParams.ACTUAL_OUTPUT,
-    ExampleParams.EXPECTED_OUTPUT,
-    ExampleParams.RETRIEVAL_CONTEXT,
-]
-
-class ContextualRecallScorer(JudgevalScorer):
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        model: Optional[Union[str, JudgevalJudge]] = None,
-        include_reason: bool = True,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-        user: Optional[str] = None
-    ):
-        super().__init__(
-            score_type=APIScorer.CONTEXTUAL_RECALL,
-            threshold=1 if strict_mode else threshold,
-            evaluation_model=None,
-            include_reason=include_reason,
-            async_mode=async_mode,
-            strict_mode=strict_mode,
-            verbose_mode=verbose_mode
-        )
-        self.user = user
-        self.model, self.using_native_model = create_judge(model)
-        self.evaluation_model = self.model.get_model_name()
-
-    def score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(self, display_meter=_show_indicator):
-            if self.async_mode:
-                loop = get_or_create_event_loop()
-                loop.run_until_complete(
-                    self.a_score_example(example, _show_indicator=False)
-                )
-            else:
-                self.verdicts: List[ContextualRecallVerdict] = (
-                    self._generate_verdicts(
-                        example.expected_output, example.retrieval_context
-                    )
-                )
-                self.score = self._calculate_score()
-                self.reason = self._generate_reason(example.expected_output)
-                self.success = self.score >= self.threshold
-                self.verbose_logs = create_verbose_logs(
-                    self,
-                    steps=[
-                        f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
-                        f"Score: {self.score}\nReason: {self.reason}",
-                    ],
-                )
-                return self.score
-
-    async def a_score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(
-            self,
-            async_mode=True,
-            display_meter=_show_indicator,
-        ):
-            self.verdicts: List[ContextualRecallVerdict] = (
-                await self._a_generate_verdicts(
-                    example.expected_output, example.retrieval_context
-                )
-            )
-            self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(
-                example.expected_output
-            )
-            self.success = self.score >= self.threshold
-            self.verbose_logs = create_verbose_logs(
-                self,
-                steps=[
-                    f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
-                    f"Score: {self.score}\nReason: {self.reason}",
-                ],
-            )
-            return self.score
-
-    async def _a_generate_reason(self, expected_output: str):
-        if self.include_reason is False:
-            return None
-
-        supportive_reasons = []
-        unsupportive_reasons = []
-        for idx, verdict in enumerate(self.verdicts):
-            if verdict.verdict.lower() == "yes":
-                supportive_reasons.append(f"Sentence {idx + 1}: {verdict.reason}")
-            else:
-                unsupportive_reasons.append(f"Sentence {idx + 1}: {verdict.reason}")
-
-        prompt = ContextualRecallTemplate.generate_reason(
-            expected_output=expected_output,
-            supportive_reasons=supportive_reasons,
-            unsupportive_reasons=unsupportive_reasons,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = await self.model.a_generate(prompt, schema=Reason)
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    def _generate_reason(self, expected_output: str):
-        if self.include_reason is False:
-            return None
-
-        supportive_reasons = []
-        unsupportive_reasons = []
-        for verdict in self.verdicts:
-            if verdict.verdict.lower() == "yes":
-                supportive_reasons.append(verdict.reason)
-            else:
-                unsupportive_reasons.append(verdict.reason)
-
-        prompt = ContextualRecallTemplate.generate_reason(
-            expected_output=expected_output,
-            supportive_reasons=supportive_reasons,
-            unsupportive_reasons=unsupportive_reasons,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = self.model.generate(prompt, schema=Reason)
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    def _calculate_score(self):
-        number_of_verdicts = len(self.verdicts)
-        if number_of_verdicts == 0:
-            return 0
-
-        justified_sentences = 0
-        for verdict in self.verdicts:
-            if verdict.verdict.lower() == "yes":
-                justified_sentences += 1
-
-        score = justified_sentences / number_of_verdicts
-        return 0 if self.strict_mode and score < self.threshold else score
-
-    async def _a_generate_verdicts(
-        self, expected_output: str, retrieval_context: List[str]
-    ) -> List[ContextualRecallVerdict]:
-        prompt = ContextualRecallTemplate.generate_verdicts(
-            expected_output=expected_output, retrieval_context=retrieval_context
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            verdicts = [
-                ContextualRecallVerdict(**item) for item in data["verdicts"]
-            ]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts: Verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                verdicts = [
-                    ContextualRecallVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def _generate_verdicts(
-        self, expected_output: str, retrieval_context: List[str]
-    ) -> List[ContextualRecallVerdict]:
-        prompt = ContextualRecallTemplate.generate_verdicts(
-            expected_output=expected_output, retrieval_context=retrieval_context
-        )
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            verdicts = [
-                ContextualRecallVerdict(**item) for item in data["verdicts"]
-            ]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts: Verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                verdicts = [
-                    ContextualRecallVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def _success_check(self) -> bool:
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score >= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Contextual Recall"