judgeval 0.0.3__py3-none-any.whl → 0.0.4__py3-none-any.whl

judgeval/scorers/judgeval_scorers/local_implementations/contextual_precision/contextual_precision_scorer.py

@@ -0,0 +1,259 @@
+from typing import Optional, List, Union
+
+from judgeval.judges import JudgevalJudge
+from judgeval.judges.utils import create_judge
+from judgeval.data import Example, ExampleParams
+from judgeval.scorers import JudgevalScorer
+from judgeval.scorers.utils import (
+    get_or_create_event_loop,
+    parse_response_json,
+    scorer_progress_meter,
+    create_verbose_logs,
+    check_example_params,
+)
+from judgeval.scorers.judgeval_scorers.local_implementations.contextual_precision.prompts import *
+
+required_params = [
+    ExampleParams.INPUT,
+    ExampleParams.ACTUAL_OUTPUT,
+    ExampleParams.RETRIEVAL_CONTEXT,
+    ExampleParams.EXPECTED_OUTPUT,
+]
+
+class ContextualPrecisionScorer(JudgevalScorer):
+    def __init__(
+        self,
+        threshold: float = 0.5,
+        model: Optional[Union[str, JudgevalJudge]] = None,
+        include_reason: bool = True,
+        async_mode: bool = True,
+        strict_mode: bool = False,
+        verbose_mode: bool = False,
+    ):
+        self.threshold = 1 if strict_mode else threshold
+        self.include_reason = include_reason
+        self.model, self.using_native_model = create_judge(model)
+        self.evaluation_model = self.model.get_model_name()
+        self.async_mode = async_mode
+        self.strict_mode = strict_mode
+        self.verbose_mode = verbose_mode
+
+    def score_example(
+        self,
+        example: Example,
+        _show_indicator: bool = True,
+    ) -> float:
+        check_example_params(example, required_params, self)
+
+        with scorer_progress_meter(self, display_meter=_show_indicator):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                loop.run_until_complete(
+                    self.a_score_example(example, _show_indicator=False)
+                )
+            else:
+                self.verdicts: List[ContextualPrecisionVerdict] = (
+                    self._generate_verdicts(
+                        example.input,
+                        example.expected_output,
+                        example.retrieval_context,
+                    )
+                )
+                self.score = self._calculate_score()
+                self.reason = self._generate_reason(example.input)
+                self.success = self.score >= self.threshold
+                self.verbose_logs = create_verbose_logs(
+                    self,
+                    steps=[
+                        # Convert to dict for serialization purposes
+                        f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
+                        f"Score: {self.score}\nReason: {self.reason}",
+                    ],
+                )
+                return self.score
+
+    async def a_score_example(
+        self,
+        example: Example,
+        _show_indicator: bool = True,
+    ) -> float:
+        check_example_params(example, required_params, self)
+        
+        with scorer_progress_meter(
+            self,
+            async_mode=True,
+            display_meter=_show_indicator,
+        ):
+            self.verdicts: List[ContextualPrecisionVerdict] = (
+                await self._a_generate_verdicts(
+                    example.input,
+                    example.expected_output,
+                    example.retrieval_context,
+                )
+            )
+            self.score = self._calculate_score()
+            self.reason = await self._a_generate_reason(example.input)
+            self.success = self.score >= self.threshold
+            self.verbose_logs = create_verbose_logs(
+                self,
+                steps=[
+                    # Convert to dict for serialization purposes
+                    f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
+                    f"Score: {self.score}\nReason: {self.reason}",
+                ],
+            )
+            return self.score
+
+    async def _a_generate_reason(self, input: str):
+        if self.include_reason is False:
+            return None
+
+        retrieval_contexts_verdicts = [
+            {"verdict": verdict.verdict, "reasons": verdict.reason}
+            for verdict in self.verdicts
+        ]
+        prompt = ContextualPrecisionTemplate.generate_reason(
+            input=input,
+            verdicts=retrieval_contexts_verdicts,
+            score=format(self.score, ".2f"),
+        )
+
+        if self.using_native_model:
+            res = await self.model.a_generate(prompt)
+            data = parse_response_json(res, self)
+            return data["reason"]
+        else:
+            try:
+                res: Reason = await self.model.a_generate(prompt, schema=Reason)
+                return res.reason
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = parse_response_json(res, self)
+                return data["reason"]
+
+    def _generate_reason(self, input: str):
+        if self.include_reason is False:
+            return None
+
+        retrieval_contexts_verdicts = [
+            {"verdict": verdict.verdict, "reasons": verdict.reason}
+            for verdict in self.verdicts
+        ]
+        prompt = ContextualPrecisionTemplate.generate_reason(
+            input=input,
+            verdicts=retrieval_contexts_verdicts,
+            score=format(self.score, ".2f"),
+        )
+
+        if self.using_native_model:
+            res = self.model.generate(prompt)
+            data = parse_response_json(res, self)
+            return data["reason"]
+        else:
+            try:
+                res: Reason = self.model.generate(prompt, schema=Reason)
+                return res.reason
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = parse_response_json(res, self)
+                return data["reason"]
+
+    async def _a_generate_verdicts(
+        self, input: str, expected_output: str, retrieval_context: List[str]
+    ) -> List[ContextualPrecisionVerdict]:
+        prompt = ContextualPrecisionTemplate.generate_verdicts(
+            input=input,
+            expected_output=expected_output,
+            retrieval_context=retrieval_context,
+        )
+        if self.using_native_model:
+            res = await self.model.a_generate(prompt)
+            data = parse_response_json(res, self)
+            verdicts = [
+                ContextualPrecisionVerdict(**item) for item in data["verdicts"]
+            ]
+            return verdicts
+        else:
+            try:
+                res: Verdicts = await self.model.a_generate(
+                    prompt, schema=Verdicts
+                )
+                verdicts = [item for item in res.verdicts]
+                return verdicts
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = parse_response_json(res, self)
+                verdicts = [
+                    ContextualPrecisionVerdict(**item)
+                    for item in data["verdicts"]
+                ]
+                return verdicts
+
+    def _generate_verdicts(
+        self, input: str, expected_output: str, retrieval_context: List[str]
+    ) -> List[ContextualPrecisionVerdict]:
+        prompt = ContextualPrecisionTemplate.generate_verdicts(
+            input=input,
+            expected_output=expected_output,
+            retrieval_context=retrieval_context,
+        )
+        if self.using_native_model:
+            res = self.model.generate(prompt)
+            data = parse_response_json(res, self)
+            verdicts = [
+                ContextualPrecisionVerdict(**item) for item in data["verdicts"]
+            ]
+            return verdicts
+        else:
+            try:
+                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
+                verdicts = [item for item in res.verdicts]
+                return verdicts
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = parse_response_json(res, self)
+                verdicts = [
+                    ContextualPrecisionVerdict(**item)
+                    for item in data["verdicts"]
+                ]
+                return verdicts
+
+    def _calculate_score(self):
+        number_of_verdicts = len(self.verdicts)
+        if number_of_verdicts == 0:
+            return 0
+
+        # Convert verdicts to a binary list where 'yes' is 1 and others are 0
+        node_verdicts = [
+            1 if v.verdict.strip().lower() == "yes" else 0
+            for v in self.verdicts
+        ]
+
+        sum_weighted_precision_at_k = 0.0
+        relevant_nodes_count = 0
+        for k, is_relevant in enumerate(node_verdicts, start=1):
+            # If the item is relevant, update the counter and add the weighted precision at k to the sum
+            if is_relevant:
+                relevant_nodes_count += 1
+                precision_at_k = relevant_nodes_count / k
+                sum_weighted_precision_at_k += precision_at_k * is_relevant
+
+        if relevant_nodes_count == 0:
+            return 0
+        # Calculate weighted cumulative precision
+        score = sum_weighted_precision_at_k / relevant_nodes_count
+        return 0 if self.strict_mode and score < self.threshold else score
+
+    def _success_check(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Contextual Precision"

judgeval/scorers/judgeval_scorers/local_implementations/contextual_precision/prompts.py

@@ -0,0 +1,106 @@
+from typing import List, Optional
+from pydantic import BaseModel, Field
+
+
+class ContextualPrecisionVerdict(BaseModel):
+    verdict: str
+    reason: str
+
+
+class Verdicts(BaseModel):
+    verdicts: List[ContextualPrecisionVerdict]
+
+
+class Reason(BaseModel):
+    reason: str
+
+
+class ContextualPrecisionTemplate:
+    @staticmethod
+    def generate_verdicts(input, expected_output, retrieval_context):
+        return f"""==== TASK INSTRUCTIONS ====\nGiven the input, expected output, and retrieval context, your task is to determine whether each document in the retrieval context was relevant to arrive at the expected output.
+You should reason through the documents in the retrieval context thoroughly, and then generate a list of JSON objects representing your decision.
+
+==== FORMAT INSTRUCTIONS ====\nIMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON. These JSON only contain the `verdict` key that outputs only 'yes' or 'no', and a `reason` key to justify the verdict. In your reason, aim to quote parts of the context to support your verdict.
+
+==== EXAMPLE ====
+Example Input: "What are the main symptoms of COVID-19?"
+Example Expected Output: "The main symptoms of COVID-19 include fever, cough, fatigue, and loss of taste or smell."
+Example Retrieval Context: ["Common COVID-19 symptoms include fever and dry cough", "Loss of taste and smell are distinctive COVID-19 symptoms", "The first COVID-19 case was reported in Wuhan", "My friend's birthday party was fun last weekend"]
+
+Example output JSON:
+{{
+    "verdicts": [
+        {{
+            "verdict": "yes", 
+            "reason": "The text directly lists key COVID-19 symptoms including 'fever and dry cough' which are part of the main symptoms."
+        }},
+        {{
+            "verdict": "yes",
+            "reason": "The text mentions 'loss of taste and smell' which are distinctive symptoms of COVID-19 that should be included."
+        }},
+        {{
+            "verdict": "no",
+            "reason": "While related to COVID-19, the origin of the first case is not relevant to listing the main symptoms."
+        }},
+        {{
+            "verdict": "no",
+            "reason": "A personal anecdote about a birthday party has no relevance to COVID-19 symptoms."
+        }}
+    ]
+}}
+
+Your task is to generate a verdict for each document in the retrieval context, so the number of 'verdicts' SHOULD BE EXACTLY EQUAL to that of the retrievalcontexts.
+
+==== YOUR TURN ====
+Input:
+{input}
+
+Expected output:
+{expected_output}
+
+Retrieval Context:
+{retrieval_context}
+
+JSON:
+"""
+
+    @staticmethod
+    def generate_reason(input, verdicts, score):
+        return f"""==== TASK INSTRUCTIONS ====\nYou will be provided with an input, retrieval contexts, and a contextual precision score. Your task is to provide a CLEAR and CONCISE reason for the score. 
+You should explain why the score is not higher, but also the current score is reasonable. Here's a further breakdown of the task:
+
+1. input (str) is a task or question that the model attempted to solve
+2. retrieval contexts (list[dict]) is a list of JSON with the following keys:
+- `verdict` (str): either 'yes' or 'no', which represents whether the corresponding document in the retrieval context is relevant to the input.
+- `reason` (str): a reason for the verdict.
+3. The contextual precision score is a float between 0 and 1 and represents if the relevant documents are ranked higher than irrelevant ones in the retrieval context. 
+The ranking can be inferred by the order of the retrieval documents: retrieval contexts is given IN THE ORDER OF THE DOCUMENT RANKINGS.
+This implies that the score will be higher if the relevant documents are ranked higher (appears earlier in the list) than irrelevant ones.
+
+==== FORMAT INSTRUCTIONS ====\nIMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason for the contextual precision score.
+Example JSON:
+{{
+    "reason": "The score is <contextual_precision_score> because <your_reason>."
+}}
+
+
+DO NOT mention 'verdict' in your reason, but instead phrase it as irrelevant nodes. The term 'verdict' are just here for you to understand the broader scope of things.
+Also DO NOT mention there are `reason` fields in the retrieval contexts you are presented with, instead just use the information in the `reason` field.
+In your reason, you MUST USE the `reason`, QUOTES in the 'reason', and the node RANK (starting from 1, eg. first node) to explain why the 'no' verdicts should be ranked lower than the 'yes' verdicts.
+When addressing nodes, make it explicit that it is nodes in retrieval context.
+If the score is 1, keep it short and say something positive with an upbeat tone (but don't overdo it otherwise it gets annoying).
+
+==== YOUR TURN ====
+Contextual Precision Score:
+{score}
+
+Input:
+{input}
+
+Retrieval Contexts:
+{verdicts}
+
+JSON:
+"""
+

judgeval/scorers/judgeval_scorers/local_implementations/contextual_recall/__init__.py

@@ -0,0 +1,3 @@
+from .contextual_recall_scorer import ContextualRecallScorer
+
+__all__ = ["ContextualRecallScorer"]

judgeval/scorers/judgeval_scorers/local_implementations/contextual_recall/contextual_recall_scorer.py

@@ -0,0 +1,249 @@
+from typing import Optional, List, Union
+
+from judgeval.scorers.utils import (
+    get_or_create_event_loop, 
+    parse_response_json, 
+    scorer_progress_meter, 
+    create_verbose_logs,
+    check_example_params
+)
+from judgeval.judges.utils import create_judge
+from judgeval.scorers import JudgevalScorer
+from judgeval.judges import JudgevalJudge
+from judgeval.judges.utils import create_judge
+from judgeval.data import Example, ExampleParams
+from judgeval.scorers.judgeval_scorers.local_implementations.contextual_recall.prompts import *
+
+required_params = [
+    ExampleParams.INPUT,
+    ExampleParams.ACTUAL_OUTPUT,
+    ExampleParams.EXPECTED_OUTPUT,
+    ExampleParams.RETRIEVAL_CONTEXT,
+]
+
+class ContextualRecallScorer(JudgevalScorer):
+    def __init__(
+        self,
+        threshold: float = 0.5,
+        model: Optional[Union[str, JudgevalJudge]] = None,
+        include_reason: bool = True,
+        async_mode: bool = True,
+        strict_mode: bool = False,
+        verbose_mode: bool = False,
+        user: Optional[str] = None
+    ):
+        self.user = user
+        self.threshold = 1 if strict_mode else threshold
+        self.model, self.using_native_model = create_judge(model)
+        self.evaluation_model = self.model.get_model_name()
+        self.include_reason = include_reason
+        self.async_mode = async_mode
+        self.strict_mode = strict_mode
+        self.verbose_mode = verbose_mode
+
+    def score_example(
+        self,
+        example: Example,
+        _show_indicator: bool = True,
+    ) -> float:
+        check_example_params(example, required_params, self)
+
+        with scorer_progress_meter(self, display_meter=_show_indicator):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                loop.run_until_complete(
+                    self.a_score_example(example, _show_indicator=False)
+                )
+            else:
+                self.verdicts: List[ContextualRecallVerdict] = (
+                    self._generate_verdicts(
+                        example.expected_output, example.retrieval_context
+                    )
+                )
+                self.score = self._calculate_score()
+                self.reason = self._generate_reason(example.expected_output)
+                self.success = self.score >= self.threshold
+                self.verbose_logs = create_verbose_logs(
+                    self,
+                    steps=[
+                        f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
+                        f"Score: {self.score}\nReason: {self.reason}",
+                    ],
+                )
+                return self.score
+
+    async def a_score_example(
+        self,
+        example: Example,
+        _show_indicator: bool = True,
+    ) -> float:
+        check_example_params(example, required_params, self)
+
+        with scorer_progress_meter(
+            self,
+            async_mode=True,
+            display_meter=_show_indicator,
+        ):
+            self.verdicts: List[ContextualRecallVerdict] = (
+                await self._a_generate_verdicts(
+                    example.expected_output, example.retrieval_context
+                )
+            )
+            self.score = self._calculate_score()
+            self.reason = await self._a_generate_reason(
+                example.expected_output
+            )
+            self.success = self.score >= self.threshold
+            self.verbose_logs = create_verbose_logs(
+                self,
+                steps=[
+                    f"Verdicts:\n{[v.model_dump() for v in self.verdicts]}",
+                    f"Score: {self.score}\nReason: {self.reason}",
+                ],
+            )
+            return self.score
+
+    async def _a_generate_reason(self, expected_output: str):
+        if self.include_reason is False:
+            return None
+
+        supportive_reasons = []
+        unsupportive_reasons = []
+        for idx, verdict in enumerate(self.verdicts):
+            if verdict.verdict.lower() == "yes":
+                supportive_reasons.append(f"Sentence {idx + 1}: {verdict.reason}")
+            else:
+                unsupportive_reasons.append(f"Sentence {idx + 1}: {verdict.reason}")
+
+        prompt = ContextualRecallTemplate.generate_reason(
+            expected_output=expected_output,
+            supportive_reasons=supportive_reasons,
+            unsupportive_reasons=unsupportive_reasons,
+            score=format(self.score, ".2f"),
+        )
+
+        if self.using_native_model:
+            res = await self.model.a_generate(prompt)
+            data = parse_response_json(res, self)
+            return data["reason"]
+        else:
+            try:
+                res: Reason = await self.model.a_generate(prompt, schema=Reason)
+                return res.reason
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = parse_response_json(res, self)
+                return data["reason"]
+
+    def _generate_reason(self, expected_output: str):
+        if self.include_reason is False:
+            return None
+
+        supportive_reasons = []
+        unsupportive_reasons = []
+        for verdict in self.verdicts:
+            if verdict.verdict.lower() == "yes":
+                supportive_reasons.append(verdict.reason)
+            else:
+                unsupportive_reasons.append(verdict.reason)
+
+        prompt = ContextualRecallTemplate.generate_reason(
+            expected_output=expected_output,
+            supportive_reasons=supportive_reasons,
+            unsupportive_reasons=unsupportive_reasons,
+            score=format(self.score, ".2f"),
+        )
+
+        if self.using_native_model:
+            res = self.model.generate(prompt)
+            data = parse_response_json(res, self)
+            return data["reason"]
+        else:
+            try:
+                res: Reason = self.model.generate(prompt, schema=Reason)
+                return res.reason
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = parse_response_json(res, self)
+                return data["reason"]
+
+    def _calculate_score(self):
+        number_of_verdicts = len(self.verdicts)
+        if number_of_verdicts == 0:
+            return 0
+
+        justified_sentences = 0
+        for verdict in self.verdicts:
+            if verdict.verdict.lower() == "yes":
+                justified_sentences += 1
+
+        score = justified_sentences / number_of_verdicts
+        return 0 if self.strict_mode and score < self.threshold else score
+
+    async def _a_generate_verdicts(
+        self, expected_output: str, retrieval_context: List[str]
+    ) -> List[ContextualRecallVerdict]:
+        prompt = ContextualRecallTemplate.generate_verdicts(
+            expected_output=expected_output, retrieval_context=retrieval_context
+        )
+        if self.using_native_model:
+            res = await self.model.a_generate(prompt)
+            data = parse_response_json(res, self)
+            verdicts = [
+                ContextualRecallVerdict(**item) for item in data["verdicts"]
+            ]
+            return verdicts
+        else:
+            try:
+                res: Verdicts = await self.model.a_generate(
+                    prompt, schema=Verdicts
+                )
+                verdicts: Verdicts = [item for item in res.verdicts]
+                return verdicts
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = parse_response_json(res, self)
+                verdicts = [
+                    ContextualRecallVerdict(**item) for item in data["verdicts"]
+                ]
+                return verdicts
+
+    def _generate_verdicts(
+        self, expected_output: str, retrieval_context: List[str]
+    ) -> List[ContextualRecallVerdict]:
+        prompt = ContextualRecallTemplate.generate_verdicts(
+            expected_output=expected_output, retrieval_context=retrieval_context
+        )
+        if self.using_native_model:
+            res = self.model.generate(prompt)
+            data = parse_response_json(res, self)
+            verdicts = [
+                ContextualRecallVerdict(**item) for item in data["verdicts"]
+            ]
+            return verdicts
+        else:
+            try:
+                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
+                verdicts: Verdicts = [item for item in res.verdicts]
+                return verdicts
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = parse_response_json(res, self)
+                verdicts = [
+                    ContextualRecallVerdict(**item) for item in data["verdicts"]
+                ]
+                return verdicts
+
+    def _success_check(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Contextual Recall"