judgeval 0.0.32__py3-none-any.whl → 0.0.33__py3-none-any.whl

judgeval/scorers/judgeval_scorers/local_implementations/contextual_relevancy/prompts.py

@@ -1,121 +0,0 @@
-from typing import List
-from pydantic import BaseModel
-
-
-class ContextualRelevancyVerdict(BaseModel):
-    statement: str
-    verdict: str
-    reason: str
-
-
-class ContextualRelevancyVerdicts(BaseModel):
-    verdicts: List[ContextualRelevancyVerdict]
-
-
-class Reason(BaseModel):
-    reason: str
-
-
-class ContextualRelevancyTemplate:
-
-    @staticmethod
-    def generate_verdicts(input: str, context: str):
-        return f"""==== TASK INSTRUCTIONS ====
-You will be provided with an input (str) and a context (str). The input is a question/task proposed to a language model and the context is a list of documents retrieved in a RAG pipeline.
-Your task is to determine whether each statement found in the context is relevant to the input. To do so, break down the context into statements (high level pieces of information), then determine whether each statement is relevant to the input.
-
-==== FORMATTING YOUR ANSWER ====
-
-You should format your answer as a list of JSON objects, with each JSON object containing the following fields:
-- 'verdict': a string that is EXACTLY EITHER 'yes' or 'no', indicating whether the statement is relevant to the input
-- 'statement': a string that is the statement found in the context
-- 'reason': an string that is the justification for why the statement is relevant to the input. IF your verdict is 'no', you MUST quote the irrelevant parts of the statement to back up your reason.
-
-IMPORTANT: Please make sure to only return in JSON format.
-
-==== EXAMPLE ====
-Example Context: "Einstein won the Nobel Prize for his discovery of the photoelectric effect. He won the Nobel Prize in 1968. There was a cat."
-Example Input: "What were some of Einstein's achievements?"
-
-Example:
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes",
-            "statement": "Einstein won the Nobel Prize for his discovery of the photoelectric effect in 1968",
-        }},
-        {{
-            "verdict": "no",
-            "statement": "There was a cat.",
-            "reason": "The retrieval context contained the information 'There was a cat' when it has nothing to do with Einstein's achievements."
-        }}
-    ]
-}}
-
-==== YOUR TURN ====
-
-Input:
-{input}
-
-Context:
-{context}
-
-JSON:
-"""
-    
-    @staticmethod
-    def generate_reason(
-        input: str,
-        irrelevancies: List[str],
-        relevant_statements: List[str],
-        score: float,
-    ):
-        return f"""==== TASK INSTRUCTIONS ====
-You will be provided with the following information:
-- An input to a RAG pipeline which is a question/task. There is an associated retrieval context to this input in the RAG pipeline (the context is not provided but is relevant to your task).
-- A list of irrelevant statements from the retrieval context. These statements are not relevant to the input query.
-- A list of relevant statements from the retrieval context. These statements are relevant to the input query.
-- A contextual relevancy score (the closer to 1 the better). Contextual relevancy is a measurement of how relevant the retrieval context is to the input query.
-
-Your task is to generate a CLEAR and CONCISE reason for the score. You should quote data provided in the reasons for the irrelevant and relevant statements to support your reason.
-
-==== FORMATTING YOUR ANSWER ====
-IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-Example JSON:
-{{
-    "reason": "The score is <contextual_relevancy_score> because <your_reason>."
-}}
-
-If the score is 1, keep it short and say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
-
-==== EXAMPLE ====
-Input: "What is the capital of France?"
-
-Contextual Relevancy Score: 0.67
-
-Irrelevant Statements from the retrieval context:
-[{{"statement": "Flights to Paris are available from San Francisco starting at $1000", "reason": "Flight prices and routes are not relevant to identifying the capital of France"}}]
-
-Relevant Statements from the retrieval context:
-[{{"statement": "Paris is the capital of France"}}, {{"statement": "Paris is a major European city"}}]
-
-Example Response:
-{{
-    "reason": "The score is 0.67 because while the context contains directly relevant information stating that 'Paris is the capital of France', it also includes irrelevant travel information about flight prices from San Francisco."
-}}
-
-==== YOUR TURN ====
-Contextual Relevancy Score:
-{score}
-
-Input:
-{input}
-
-Irrelevant Statements from the retrieval context:
-{irrelevancies}
-
-Relevant Statements from the retrieval context:
-{relevant_statements}
-
-JSON:
-"""

judgeval/scorers/judgeval_scorers/local_implementations/execution_order/__init__.py

@@ -1,3 +0,0 @@
-from judgeval.scorers.judgeval_scorers.local_implementations.execution_order.execution_order import ExecutionOrderScorer
-
-__all__ = ["ExecutionOrderScorer"]

judgeval/scorers/judgeval_scorers/local_implementations/execution_order/execution_order.py

@@ -1,156 +0,0 @@
-from typing import List
-
-from judgeval.constants import APIScorer
-from judgeval.scorers.utils import (
-    scorer_progress_meter,
-    create_verbose_logs,
-    check_example_params
-)
-from judgeval.data import Example, ExampleParams
-from judgeval.scorers import JudgevalScorer
-
-
-required_params = [
-    ExampleParams.INPUT,
-    ExampleParams.ACTUAL_OUTPUT,
-    ExampleParams.EXPECTED_TOOLS,
-    ExampleParams.TOOLS_CALLED,
-]
-
-
-def get_lcs(seq1, seq2):
-    m, n = len(seq1), len(seq2)
-    dp = [[0] * (n + 1) for _ in range(m + 1)]
-
-    for i in range(1, m + 1):
-        for j in range(1, n + 1):
-            if seq1[i - 1] == seq2[j - 1]:
-                dp[i][j] = dp[i - 1][j - 1] + 1
-            else:
-                dp[i][j] = max(dp[i - 1][j], dp[i][j - 1])
-
-    # Reconstruct the LCS
-    lcs = []
-    i, j = m, n
-    while i > 0 and j > 0:
-        if seq1[i - 1] == seq2[j - 1]:
-            lcs.append(seq1[i - 1])
-            i -= 1
-            j -= 1
-        elif dp[i - 1][j] > dp[i][j - 1]:
-            i -= 1
-        else:
-            j -= 1
-
-    return lcs[::-1]
-
-
-class ExecutionOrderScorer(JudgevalScorer):
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        include_reason: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-        should_exact_match: bool = False,
-        should_consider_ordering: bool = False,
-    ):
-        super().__init__(
-            score_type=APIScorer.EXECUTION_ORDER,
-            threshold=1 if strict_mode else threshold,
-            evaluation_model=None,
-            include_reason=include_reason,
-            async_mode=False,
-            strict_mode=strict_mode,
-            verbose_mode=verbose_mode
-        )
-        self.should_exact_match = should_exact_match
-        self.should_consider_ordering = should_consider_ordering
-
-    def measure(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(self, display_meter=_show_indicator):
-            self.tools_called: List[str] = example.tools_called
-            self.expected_tools: List[str] = example.expected_tools
-            self.score = self._calculate_score()
-            self.reason = self._generate_reason()
-            self.success = self.score >= self.threshold
-            self.verbose_logs = create_verbose_logs(
-                self,
-                steps=[
-                    f"Expected Tools:\n{self.expected_tools}",
-                    f"Tools Called:\n{self.tools_called}",
-                    f"Score: {self.score}\nReason: {self.reason}",
-                ],
-            )
-            return self.score
-
-    async def a_measure(
-        self, test_case: Example, _show_indicator: bool = True
-    ) -> float:
-        check_example_params(test_case, required_params, self)
-        return self.measure(test_case, _show_indicator=_show_indicator)
-
-    def _generate_reason(self):
-        if self.should_exact_match:
-            return f"{'Exact match' if self.tools_called == self.expected_tools else 'Not an exact match'}: expected {self.expected_tools}, called {self.tools_called}."
-
-        elif self.should_consider_ordering:
-            lcs = get_lcs(self.expected_tools, self.tools_called)
-            missing = set(self.expected_tools) - set(self.tools_called)
-            out_of_order = set(self.expected_tools) - set(lcs)
-
-            if len(lcs) == len(self.expected_tools):
-                return f"Correct ordering: all expected tools {self.expected_tools} were called in the correct order."
-            else:
-                issues = []
-                if missing:
-                    issues.append(f"missing tools {list(missing)}")
-                if out_of_order:
-                    issues.append(f"out-of-order tools {list(out_of_order)}")
-                return f"Incorrect tool usage: {' and '.join(issues)}; expected {self.expected_tools}, called {self.tools_called}."
-
-        else:
-            used_expected = set(self.tools_called).intersection(
-                set(self.expected_tools)
-            )
-            missing = set(self.expected_tools) - used_expected
-
-            if len(used_expected) == len(self.expected_tools):
-                return f"All expected tools {self.expected_tools} were called (order not considered)."
-            else:
-                return f"Incomplete tool usage: missing tools {list(missing)}; expected {self.expected_tools}, called {self.tools_called}."
-
-    def _calculate_score(self):
-        if self.should_exact_match:
-            return 1.0 if self.tools_called == self.expected_tools else 0.0
-
-        elif self.should_consider_ordering:
-            longest_common_subsequence = get_lcs(
-                self.expected_tools, self.tools_called
-            )
-            score = len(longest_common_subsequence) / len(self.expected_tools)
-
-        else:
-            used_expected_tools = set(self.tools_called).intersection(
-                set(self.expected_tools)
-            )
-            score = len(used_expected_tools) / len(self.expected_tools)
-        return 0 if self.strict_mode and score < self.threshold else score
-
-    def _success_check(self) -> bool:
-        try:
-            self.success = self.score >= self.threshold
-        except:
-            self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Execution Order"
-    

judgeval/scorers/judgeval_scorers/local_implementations/faithfulness/__init__.py

@@ -1,3 +0,0 @@
-from judgeval.scorers.judgeval_scorers.local_implementations.faithfulness.faithfulness_scorer import FaithfulnessScorer
-
-__all__ = ["FaithfulnessScorer"]

judgeval/scorers/judgeval_scorers/local_implementations/faithfulness/faithfulness_scorer.py

@@ -1,318 +0,0 @@
-"""
-Code for the local implementation of the Faithfulness metric.
-"""
-from typing import List, Optional, Union
-from judgeval.constants import APIScorer
-from judgeval.data import (
-    Example, 
-    ExampleParams
-)
-from judgeval.scorers import JudgevalScorer
-from judgeval.scorers.utils import (
-    get_or_create_event_loop, 
-    check_example_params
-)
-from judgeval.judges.utils import create_judge
-from judgeval.judges import JudgevalJudge
-from judgeval.scorers.utils import (
-    scorer_progress_meter, 
-    create_verbose_logs, 
-    parse_response_json
-)
-from judgeval.scorers.judgeval_scorers.local_implementations.faithfulness.prompts import *
-
-
-required_params = [
-    ExampleParams.INPUT,
-    ExampleParams.ACTUAL_OUTPUT,
-    ExampleParams.RETRIEVAL_CONTEXT,
-]
-
-
-class FaithfulnessScorer(JudgevalScorer):
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        model: Optional[Union[str, JudgevalJudge]] = None,
-        include_reason: bool = True,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-        user: Optional[str] = None
-    ):
-        super().__init__(
-            score_type=APIScorer.FAITHFULNESS,
-            threshold=1 if strict_mode else threshold,
-            evaluation_model=None,
-            include_reason=include_reason,
-            async_mode=async_mode,
-            strict_mode=strict_mode,
-            verbose_mode=verbose_mode
-        )
-        self.user = user
-        self.model, self.using_native_model = create_judge(model)
-        self.using_native_model = True  # NOTE: SETTING THIS FOR LITELLM and TOGETHER usage
-        self.evaluation_model = self.model.get_model_name()
-
-    def score_example(
-        self,
-        example: Example,
-        all_claims: bool = False,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-        
-        with scorer_progress_meter(self, display_meter=_show_indicator):
-            if self.async_mode:
-                loop = get_or_create_event_loop()
-                loop.run_until_complete(
-                    self.a_score_example(
-                        example, 
-                        all_claims=all_claims, 
-                        _show_indicator=False
-                    )
-                )
-            else:
-                self.claims = self._generate_claims(example.actual_output, all_claims=all_claims)
-                if self.additional_metadata is None:
-                    self.additional_metadata = {}
-                self.additional_metadata["claims"] = self.claims  # Add claims generated to metadata
-
-                self.verdicts = self._generate_verdicts(example.retrieval_context)
-                self.additional_metadata["verdicts"] = [v.model_dump() for v in self.verdicts]  # Add verdicts generated to metadata
-                
-                self.score = self._calculate_score()
-                self.reason = self._generate_reason()
-                self.success = self.score >= self.threshold
-                self.verbose_logs = create_verbose_logs(
-                    self,
-                    steps=[
-                        f"Claims:\n{self.claims}",
-                        f"Verdicts:\n{self.verdicts}",
-                        f"Score: {self.score}\nReason: {self.reason}",
-                    ],
-                )
-
-                return self.score
-
-    async def a_score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(
-            self, async_mode=True, display_meter=_show_indicator
-        ):
-            self.claims = await self._a_generate_claims(example.actual_output)
-
-
-            if self.additional_metadata is None:
-                self.additional_metadata = {}
-            self.additional_metadata["claims"] = self.claims
-
-            self.verdicts = await self._a_generate_verdicts(example.retrieval_context)  
-
-            self.additional_metadata["verdicts"] = [v.model_dump() for v in self.verdicts]  # Add verdicts generated to metadata
-
-            self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason()
-            self.success = self.score >= self.threshold
-            self.verbose_logs = create_verbose_logs(
-                self,
-                steps=[
-                    f"Claims:\n{self.claims}",
-                    f"Verdicts:\n{self.verdicts}",
-                    f"Score: {self.score}\nReason: {self.reason}",
-                ],
-            )
-
-            return self.score
-
-    async def _a_generate_reason(self) -> str:
-        if self.include_reason is False:
-            return None
-
-        contradictions = []
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() == "no":
-                contradictions.append(verdict.model_dump())
-
-        prompt: dict = FaithfulnessTemplate.justify_reason(
-            contradictions=contradictions,
-            score=format(self.score, ".2f"),
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = await self.model.a_generate(prompt, schema=Reason)
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    def _generate_reason(self) -> str:
-        if self.include_reason is False:
-            return None
-
-        contradictions = []
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() == "no":
-                contradictions.append(verdict.reason)
-
-        prompt: dict = FaithfulnessTemplate.justify_reason(
-            contradictions=contradictions,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: Reason = self.model.generate(prompt, schema=Reason)
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return data["reason"]
-
-    async def _a_generate_verdicts(self, retrieval_context: str) -> List[FaithfulnessVerdict]:
-        if len(self.claims) == 0:
-            return []
-
-        verdicts: List[FaithfulnessVerdict] = []
-
-        claims = [
-            claim["claim"] for claim in self.claims
-        ]  # We only need the claims, not the quotes involved
-
-        prompt = FaithfulnessTemplate.create_verdicts(
-            claims=claims,
-            retrieval_context=retrieval_context,
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            verdicts = [
-                FaithfulnessVerdict(**item) for item in data["verdicts"]
-            ]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                verdicts = [
-                    FaithfulnessVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def _generate_verdicts(self, retrieval_context: str) -> List[FaithfulnessVerdict]:
-        if len(self.claims) == 0:
-            return []
-
-        verdicts: List[FaithfulnessVerdict] = []
-
-        claims = [
-            claim["claim"] for claim in self.claims
-        ]  # We only need the claims, not the quotes involved
-
-        prompt = FaithfulnessTemplate.create_verdicts(
-            claims=claims,
-            retrieval_context=retrieval_context,
-        )
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            verdicts = [
-                FaithfulnessVerdict(**item) for item in data["verdicts"]
-            ]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                verdicts = [
-                    FaithfulnessVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
-
-    async def _a_generate_claims(self, actual_output: str) -> List[str]:
-        prompt = FaithfulnessTemplate.find_claims(text=actual_output)
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return data["claims"]
-        else:
-            try:
-                res: Claims = await self.model.a_generate(prompt, schema=Claims)
-                return res.claims
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return data["claims"]
-
-    def _generate_claims(self, actual_output: str, all_claims: bool = False) -> List[str]:
-        prompt = FaithfulnessTemplate.find_claims(text=actual_output)
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return data["claims"]
-        else:
-            try:
-                res: Claims = self.model.generate(prompt, schema=Claims)
-                return res.claims
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return data["claims"]
-
-    def _calculate_score(self) -> float:
-        number_of_verdicts = len(self.verdicts)
-        if number_of_verdicts == 0:
-            return 1
-
-        faithfulness_count = 0
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() != "no":
-                faithfulness_count += 1
-
-        score = faithfulness_count / number_of_verdicts
-        return 0 if self.strict_mode and score < self.threshold else score
-
-    def _success_check(self) -> bool:
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score >= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    def get_claims(self):
-        return self.claims
-    
-    def get_verdicts(self):
-        return self.verdicts
-
-    @property
-    def __name__(self):
-        return "Faithfulness"
-