judgeval 0.0.32__py3-none-any.whl → 0.0.33__py3-none-any.whl

judgeval/scorers/judgeval_scorers/local_implementations/instruction_adherence/instruction_adherence.py

@@ -1,232 +0,0 @@
-from typing import Optional, List, Union, Tuple
-from pydantic import BaseModel
-
-from judgeval.constants import APIScorer
-from judgeval.scorers.utils import (get_or_create_event_loop,
-                                    scorer_progress_meter,
-                                    create_verbose_logs,
-                                    parse_response_json,
-                                    check_example_params
-                                    )
-from judgeval.scorers import JudgevalScorer
-from judgeval.judges import JudgevalJudge
-from judgeval.judges.utils import create_judge
-from judgeval.data import Example, ExampleParams
-from judgeval.scorers.judgeval_scorers.local_implementations.instruction_adherence.prompt import (
-    InstructionAdherenceTemplate,
-)
-required_params = [
-    ExampleParams.INPUT,
-    ExampleParams.ACTUAL_OUTPUT,
-]
-
-class Instructions(BaseModel):
-    instructions: List[str]
-
-class Verdict(BaseModel):
-    instruction: str
-    score: float
-    reason: str
-
-class ListOfVerdicts(BaseModel):
-    verdicts: List[Verdict]
-
-class InstructionAdherenceScorer(JudgevalScorer):
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        model: Optional[Union[str, JudgevalJudge]] = None,
-        include_reason: bool = True,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-    ):
-        super().__init__(
-            score_type=APIScorer.INSTRUCTION_ADHERENCE,
-            threshold=1 if strict_mode else threshold,
-            evaluation_model=None,
-            include_reason=include_reason,
-            async_mode=async_mode,
-            strict_mode=strict_mode,
-            verbose_mode=verbose_mode
-        )
-        self.model, self.using_native_model = create_judge(model)
-        self.evaluation_model = self.model.get_model_name()
-
-    def score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-
-        with scorer_progress_meter(self, display_meter=_show_indicator):
-            try:
-                if self.async_mode:
-                    loop = get_or_create_event_loop()
-                    loop.run_until_complete(
-                        self.a_score_example(example, _show_indicator=False)
-                    )
-                else:
-                    self.instructions: List[str] = self._get_instructions(example.input)
-                    self.verdicts: List[Verdict] = (
-                        self._get_verdicts(self.instructions, example.actual_output)
-                    )
-                    self.score = self._compute_score()
-                    self.reason = str(self.verdicts)
-                    self.success = self.score >= self.threshold
-                    self.verbose_logs = create_verbose_logs(
-                        self,
-                        steps=[
-                            f"Instructions:\n{self.instructions}",
-                            f"Score: {self.score}\nReason: {self.reason}",
-                        ],
-                    )
-                    return self.score
-            except Exception as e:
-                raise
-
-    async def a_score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        check_example_params(example, required_params, self)
-        try:
-            with scorer_progress_meter(
-                self, async_mode=True, display_meter=_show_indicator
-            ):
-                self.instructions: List[str] = await self._a_get_instructions(example.input)
-                self.verdicts: List[Verdict] = (
-                    await self._a_get_verdicts(self.instructions, example.actual_output)
-                )
-                self.score = self._compute_score()
-                self.reason = str(self.verdicts)
-                self.success = self.score >= self.threshold
-                self.verbose_logs = create_verbose_logs(
-                    self,
-                    steps=[
-                        f"Instructions:\n{self.instructions}",
-                        f"Score: {self.score}\nReason: {self.reason}",
-                    ],
-                )
-                return self.score
-        except Exception as e:
-            raise e
-
-
-    async def _a_get_verdicts(
-        self, instructions: List[str], actual_output: str
-    ) -> List[Verdict]:
-        if len(instructions) == 0:
-            return []
-
-        prompt = InstructionAdherenceTemplate.generate_verdicts(
-            instructions=instructions,
-            actual_output=actual_output,
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return [
-                Verdict(**item) for item in data["verdicts"]
-            ]
-        else:
-            try:
-                res: List[Verdict] = await self.model.a_generate(
-                    prompt, schema=List[Verdict]
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return [
-                    Verdict(**item) for item in data["verdicts"]
-                ]
-
-    def _get_verdicts(self, instructions: List[str], actual_output: str) -> List[Verdict]:
-        if len(instructions) == 0:
-            return []
-
-        prompt = InstructionAdherenceTemplate.generate_verdicts(
-            instructions=instructions,
-            actual_output=actual_output,
-        )
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return [Verdict(**item) for item in data["verdicts"]]
-        else:
-            try:
-                res: List[Verdict] = self.model.generate(prompt, schema=List[Verdict])
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return [
-                    Verdict(**item) for item in data["verdicts"]
-                ]
-
-    async def _a_get_instructions(
-        self,
-        input: str,
-    ) -> List[str]:
-        prompt = InstructionAdherenceTemplate.get_instructions(
-            input=input,
-        )
-        if self.using_native_model:
-            res = await self.model.a_generate(prompt)
-            data = parse_response_json(res, self)
-            return data["instructions"]
-        else:
-            try:
-                res: List[str] = await self.model.a_generate(
-                    prompt, schema=List[str]
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = parse_response_json(res, self)
-                return data["instructions"]
-
-    def _get_instructions(
-        self,
-        input: str,
-    ) -> List[str]:
-        prompt = InstructionAdherenceTemplate.get_instructions(
-            input=input,
-        )
-        if self.using_native_model:
-            res = self.model.generate(prompt)
-            data = parse_response_json(res, self)
-            return data["instructions"]
-        else:
-            try:
-                res: List[str] = self.model.generate(prompt, schema=List[str])
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = parse_response_json(res, self)
-                return data["instructions"]
-
-    def _compute_score(self):
-        if len(self.verdicts) == 0:
-            return 1
-        score = 0
-        for verdict in self.verdicts:
-            score += verdict.score
-        return score / len(self.verdicts)
-
-    def success_check(self) -> bool:
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score >= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Instruction Adherence"

judgeval/scorers/judgeval_scorers/local_implementations/instruction_adherence/prompt.py

@@ -1,102 +0,0 @@
-"""
-Util prompts for InstructionAdherenceScorer
-"""
-
-from typing import List, Optional, Tuple
-from pydantic import BaseModel, Field
-
-
-class InstructionAdherenceTemplate:
-    @staticmethod
-    def get_instructions(input):
-        return f"""You will be presented with a piece of text. Your task is to break down the text and generate a list of the instructions contained within the text.
-
-===== START OF EXAMPLES =====
-Example 1:
-Example text: Hello my name is John Doe. I like cars. Write two poems about the weather and create a joke. Also what is 5 + 5?
-
-Output:
-{{
-    "instructions": ["Write two poem about the weather", "Create a joke", "What is 5 + 5?"]
-}}
-===== END OF EXAMPLES =====
-
-        
-**
-IMPORTANT: Please return your answer in valid JSON format, with the "instructions" key mapping to a list of strings. No words or explanation is needed.
-**
-
-==== START OF INPUT ====
-Text:
-{input}
-==== END OF INPUT ====
-
-==== YOUR ANSWER ====
-JSON:
-"""
-
-    @staticmethod
-    def generate_verdicts(instructions, actual_output):
-        return f"""
-        You will be presented with a list of instructions and a piece of text. For each instruction, determine if the instruction was completed in the text. There are 3 categories: either completed, partially completed, or not completed. The scores for these will be 1, 0.5, and 0 respectively.
-        Go through each instruction and provide score for each instruction as well as the reasoning for that score.
-
-        ==== FORMATTING YOUR ANSWER ====
-        Please return your answer in JSON format, with a list of JSON objects with keys "instruction", "score", and "reason". No words or explanation beyond the output JSON is needed.
-
-
-        ===== START OF EXAMPLES =====
-        Example 1:
-        instructions: ["Write two poems about the weather", "Create a joke", "What is 5 + 5?"]
-        output: Poem 1: The Sun's Embrace
-        The sun climbs high, a golden flame,
-        It whispers warmth, it calls my name.
-        The sky, a canvas, blue and clear,
-        A perfect day for cars, my dear.
-
-        The asphalt hums beneath the wheels,
-        A symphony of speed it feels.
-        The weather smiles, no clouds in sight,
-        A driver's joy, pure delight.
-
-        Poem 2: The Storm's Dance
-        A sunlit meadow, alive with whispers of wind, where daisies dance and hope begins again. Each petal holds a promise—bright, unbruised— a symphony of light that cannot be refused.
-
-        Joke
-        Why dont cars ever get cold in the winter?
-        Because they have radiators!
-
-        Math Answer
-        5 + 5 = 10
-        
-        YOUR JSON OUTPUT:
-        {{
-            [
-                {{
-                    "instruction": "Write two poem about the weather",
-                    "score": 0.5,
-                    "reason": "The output contained one poem about the weather, but the other poem was not about the weather."
-                }},
-                {{
-                    "instruction": "Create a joke",
-                    "score": 1,
-                    "reason": "There was a joke created in the output."
-                }},
-                {{
-                    "instruction": "What is 5 + 5?",
-                    "score": 1,
-                    "reason": "The answer to the math question was provided in the output."
-                }}
-            ]
-        }}
-        ===== END OF EXAMPLES =====
-        
-        ==== START OF INPUT ====
-        instructions: {instructions}
-        output: {actual_output}
-        ==== END OF INPUT ====
-
-        ==== YOUR ANSWER ====
-        JSON:
-        """
-

judgeval/scorers/judgeval_scorers/local_implementations/json_correctness/__init__.py

@@ -1,5 +0,0 @@
-from judgeval.scorers.judgeval_scorers.local_implementations.json_correctness.json_correctness_scorer import JsonCorrectnessScorer
-
-__all__ = [
-    "JsonCorrectnessScorer",
-]

judgeval/scorers/judgeval_scorers/local_implementations/json_correctness/json_correctness_scorer.py

@@ -1,134 +0,0 @@
-from typing import Optional, Union, Any
-from pydantic import BaseModel, ValidationError, create_model
-
-from judgeval.constants import APIScorer
-from judgeval.judges import JudgevalJudge
-from judgeval.judges.utils import create_judge
-from judgeval.scorers.utils import (
-    get_or_create_event_loop,
-    scorer_progress_meter,
-    create_verbose_logs,
-    check_example_params
-)
-from judgeval.scorers import JudgevalScorer
-from judgeval.data import Example, ExampleParams
-
-
-required_params = [
-    ExampleParams.INPUT,
-    ExampleParams.ACTUAL_OUTPUT
-]
-
-
-class JsonCorrectnessScorer(JudgevalScorer):
-
-    def __init__(
-        self,
-        json_schema: Union[BaseModel, dict],
-        model: Optional[Union[str, JudgevalJudge]] = None,
-        threshold: float = 0.5,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-        user: Optional[str] = None
-    ):
-        super().__init__(
-            score_type=APIScorer.JSON_CORRECTNESS,
-            threshold=1 if strict_mode else threshold,
-            evaluation_model=None,
-            include_reason=False,
-            async_mode=async_mode,
-            strict_mode=strict_mode,
-            verbose_mode=verbose_mode
-        )
-        self.user = user
-        self.model, self.using_native_model = create_judge(model)
-        self.evaluation_model = self.model.get_model_name()
-
-        if isinstance(json_schema, dict):
-            # Convert to BaseModel
-            fields = {
-                key: (str if prop["type"] == "string" else Any, ...)
-                for key, prop in json_schema["properties"].items()
-            }
-
-            # Dynamically create the model
-            DynamicModel = create_model(json_schema["title"], **fields)
-
-            self.json_schema = DynamicModel
-        else:
-            self.json_schema = json_schema
-
-    def score_example(self, example: Example, _show_indicator: bool = True) -> float:
-        check_example_params(example, required_params, self)
-        with scorer_progress_meter(
-            self,
-            async_mode=self.async_mode,
-            display_meter=_show_indicator,
-        ):
-            if self.async_mode:
-                loop = get_or_create_event_loop()
-                loop.run_until_complete(
-                    self.a_measure(example, _show_indicator=False)
-                )
-            else:
-                valid_json = True
-                try:
-                    self.json_schema.model_validate_json(
-                        example.actual_output
-                    )
-                except ValidationError as e:
-                    valid_json = False
-
-                self.score = 1.0 if valid_json else 0
-                self.success = self.score >= self.threshold
-                self.verbose_logs = create_verbose_logs(
-                    self,
-                    steps=[
-                        f"LLM outputed Json:\n{example.actual_output}",
-                        f"Score: {self.score}",
-                    ],
-                )
-
-                return self.score
-
-    async def a_score_example(self, example: Example, _show_indicator: bool = True) -> float:
-        check_example_params(example, required_params, self)
-        with scorer_progress_meter(
-            self,
-            async_mode=self.async_mode,
-            display_meter=_show_indicator,
-        ):
-            valid_json = True
-            try:
-                self.json_schema.model_validate_json(
-                    example.actual_output
-                )
-            except ValidationError as e:
-                valid_json = False
-
-            self.score = 1.0 if valid_json else 0
-            self.success = self.score >= self.threshold
-            self.verbose_logs = create_verbose_logs(
-                self,
-                steps=[
-                    f"LLM outputed Json:\n{example.actual_output}",
-                    f"Score: {self.score}",
-                ],
-            )
-            return self.score
-
-    def _success_check(self):
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score >= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "JSON Correctness"
-    

judgeval/scorers/judgeval_scorers/local_implementations/summarization/__init__.py

@@ -1,3 +0,0 @@
-from .summarization_scorer import SummarizationScorer
-
-__all__ = ["SummarizationScorer"]