judgeval 0.0.52__py3-none-any.whl → 0.0.54__py3-none-any.whl

judgeval/scorers/prompt_scorer.py

@@ -1,296 +0,0 @@
-"""
-Code that implements a prompt-based scorer for evaluating examples.
-
-The PromptScorer class is a base class that can be used to create custom scoring metrics using LLM prompts.
-To implement a subclass of PromptScorer, you need to implement the following methods:
-- build_measure_prompt(): builds the conversation prompt that is sent to the LLM judge
-- build_schema(): defines the expected response schema from the LLM
-- process_response(): parses the response from the LLM judge
-- success_check(): determines whether the evaluation was successful
-
-The core idea of PromptScorer is to provide a flexible way to create custom scoring metrics
-by leveraging LLM judges to evaluate examples. The scorer constructs a prompt, sends it to
-the judge, and parses the structured response to determine a score.
-
-For example, the SentimentScorer subclass uses PromptScorer to detect negative sentiment in responses
-by prompting an LLM to rate the negativity on a 1-5 scale and provide a reason for the rating.
-
-The PromptScorer supports both synchronous and asynchronous evaluation modes, includes optional
-reason fields in responses, and can operate in strict mode with higher thresholds.
-
-NOTE: When implementing build_measure_prompt and build_schema:
-- The prompt should guide the LLM to generate a response matching your schema
-- The schema should include "score" and optionally "reason" fields
-- The score field type and range should match your scoring criteria
-- The reason field provides explanatory context for the score
-"""
-
-from abc import abstractmethod
-from typing import List, Optional, Tuple, Any
-from pydantic import BaseModel, Field
-
-from judgeval.data import Example
-from judgeval.data.example import ExampleParams
-from judgeval.scorers import JudgevalScorer
-from judgeval.scorers.utils import (
-    scorer_progress_meter,
-    parse_response_json,
-    get_or_create_event_loop,
-    create_verbose_logs,
-)
-from judgeval.judges import JudgevalJudge
-
-
-class ReasonScore(BaseModel):
-    reason: str
-    score: float
-
-
-class PromptScorer(JudgevalScorer, BaseModel):
-    name: str
-    score_type: str
-    threshold: float = Field(default=0.5)
-    using_native_model: bool = Field(default=True)
-    model: Optional[JudgevalJudge] = Field(default=None)
-    skipped: bool = Field(default=False)
-    # DO NOT SET THESE FIELDS MANUALLY, THEY ARE SET BY THE SCORE_EXAMPLE METHOD
-    _response: Optional[dict] = None
-    _result: Optional[float] = None
-
-    def __init__(
-        self,
-        name: str,
-        threshold: float = 0.5,
-        include_reason: bool = True,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-        required_params: Optional[List[ExampleParams]] = None,
-    ):
-        # Initialize BaseModel first
-        BaseModel.__init__(
-            self,
-            name=name,
-            score_type=name,
-            threshold=1 if strict_mode else threshold,
-            include_reason=include_reason,
-            async_mode=async_mode,
-            strict_mode=strict_mode,
-            verbose_mode=verbose_mode,
-        )
-        # Then initialize JudgevalScorer
-        JudgevalScorer.__init__(
-            self,
-            score_type=name,
-            threshold=1 if strict_mode else threshold,
-            include_reason=include_reason,
-            async_mode=async_mode,
-            strict_mode=strict_mode,
-            verbose_mode=verbose_mode,
-            required_params=required_params,
-        )
-
-    def score_example(
-        self, example: Example, _show_indicator: bool = True
-    ) -> float | None:
-        """
-        Synchronous method for scoring an example using the prompt criteria.
-        """
-        with scorer_progress_meter(self, display_meter=_show_indicator):
-            if self.async_mode:
-                loop = get_or_create_event_loop()
-                loop.run_until_complete(
-                    self.a_score_example(example, _show_indicator=False)
-                )
-                return self._result
-            else:
-                result, reason = self.evaluate(example)
-                self.reason = reason
-                self._result = result
-                self.verbose_logs = create_verbose_logs(
-                    self,
-                    steps=[
-                        f"Results: {self._result}\nReason: {self.reason}",
-                    ],
-                )
-                return result
-
-    async def a_score_example(
-        self,
-        example: Example,
-        _show_indicator: bool = True,
-    ) -> float:
-        """
-        Async method for scoring an example using the prompt criteria.
-        """
-        with scorer_progress_meter(self, display_meter=_show_indicator):
-            result, reason = await self.a_evaluate(example)
-            self.reason = reason
-            self._result = result
-            self.verbose_logs = create_verbose_logs(
-                self,
-                steps=[
-                    f"Results: {self._result}\nReason: {self.reason}",
-                ],
-            )
-            return result
-
-    def evaluate(self, example: Example) -> Tuple[Any, str]:
-        """
-        Synchronous helper method for evaluating an example using the prompt criteria.
-
-        Builds a custom prompt using `build_measure_prompt` and sends it to the judge model
-        for evaluation. The result is then parsed as JSON and returned.
-
-        NOTE: It is assumed that the model response will be JSON and contain a "score" and "reason" field.
-        """
-        prompt = self._build_measure_prompt(example)
-        if self.using_native_model and self.model:
-            res = self.model.generate(prompt)
-            response = parse_response_json(res, self)
-            result, reason = self._process_response(response)
-            return result, reason
-        else:
-            raise NotImplementedError(
-                "Non-native judge models are not supported in synchronous mode yet."
-            )
-
-    async def a_evaluate(self, example: Example) -> Tuple[Any, str]:
-        """
-        Asynchronous helper method for evaluating an example using the prompt criteria.
-
-        Builds a custom prompt using `build_measure_prompt` and sends it to the judge model
-        for evaluation. The result is then parsed as JSON and returned.
-
-        NOTE: It is assumed that the model response will be JSON and contain a "score" and "reason" field.
-        """
-        judge_prompt = self._build_measure_prompt(example)
-        schema = self._build_schema()
-        prompt = self._enforce_prompt_format(judge_prompt=judge_prompt, schema=schema)
-        if self.using_native_model and self.model:
-            res = await self.model.a_generate(prompt)
-            response = parse_response_json(res, self)
-            self._response = response
-
-            result, reason = self._process_response(response)
-            self.score = result
-            self.reason = reason
-            self._response = response
-            return result, reason
-        else:
-            raise NotImplementedError(
-                "Non-native judge models are not supported in async mode yet."
-            )
-
-    # TODO: can we make this take *args and **kwargs? How does that work with a_evaluate() since we'd have to pass the same args
-    @abstractmethod
-    def _build_measure_prompt(self, example: Example) -> List[dict]:
-        # builds the prompt that is sent to the model inside of the `score_example()` method
-        # returns either a string prompt or a conversation prompt of the form [{"role": "system", "content": "..."}, ...]
-
-        """
-        This function creates the prompt that the judge model uses to evaluate examples.
-
-        The prompt is typically a set of instructions that the judge model uses to evaluate the example.
-
-        This function returns a conversation prompt of the form
-        [{"role": "system", "content": "..."}, {"role": "user", "content": "..."}]
-
-        A basic version of implementing this function could be as follows:
-        SYSTEM_ROLE = ...
-        return [
-            {"role": "system", "content": SYSTEM_ROLE},
-            {"role": "user", "content": f"Response: {example.actual_output}\n\nYour judgment: "}
-        ]
-        """
-        pass
-
-    # TODO: does this need to take *args and **kwargs? How does that work with a_evaluate() since we'd have to pass the same args
-    @abstractmethod
-    def _build_schema(self) -> dict:
-        """
-        This function returns a dictionary that represents the schema of the JSON response that the judge model should return.
-
-        The keys of the dictionary are the expected keys in the response, and the values are the types of the corresponding values.
-
-        Example: If you want to have the judge model return a score and a reason, you would write:
-        return {"score": int, "reason": str}
-        """
-        pass
-
-    def _enforce_prompt_format(self, judge_prompt: List[dict], schema: dict):
-        """
-        Formats the final prompt to the judge model.
-
-        This function takes a list of dictionaries (`judge_prompt`) and a schema dictionary (`schema`),
-        and appends a schema enforcement prompt to the content of the first dictionary in the list, which is assumed to be the system prompt.
-        The schema enforcement prompt instructs the judge model to provide its response in a specific JSON format.
-
-        Args:
-            judge_prompt (List[dict]): A list of dictionaries representing the judge prompt.
-                                       Each dictionary should contain a "content" key.
-            schema (dict): A dictionary representing the schema. The keys are the expected keys in the response,
-                           and the values are the types of the corresponding values.
-
-        Returns:
-            List[dict]: The modified judge prompt with the schema enforcement prompt appended to the content
-                        of the first dictionary.
-
-        Raises:
-            TypeError: If `judge_prompt` is not a list of dictionaries.
-
-        Example:
-            judge_prompt = [{"content": "Please evaluate the following:"}]
-            schema = {"score": int, "comments": str}
-            formatted_prompt = format_measure_prompt(judge_prompt, schema)
-            # formatted_prompt[0]["content"] will include the schema enforcement prompt
-        """
-        SCHEMA_ENFORCEMENT_PROMPT = (
-            "\n\nPlease provide your response in the following JSON format: {"
-        )
-        if isinstance(judge_prompt, list) and all(
-            isinstance(item, dict) for item in judge_prompt
-        ):
-            # create formatting string for schema enforcement
-            # schema is a map between key and type of the value
-            for key, key_type in schema.items():
-                SCHEMA_ENFORCEMENT_PROMPT += f'"{key}": <{key}> ({key_type.__name__}), '
-            SCHEMA_ENFORCEMENT_PROMPT = (
-                SCHEMA_ENFORCEMENT_PROMPT[:-2] + "}"
-            )  # remove trailing comma and space
-            judge_prompt[0]["content"] += SCHEMA_ENFORCEMENT_PROMPT
-            return judge_prompt
-        else:
-            raise TypeError(
-                f"Prompt must be a list of dictionaries. Got {type(judge_prompt)} instead."
-            )
-
-    @abstractmethod
-    def _process_response(self, response: dict):
-        """
-        Customizable method for processing the response from the judge model.
-
-        You can add any additional logic to parse the JSON response here and return the result and reason for decision.
-
-        If you don't need a reason for the decision, you can simply return (score, None).
-
-        Example:
-        score = response["score"]
-        reason = response["reason"]
-        return score, reason
-        """
-        pass
-
-    @abstractmethod
-    def _success_check(self, **kwargs) -> bool:
-        """
-        Determines whether or not the PromptScorer should consider the evaluation of a single example successful.
-        """
-        pass
-
-    @property
-    def __name__(self):
-        return self.name
-
-    class Config:
-        arbitrary_types_allowed = True

judgeval-0.0.52.dist-info/RECORD

@@ -1,69 +0,0 @@
-judgeval/__init__.py,sha256=HM1M8hmqRum6G554QKkXhB4DF4f5eh_xtYo0Kf-t3kw,332
-judgeval/clients.py,sha256=JnB8n90GyXiYaGmSEYaA67mdJSnr3SIrzArao7NGebw,980
-judgeval/constants.py,sha256=IwW428u2VxThczHiL6ZnRwrIzb6QwOE4kdKonktVFYA,6032
-judgeval/evaluation_run.py,sha256=9fYFWJ2ZXtnNcRqxLjzKkZHAba2xi_f1uzOXDJ37Pgw,3233
-judgeval/judgment_client.py,sha256=RGqjw6Q50DOaTPa5SfCzSSGjsm7zlkZ6N7LOvewCxVU,21510
-judgeval/rules.py,sha256=TKI1K_Wlo3GDoSCztGcDoTioVKpvfG6zVkONyou8v5c,20465
-judgeval/run_evaluation.py,sha256=JohxsU5EajwPgBhBGt_wTrNSGdVIbSJmMAR5ffCSg7c,51478
-judgeval/version_check.py,sha256=FlKE8AQGwu50d3kdWSiBZYVW9sicnFInCZjakKt37w4,1003
-judgeval/common/__init__.py,sha256=KH-QJyWtQ60R6yFIBDYS3WGRiNpEu1guynpxivZvpBQ,309
-judgeval/common/exceptions.py,sha256=OkgDznu2wpBQZMXiZarLJYNk1HIcC8qYW7VypDC3Ook,556
-judgeval/common/logger.py,sha256=_nNV4waaMB4NkjwAG0kYZ3cfBe19BY6b2vsCdKd2YR4,6112
-judgeval/common/s3_storage.py,sha256=ukylTrBZ2QuT8BGbOY7D738RvHFAzVaPwmuWQ4R5xkE,3986
-judgeval/common/tracer.py,sha256=7vvPY632z4ExsqIuNRjfpJfa6CpJKohz8kvBiSwbjFE,129624
-judgeval/common/utils.py,sha256=p8C_BM0nNcIiVHTBH4BqsR106RNUlZ9wM0SxWY4IozE,35543
-judgeval/data/__init__.py,sha256=Nuy_F6oll5c5qLOF2gGFWFYyXeOgXSh7R4Vm2kMiXDM,531
-judgeval/data/custom_example.py,sha256=o4baSEeyNhS-k9PiOJdN4NfBFBGJMvhnU5RBvVRFRd8,734
-judgeval/data/example.py,sha256=8wsqBJ98Nw7IaVNXZmUoz3UuQUWkBbnHI6t4_1pqmr8,7234
-judgeval/data/result.py,sha256=4TfBPukRpF2iaF14zEU1RP-wHxsPWrX8PaXYnhxN8MM,3132
-judgeval/data/scorer_data.py,sha256=FnePIXS-4oNqrM2Eo97-hL3g3ZKFIvEKLdkl0CnpHEI,3283
-judgeval/data/tool.py,sha256=QMYJO8kyhGum8iiXxZZ_9pGcxcqp7Fjp0R0sh6i_9rU,1915
-judgeval/data/trace.py,sha256=tn1ctv99UI_vG_1UmFlzvt2L20mviUSwbvVs8ow8X-o,5797
-judgeval/data/trace_run.py,sha256=NMUkf5bxMW_jWXxZ-JI8-gOKSASldS7oAMH4MH4oSYE,1841
-judgeval/data/datasets/__init__.py,sha256=IdNKhQv9yYZ_op0rdBacrFaFVmiiYQ3JTzXzxOTsEVQ,176
-judgeval/data/datasets/dataset.py,sha256=VDHQpOUoWUfaPmCeolKP-hhSzQcCHq1muRg3EtLRpf0,12933
-judgeval/data/datasets/eval_dataset_client.py,sha256=93Pxb3aCgDwvi263N0CgugApIwKbHbPSfuz7j0IhHSY,12880
-judgeval/integrations/langgraph.py,sha256=3fKMOhAjuDdH_q3F9OlW2T_fx_vzBg2Sz4WP4WFvBuw,35909
-judgeval/judges/__init__.py,sha256=6X7VSwrwsdxGBNxCyapVRWGghhKOy3MVxFNMQ62kCXM,308
-judgeval/judges/base_judge.py,sha256=_dz0qWsKRxzXxpRY9l6mrxTRYPSF2FE4ZXkrzhZ4gbY,986
-judgeval/judges/litellm_judge.py,sha256=pHKdNkhdBMlrok3ZMTWaomGX6DKFXYV9zHqvCL7_2jo,2653
-judgeval/judges/mixture_of_judges.py,sha256=jcE3H47bVMdqzYRuxa0LD8wudF1kxkRujEpbVV-rkcM,15913
-judgeval/judges/together_judge.py,sha256=DZKlsij2ikmDiYbLZKWm8oqDRNNuvCBiGM0JcycwqWM,2424
-judgeval/judges/utils.py,sha256=0CF9qtIUQUL3-W-qTGpmTjZbkUUBAM6TslDsrCHnTBU,2725
-judgeval/scorers/__init__.py,sha256=VKPveyGCv5Rc0YtuT7iAxSv-M5EuikqAVeaGNnYMuWE,1340
-judgeval/scorers/api_scorer.py,sha256=2LNqcwIMerb37WooGD-hw5WIVLcTXnxWxzwZ0h9CXq0,2795
-judgeval/scorers/exceptions.py,sha256=ACDHK5-TWiF3NTk-wycaedpbrdobm-CvvC1JA_iP-Mk,179
-judgeval/scorers/judgeval_scorer.py,sha256=VoiAQdJzgoiVyFYS9gLEGtQwfQY6tUBoWBBDyGBfo-Q,7321
-judgeval/scorers/prompt_scorer.py,sha256=w0tW76J956smL4D8PsOHswjwYFb8W08E_0E9ad5_aQ8,12124
-judgeval/scorers/score.py,sha256=_mKQuoZHEqrF9PaydPtzWN3zjE6PeKYETw_1UryzJ3s,19399
-judgeval/scorers/utils.py,sha256=UKssYyqsJ_hckeqa1aGcXLLxiScRDzYilyuT1RqkVyo,6853
-judgeval/scorers/judgeval_scorers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-judgeval/scorers/judgeval_scorers/api_scorers/__init__.py,sha256=mmGIBCWN2WByjSUn9o5-xmHV2W-fDNyRofNsEpSuqyQ,2248
-judgeval/scorers/judgeval_scorers/api_scorers/answer_correctness.py,sha256=xY7vY4uIfncEiCksGu5SFT8dUjzkY9suNgyvipQ1avU,712
-judgeval/scorers/judgeval_scorers/api_scorers/answer_relevancy.py,sha256=t2ClO5nL6rM_atKV9YFgOCrQEPI_KjNqs1tyF3WqQig,659
-judgeval/scorers/judgeval_scorers/api_scorers/classifier_scorer.py,sha256=USeIQ1nozvQhMIrRLpST3nqNOekOFW5XJG4NSP7w0RI,4430
-judgeval/scorers/judgeval_scorers/api_scorers/comparison.py,sha256=H4K_NIMabYd_OPlMz3CNNMIM3vYk7PunTXygMnyp6sc,1240
-judgeval/scorers/judgeval_scorers/api_scorers/contextual_precision.py,sha256=QldMhW7k16jPPiHQAeLH-2VilPTuNHVi6OMsWvWnycE,771
-judgeval/scorers/judgeval_scorers/api_scorers/contextual_recall.py,sha256=GDxEljGD4E-8j6t9DpV2cve0gcKZiUYHn2bfyXChbu0,759
-judgeval/scorers/judgeval_scorers/api_scorers/contextual_relevancy.py,sha256=4E6Sa1aaI1k9PvA8afzNwIdrBCxv4UOqMtmfnLlWeWs,826
-judgeval/scorers/judgeval_scorers/api_scorers/derailment_scorer.py,sha256=jiKi8EfwP_yuOwHhYStbIUQIn2LPwJEbkh8PQeOoDTs,475
-judgeval/scorers/judgeval_scorers/api_scorers/execution_order.py,sha256=guG37tQm8m4Gs1bwYS1eaNau-RJYwteb1hwYQ0YMIbk,1357
-judgeval/scorers/judgeval_scorers/api_scorers/faithfulness.py,sha256=6iK6Da0FWoyDe_OH7UMnc4gpnByNqfIx6BW8nUbvlC0,693
-judgeval/scorers/judgeval_scorers/api_scorers/groundedness.py,sha256=RrGgBMgwVPpxb9cHm-yXQBgoh6CHUm_GkFYGSp-KcUc,693
-judgeval/scorers/judgeval_scorers/api_scorers/hallucination.py,sha256=VbvEEawOZ1XA3SWS986cbR2m3Clyliv21nzHe9GrQxo,687
-judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py,sha256=nk4_lpq2eIe6v8GtBm2g6O1CLCg5sP7-wspye6qNuXE,679
-judgeval/scorers/judgeval_scorers/api_scorers/json_correctness.py,sha256=9gKX32g9dAvFdHXzQmR-CFabBPLIZHu8aCnICK3t7j8,1066
-judgeval/scorers/judgeval_scorers/api_scorers/summarization.py,sha256=Wz5wtpqeXMdK8oRXRKnWqow4s1mmqGFQqHK42wo6cNQ,648
-judgeval/scorers/judgeval_scorers/api_scorers/tool_dependency.py,sha256=wzgprwQ3hcsc9itHG0DkcXyNnvVVd-s1UpNyZxw49Sw,590
-judgeval/scorers/judgeval_scorers/api_scorers/tool_order.py,sha256=462fR2m-67FR2TdHu6cCNZLRkIT_yTAOrMeb-1AuQe8,576
-judgeval/scorers/judgeval_scorers/classifiers/__init__.py,sha256=Qt81W5ZCwMvBAne0LfQDb8xvg5iOG1vEYP7WizgwAZo,67
-judgeval/scorers/judgeval_scorers/classifiers/text2sql/__init__.py,sha256=8iTzMvou1Dr8pybul6lZHKjc9Ye2-0_racRGYkhEdTY,74
-judgeval/scorers/judgeval_scorers/classifiers/text2sql/text2sql_scorer.py,sha256=gloLzThkFsr8sHQargDAH8XaDrlF6OCuc_69hyNslFU,2589
-judgeval/tracer/__init__.py,sha256=wkuXtOGDCrwgPPXlh_sSJmvGuWaAMHyNzk1TzB5f9aI,148
-judgeval/utils/alerts.py,sha256=3w_AjQrgfmOZvfqCridW8WAnHVxHHXokX9jNzVFyGjA,3297
-judgeval/utils/file_utils.py,sha256=M6a_BPRGMwEFBPdF_Tbcbbk4YldHcOhuoU9oRlmninE,1858
-judgeval/utils/requests.py,sha256=rbmZTaiyWI8t2YUkhk11SIe3dF7j2j25L1BuFp_1PII,770
-judgeval-0.0.52.dist-info/METADATA,sha256=bEoAZvedhy0uEclNseDaiC37z8ST_zKg82QTtQ5EV98,54719
-judgeval-0.0.52.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-judgeval-0.0.52.dist-info/licenses/LICENSE.md,sha256=tKmCg7k5QOmxPK19XMfzim04QiQJPmgIm0pAn55IJwk,11352
-judgeval-0.0.52.dist-info/RECORD,,