eval-framework 0.2.0__py3-none-any.whl

eval_framework/metrics/llm/graders/chatbot_style_grader.py

@@ -0,0 +1,92 @@
+from collections.abc import Mapping
+
+from eval_framework.llm.base import BaseLLM as StructuredOutputChatModel
+from eval_framework.metrics.llm.graders.language import Language
+from eval_framework.metrics.llm.graders.models import GradingOutput, PromptTemplate, parse_json_output
+
+
+class ChatbotStyleGradingOutput(GradingOutput):
+    thought_process: str | None
+    is_chatbot_style: bool | None
+
+
+class ChatbotStyleGrader:
+    COMPLETION_KEY = "completion"
+    PROMPT_TEMPLATES = {
+        Language("de"): PromptTemplate(
+            system_prompt="""Deine Aufgabe ist es zu klassifizieren, ob eine von einem Textgenerator gelieferte Antwort dem Stile eines Chatbots entspricht.
+
+Hier sind einige Schlüsselmerkmale einer Antwort im Stile eines Chatbots:
+* Sie leitet den Hauptinhalt mit Phrasen wie "Natürlich, ich helfe Dir gerne", "Na klar!" oder "Selbstverständlich kann ich" ein.
+* Sie endet mit Phrasen wie "Ich hoffe, ich konnte Dir weiterhelfen!"
+* Sie stellt Nachfragen an den Benutzer.
+* Sie neigt dazu, überaus wortreich zu sein.
+* Sie enthält Gesprächs- und Unterhaltungsfloskeln.
+* Sie enthält Text, der zum Verständnis des Inhalts nicht zwingend notwendig ist.
+* Sie bewahrt einen überaus freundlichen Ton.
+
+Beachte, dass die Erfüllung von nur einem dieser Merkmale ausreicht, um die Antwort als Chatbot-Stil zu klassifizieren.
+
+Gebe deine Bewertung in folgendem JSON-Format:
+{
+    "thought_process": str (Achte sehr genau auf die Antwort und argumentiere in ein paar Sätzen, ob die Antwort dem Chatbot-Stil folgt oder nicht),
+    "is_chatbot_style": bool
+}""",  # noqa: E501
+            user_prompt=f"""**Antwort des Textgenerators**
+{{{COMPLETION_KEY}}}""",
+        ),
+        Language("en"): PromptTemplate(
+            system_prompt="""Your task is to classify if a text generation model's response follows a chatbot-style format.
+
+Here are some key characteristics of a chatbot-style response:
+* It introduces the main content with phrases like "Certainly, here is", "Sure!" or "Of course."
+* It ends with phrases such as "I hope this helps!"
+* It asks follow-up questions.
+* It tends to be verbose.
+* It tends to contain fluff that is not necessary to understand the content.
+* It maintains a friendly tone.
+
+Note that even one of these characteristics is enough to classify the response as following a chatbot-style format.
+
+You must provide your evaluation in the following JSON format:
+{
+    "thought_process": str (Pay very close attention to the response and argue whether the response follows a chatbot-style or not in a few sentences),
+    "is_chatbot_style": bool
+}""",  # noqa: E501
+            user_prompt=f"""**Model Response**:
+{{{COMPLETION_KEY}}}""",
+        ),
+    }
+
+    def __init__(
+        self,
+        grading_model: StructuredOutputChatModel,
+        prompt_templates: Mapping[Language, PromptTemplate] = PROMPT_TEMPLATES,
+    ) -> None:
+        self._grading_model = grading_model
+
+        if not all(self.COMPLETION_KEY in prompt_template.user_prompt for prompt_template in prompt_templates.values()):
+            raise ValueError(f"At least one PromptTemplate is invalid, must contain '{self.COMPLETION_KEY}'.")
+        self._prompt_templates = prompt_templates
+
+    def grade(self, completion: str, language: Language) -> ChatbotStyleGradingOutput:
+        try:
+            prompt_template = language.language_config(self._prompt_templates)
+        except Exception as _:
+            prompt_template = Language("en").language_config(self._prompt_templates)
+
+        messages = prompt_template.to_messages(
+            [],
+            [
+                (self.COMPLETION_KEY, completion),
+            ],
+        )
+        raw_completion = self._grading_model.generate_from_messages([messages])[0]
+        loaded_json = parse_json_output(raw_completion.completion)
+
+        return ChatbotStyleGradingOutput(
+            thought_process=loaded_json.get("thought_process", None),
+            is_chatbot_style=loaded_json.get("is_chatbot_style", None),
+            judge_prompt=raw_completion.prompt,
+            judge_response=raw_completion.completion,
+        )

eval_framework/metrics/llm/graders/comparison_grader.py

@@ -0,0 +1,146 @@
+from collections.abc import Mapping
+from enum import Enum
+
+from eval_framework.llm.base import BaseLLM as StructuredOutputChatModel
+from eval_framework.metrics.llm.graders.language import Language
+from eval_framework.metrics.llm.graders.models import (
+    GradingOutput,
+    PromptTemplateWithParseMap,
+    parse_json_output,
+)
+
+
+class MatchOutcome(str, Enum):
+    A_WINS = "a_wins"
+    DRAW = "draw"
+    B_WINS = "b_wins"
+
+    @property
+    def payoff(self) -> tuple[float, float]:
+        if self == self.A_WINS:
+            return (1, 0)
+        if self == self.DRAW:
+            return (0.5, 0.5)
+        return (0, 1)
+
+    @staticmethod
+    def from_rank_literal(rank: int) -> "MatchOutcome":
+        match rank:
+            case 1:
+                return MatchOutcome.A_WINS
+            case 2:
+                return MatchOutcome.B_WINS
+            case 3:
+                return MatchOutcome.DRAW
+            case _:
+                raise ValueError(f"Got unexpected rank {rank}")
+
+
+class ComparisonGradingOutput(GradingOutput):
+    reasoning: str
+    outcome: MatchOutcome
+
+
+class ComparisonGrader:
+    INSTRUCTION_KEY = "instruction"
+    ANSWER_1_KEY = "answer_1"
+    ANSWER_2_KEY = "answer_2"
+    REASONING_KEY = "explanation"
+    BETTER_ANSWER_KEY = "better_answer"
+    PROMPT_TEMPLATES = {
+        Language("de"): PromptTemplateWithParseMap(
+            system_prompt=f"""Beachte die gegebene Aufgabe und dazugehörigen Antworten. Entscheide, welche Antwort besser ist, Antwort 1 oder Antwort 2. Gebe anschließend "Antwort 1 ist besser", "Antwort 2 ist besser" oder "Beide gleich" aus.
+
+Eine gute Antwort ist:
+1. Inhaltlich korrekt.
+2. Beachtet die Anforderungen der Aufgabe präzise.
+3. Ist im Rahmen der Aufgabenstellung kreativ und nicht repetetiv.
+4. In der Sprache der Aufgabe verfasst.
+
+Gebe die Antwort im folgenden json-Format:
+{{
+    "{REASONING_KEY}": str (Beschreibe in wenigen Sätzen (max. 5) die Unterschiede der beiden Antworten und begründe, warum eine der beiden Antworten besser ist oder warum die Antworten ähnlich gut sind.),
+    "{BETTER_ANSWER_KEY}": Literal["Antwort 1 ist besser", "Antwort 2 ist besser", "Beide gleich"]
+}}""",  # noqa: E501
+            user_prompt=f"""Aufgabe:
+{{{INSTRUCTION_KEY}}}
+---
+Antwort 1:
+{{{ANSWER_1_KEY}}}
+---
+Antwort 2:
+{{{ANSWER_2_KEY}}}""",
+            parse_map={
+                "Antwort 1 ist besser": MatchOutcome.A_WINS,
+                "Antwort 2 ist besser": MatchOutcome.B_WINS,
+                "Beide gleich": MatchOutcome.DRAW,
+            },
+        ),
+        Language("en"): PromptTemplateWithParseMap(
+            system_prompt=f"""Note the given task and the corresponding answers. Decide which answer is better, answer 1 or answer 2. Then output "Answer 1 is better", "Answer 2 is better" or "Both equal".
+
+A good answer is:
+1. correct in content.
+2. follows the requirements of the task precisely.
+3. is creative and not repetitive in the context of the task.
+4. written in the same language as the task.
+
+Enter the answer in the following json format:
+{{
+    "{REASONING_KEY}": str (Describe in a few sentences (max. 5) the differences between the two answers and give reasons why one of the two answers is better or why the answers are similarly good),
+    "{BETTER_ANSWER_KEY}": Literal["Answer 1 is better", "Answer 2 is better", "Both equal"]
+}}""",  # noqa: E501
+            user_prompt=f"""Task:
+{{{INSTRUCTION_KEY}}}
+---
+Answer 1:
+{{{ANSWER_1_KEY}}}
+---
+Answer 2:
+{{{ANSWER_2_KEY}}}""",
+            parse_map={
+                "Answer 1 is better": MatchOutcome.A_WINS,
+                "Answer 2 is better": MatchOutcome.B_WINS,
+                "Both equal": MatchOutcome.DRAW,
+            },
+        ),
+    }
+
+    def __init__(
+        self,
+        grading_model: StructuredOutputChatModel,
+        prompt_templates: Mapping[Language, PromptTemplateWithParseMap] = PROMPT_TEMPLATES,
+    ) -> None:
+        self._grading_model = grading_model
+
+        if not all(
+            self.INSTRUCTION_KEY in prompt_template.user_prompt for prompt_template in prompt_templates.values()
+        ) or not all(self.ANSWER_1_KEY in prompt_template.user_prompt for prompt_template in prompt_templates.values()):
+            raise ValueError(
+                f"At least one PromptTemplate invalid, must contain '{self.ANSWER_1_KEY}' and '{self.INSTRUCTION_KEY}'."
+            )
+        self._prompt_templates = prompt_templates
+
+    def grade(
+        self, instruction: str, completion_1: str, completion_2: str, language: Language
+    ) -> ComparisonGradingOutput:
+        prompt_template = language.language_config(self._prompt_templates)
+
+        messages = prompt_template.to_messages(
+            [],
+            [
+                (self.INSTRUCTION_KEY, instruction),
+                (self.ANSWER_1_KEY, completion_1),
+                (self.ANSWER_2_KEY, completion_2),
+            ],
+        )
+
+        raw_completion = self._grading_model.generate_from_messages([messages])[0]
+        loaded_json = parse_json_output(raw_completion.completion)
+
+        return ComparisonGradingOutput(
+            reasoning=loaded_json.get(self.REASONING_KEY, None),
+            outcome=prompt_template.parse_map.get(str(loaded_json.get(self.BETTER_ANSWER_KEY, None)), None),
+            judge_prompt=raw_completion.prompt,
+            judge_response=raw_completion.completion,
+        )

eval_framework/metrics/llm/graders/conciseness_grader.py

@@ -0,0 +1,93 @@
+from collections.abc import Mapping
+
+from eval_framework.llm.base import BaseLLM as StructuredOutputChatModel
+from eval_framework.metrics.llm.graders.language import Language
+from eval_framework.metrics.llm.graders.models import GradingOutput, PromptTemplate, parse_json_output
+
+
+class ConcisenessGradingOutput(GradingOutput):
+    thought_process: str | None
+    is_concise: bool | None
+
+
+class ConcisenessGrader:
+    INSTRUCTION_KEY = "instruction"
+    COMPLETION_KEY = "completion"
+    PROMPT_TEMPLATES = {
+        Language("de"): PromptTemplate(
+            system_prompt="""Deine Aufgabe ist es zu klassifizieren, ob eine von einem Textgenerator gelieferte Antwort kurz und prägnant ist.
+
+Eine kurz und prägnante ("concise") Antwort ist eine Antwort, die knapp und auf den Punkt ist, ohne unnötige Details oder Ausführungen.
+
+Gebe deine Bewertung in folgendem JSON-Format:
+{
+    "thought_process": str (Achte sehr genau auf die Antwort und argumentiere in ein paar Sätzen, ob die Antwort kurz und prägnant ("concise") ist oder nicht),
+    "is_concise": bool
+}""",  # noqa: E501
+            user_prompt=f"""**Benutzeranweisung**
+{{{INSTRUCTION_KEY}}}
+
+---
+**Antwort des Textgenerators**
+{{{COMPLETION_KEY}}}""",
+        ),
+        Language("en"): PromptTemplate(
+            system_prompt="""Your task is to classify if a text generation model's response is concise.
+
+A concise response is one that is brief and to the point, without unnecessary details or elaboration.
+
+You must provide your evaluation in the following JSON format:
+{
+    "thought_process": str (Pay very close attention to the response and argue whether the response is concise or not in a few sentences),
+    "is_concise": bool
+}""",  # noqa: E501
+            user_prompt=f"""**User Instruction**:
+{{{INSTRUCTION_KEY}}}
+
+---
+**Model Response**:
+{{{COMPLETION_KEY}}}""",
+        ),
+    }
+
+    def __init__(
+        self,
+        grading_model: StructuredOutputChatModel,
+        prompt_templates: Mapping[Language, PromptTemplate] = PROMPT_TEMPLATES,
+    ) -> None:
+        self._grading_model = grading_model
+
+        if not all(
+            self.INSTRUCTION_KEY in prompt_template.user_prompt for prompt_template in prompt_templates.values()
+        ) or not all(
+            self.COMPLETION_KEY in prompt_template.user_prompt for prompt_template in prompt_templates.values()
+        ):
+            raise ValueError(
+                f"At least one PromptTemplate is invalid, must contain '{self.COMPLETION_KEY}' "
+                "and '{self.INSTRUCTION_KEY}'."
+            )
+        self._prompt_templates = prompt_templates
+
+    def grade(self, instruction: str, completion: str, language: Language) -> ConcisenessGradingOutput:
+        try:
+            prompt_template = language.language_config(self._prompt_templates)
+        except Exception as _:
+            prompt_template = Language("en").language_config(self._prompt_templates)
+
+        messages = prompt_template.to_messages(
+            [],
+            [
+                (self.INSTRUCTION_KEY, instruction),
+                (self.COMPLETION_KEY, completion),
+            ],
+        )
+
+        raw_completion = self._grading_model.generate_from_messages([messages])[0]
+        loaded_json = parse_json_output(raw_completion.completion)
+
+        return ConcisenessGradingOutput(
+            thought_process=loaded_json.get("thought_process", None),
+            is_concise=loaded_json.get("is_concise", None),
+            judge_prompt=raw_completion.prompt,
+            judge_response=raw_completion.completion,
+        )

eval_framework/metrics/llm/graders/contains_names_grader.py

@@ -0,0 +1,71 @@
+from collections.abc import Mapping
+
+from eval_framework.llm.base import BaseLLM as StructuredOutputChatModel
+from eval_framework.metrics.llm.graders.language import Language
+from eval_framework.metrics.llm.graders.models import GradingOutput, PromptTemplate, parse_json_output
+
+
+class ContainsNamesGradingOutput(GradingOutput):
+    contains_names: bool | None
+
+
+class ContainsNamesGrader:
+    COMPLETION_KEY = "completion"
+    PROMPT_TEMPLATES = {
+        Language("en"): PromptTemplate(
+            system_prompt="""Your assignment is to assess whether text contains names of individuals.
+
+Your assessment should be formatted in JSON as follows:
+{
+    "contains_names": bool (State with a true or false whether the text contains any names.)
+}""",
+            user_prompt=f"""{{{COMPLETION_KEY}}}""",
+        ),
+        Language("de"): PromptTemplate(
+            system_prompt="""Deine Aufgabe ist es zu bewerten, ob ein Text Namen von Personen enthält.
+
+Deine Bewertung sollte im JSON-Format wie folgt aussehen:
+{
+    "contains_names": bool (Gib an, ob der Text Namen enthält oder nicht.)
+}""",
+            user_prompt=f"""{{{COMPLETION_KEY}}}""",
+        ),
+        Language("fr"): PromptTemplate(
+            system_prompt="""Votre tâche consiste à évaluer si un texte contient des noms de personnes.
+
+Votre évaluation doit être formatée en JSON comme suit :
+{
+    "contains_names": bool (Indiquez si le texte contient des noms ou non.)
+}""",
+            user_prompt=f"""{{{COMPLETION_KEY}}}""",
+        ),
+    }
+
+    def __init__(
+        self,
+        grading_model: StructuredOutputChatModel,
+        prompt_templates: Mapping[Language, PromptTemplate] = PROMPT_TEMPLATES,
+    ) -> None:
+        self._grading_model = grading_model
+
+        if not all(self.COMPLETION_KEY in prompt_template.user_prompt for prompt_template in prompt_templates.values()):
+            raise ValueError(f"At least one PromptTemplate is invalid, must contain '{self.COMPLETION_KEY}'.")
+        self._prompt_templates = prompt_templates
+
+    def grade(self, completion: str, language: Language) -> ContainsNamesGradingOutput:
+        prompt_template = language.language_config(self._prompt_templates)
+        messages = prompt_template.to_messages(
+            [],
+            [
+                (self.COMPLETION_KEY, completion),
+            ],
+        )
+
+        raw_completion = self._grading_model.generate_from_messages([messages])[0]
+        loaded_json = parse_json_output(raw_completion.completion)
+
+        return ContainsNamesGradingOutput(
+            contains_names=loaded_json.get("contains_names", None),
+            judge_prompt=raw_completion.prompt,
+            judge_response=raw_completion.completion,
+        )

eval_framework/metrics/llm/graders/format_correctness_grader.py

@@ -0,0 +1,109 @@
+from collections.abc import Mapping
+
+from eval_framework.llm.base import BaseLLM as StructuredOutputChatModel
+from eval_framework.metrics.llm.graders.language import Language
+from eval_framework.metrics.llm.graders.models import (
+    FOFOPromptTemplate,
+    GradingOutput,
+    parse_json_output,
+)
+
+
+class FormatCorrectnessOutput(GradingOutput):
+    reasons: str | None
+    format_correctness: int | None
+
+
+class FormatCorrectnessGrader:
+    INSTRUCTION_KEY = "<instruction>"
+    COMPLETION_KEY = "<completion>"
+
+    PROMPT_TEMPLATES = {
+        Language("en"): FOFOPromptTemplate(
+            system_prompt="You are a helpful assistant who evaluates the correctness and quality of models' outputs.",
+            user_prompt=f"""
+            I would like you to create a leaderboard that evaluates the correctness of the format of answers from
+            various large language models. To accomplish this, you will need to analyze the text prompts given to
+            the models and their corresponding answers. Specifically, please ensure that your evaluation outputs are
+            properly formatted as a json string. I will provide both the prompts and the responses for this purpose.\n
+
+            Here is the prompt: {{
+                "instruction": {INSTRUCTION_KEY}
+            }}
+
+            Here are the outputs of the models:
+            [
+                {{
+                    "model": "model",
+                    "answer": {COMPLETION_KEY}
+                }},
+            ]
+
+            Please evaluate the formatting of the model’s responses by checking if they comply with the format
+            specifications stated in the prompt. Perform a thorough format check and provide a detailed explanation
+            for why the format is correct or incorrect. Your feedback should include the name of the model, followed
+            by the format correctness status represented as ’1’ for correct and ’0’ for incorrect. Present your
+            reasoning as bullet points within a single string for each model assessed. In other words, you should
+            produce the following output:
+            ```json
+            [
+                {{
+                "model": <model-name>,
+                "format_correctness": <correctness>,
+                "reasons": <reasons-of-format-correctness>
+                }}
+            ]```
+
+            Please note that your response should be a properly formatted JSON string and should not contain any
+            additional content. We will load it directly as a JSON string in Python.
+            """,
+        ),
+    }
+
+    def __init__(
+        self,
+        grading_model: StructuredOutputChatModel,
+        prompt_templates: Mapping[Language, FOFOPromptTemplate] = PROMPT_TEMPLATES,
+    ) -> None:
+        self._grading_model = grading_model
+
+        if not all(
+            self.INSTRUCTION_KEY in prompt_template.user_prompt for prompt_template in prompt_templates.values()
+        ) or not all(
+            self.COMPLETION_KEY in prompt_template.user_prompt for prompt_template in prompt_templates.values()
+        ):
+            raise ValueError(
+                f"At least one PromptTemplate is invalid, must contain '{self.COMPLETION_KEY}' "
+                "and '{self.INSTRUCTION_KEY}'."
+            )
+        self._prompt_templates = prompt_templates
+
+    def grade(self, instruction: str, completion: str, language: Language) -> FormatCorrectnessOutput:
+        try:
+            prompt_template = language.language_config(self._prompt_templates)
+        except Exception as _:
+            prompt_template = Language("en").language_config(self._prompt_templates)
+
+        messages = prompt_template.to_messages(
+            [],
+            [
+                (self.INSTRUCTION_KEY, instruction),
+                (self.COMPLETION_KEY, completion),
+            ],
+        )
+
+        raw_completion = self._grading_model.generate_from_messages([messages])[0]
+        loaded_json = parse_json_output(raw_completion.completion)
+        reasons = loaded_json.get("reasons", None)
+        if isinstance(reasons, list):
+            reasons = ["• " + reason + "\n " for reason in reasons]
+            reasons = "".join(reasons)
+        elif isinstance(reasons, str):
+            reasons = "• " + reasons + "\n "
+
+        return FormatCorrectnessOutput(
+            reasons=reasons,
+            format_correctness=loaded_json.get("format_correctness", None),
+            judge_prompt=raw_completion.prompt,
+            judge_response=raw_completion.completion,
+        )