eval-framework 0.2.7__py3-none-any.whl

eval_framework/metrics/completion/language_checker.py

@@ -0,0 +1,74 @@
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.metrics.llm.graders.language import AVAILABLE_LANGUAGES
+from eval_framework.shared.types import Completion
+
+
+class LanguageChecker(BaseMetric[Completion]):
+    NAME = "Language Check"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        if response.ground_truth is None:
+            raise LogicError("Language detection needs ground_truth.")
+        if response.ground_truth not in AVAILABLE_LANGUAGES:
+            raise LogicError("Checking for unknown or unavailable language.")
+
+        completion_language = response.get_completion_language()
+        target_language = response.ground_truth
+        value = float(completion_language == target_language)
+        return [MetricResult(metric_name=self.NAME, value=value, higher_is_better=True, error=response.error)]
+
+
+class GermanCompletionChecker(BaseMetric[Completion]):
+    NAME = "German Completion Check"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        raw_completion_language = response.get_raw_completion_language()
+        value = float(raw_completion_language == "de")
+        return [MetricResult(metric_name=self.NAME, value=value, higher_is_better=True, error=response.error)]
+
+
+class LanguageConsistencyChecker(BaseMetric[Completion]):
+    NAME = "Language Consistency"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        completion_language = response.get_completion_language()
+        target_language = response.get_instruction_language()
+        if completion_language == target_language == "":
+            return []  # No language information could be determined
+        else:
+            value = float(completion_language == target_language)
+            return [MetricResult(metric_name=self.NAME, value=value, higher_is_better=True, error=response.error)]
+
+
+class LanguageRawConsistencyChecker(BaseMetric[Completion]):
+    NAME = "Language Consistency Raw"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        raw_completion_language = response.get_raw_completion_language()
+        target_language = response.get_instruction_language()
+
+        if raw_completion_language == target_language == "":
+            return []  # No language information could be determined
+        else:
+            value = float(raw_completion_language == target_language)
+            return [
+                MetricResult(
+                    metric_name=self.NAME,
+                    value=value,
+                    higher_is_better=True,
+                    error=response.error,
+                )
+            ]

eval_framework/metrics/completion/length_control.py

@@ -0,0 +1,83 @@
+import json
+from enum import Enum
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.metrics.completion.text_counter import ParagraphCounter, SentenceCounter, WordCounter
+from eval_framework.shared.types import Completion
+
+
+class LengthRequirementUnit(Enum):
+    WORDS = "words"
+    SENTENCES = "sentences"
+    PARAGRAPHS = "paragraphs"
+
+
+class LengthRequirementType(Enum):
+    MIN = "minimum"
+    MAX = "maximum"
+    TARGET = "target"
+
+
+class LengthControl(BaseMetric[Completion]):
+    NAME = "length_control"
+
+    def __init__(self, tolerance: float = 1 / 6) -> None:
+        super().__init__()
+        self.tolerance = tolerance
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [
+                MetricResult(
+                    metric_name=f"{self.NAME}/fulfills_length_requirement",
+                    value=None,
+                    higher_is_better=True,
+                    error=response.error if response.error is not None else None,
+                )
+            ]
+
+        expectations = json.loads(str(response.ground_truth))
+        stripped_completion = response.completion.strip()
+
+        match LengthRequirementUnit(expectations["unit"]):
+            case LengthRequirementUnit.WORDS:
+                count = WordCounter._count_words(stripped_completion)
+            case LengthRequirementUnit.SENTENCES:
+                count = SentenceCounter._count_sentences(stripped_completion)
+            case LengthRequirementUnit.PARAGRAPHS:
+                count = ParagraphCounter._count_paragraphs(stripped_completion)
+            case _:
+                raise NotImplementedError(f"LengthRequirementUnit {expectations['unit']} is not supported.")
+
+        expected_count = int(expectations["count"])
+        normalized_distance_to_target = (count - expected_count) / float(expected_count)
+        absolute_normalized_distance_to_target = abs(normalized_distance_to_target)
+
+        match LengthRequirementType(expectations["type"]):
+            case LengthRequirementType.TARGET:
+                fulfills_length_requirement = absolute_normalized_distance_to_target <= self.tolerance
+            case LengthRequirementType.MIN:
+                fulfills_length_requirement = count >= expected_count
+            case LengthRequirementType.MAX:
+                fulfills_length_requirement = count <= expected_count
+            case _:
+                raise NotImplementedError(f"LengthRequirementType {expectations['type']} is not supported.")
+
+        return [
+            MetricResult(
+                metric_name=f"{self.NAME}/normalized_distance_to_target",
+                value=float(normalized_distance_to_target),
+                higher_is_better=False,
+            ),
+            MetricResult(
+                metric_name=f"{self.NAME}/absolute_normalized_distance_to_target",
+                value=float(absolute_normalized_distance_to_target),
+                higher_is_better=False,
+            ),
+            MetricResult(
+                metric_name=f"{self.NAME}/fulfills_length_requirement",
+                value=float(fulfills_length_requirement),
+                higher_is_better=True,
+                error=response.error,
+            ),
+        ]

eval_framework/metrics/completion/math_reasoning_completion.py

@@ -0,0 +1,307 @@
+import re
+import signal
+from collections.abc import Callable, Iterable
+from typing import Any
+
+from sympy import Basic, S, SympifyError, factor, simplify
+from sympy.parsing.latex import parse_latex
+from sympy.parsing.latex.errors import LaTeXParsingError
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+def timeout_handler(signum: Any, frame: Any) -> None:
+    raise TimeoutError()
+
+
+class MathReasoningCompletion(BaseMetric[Completion]):
+    #
+    # Math Reasoning Completion (symbolic)
+    #
+    # This metric evaluates the correctness of the completion of a math reasoning task without
+    # correcting LaTeX expressions. Normalization occurs on the strings, only to remove formatting
+    # and units.
+    #
+    # The metric is designed to evaluate the correctness of the completion of a math reasoning task
+    # without correcting LaTeX expressions.
+    #
+
+    NAME = "Math Reasoning Completion (symbolic)"
+
+    # Substitutions to apply to the final answer
+    SUBSTITUTIONS = [
+        (r"\ban\b(?!\w)", ""),  # Remove "an" if not part of a word
+        (r"\ba\b(?!\w)", ""),  # Remove "a" if not part of a word
+        (r"\.\$", "$"),  # Replace ".$" with "$"
+        (r"\\\$", ""),  # Remove "\$"
+        (r"\\ ", ""),  # Remove "\ " (escaped space)
+        (r"\s+", ""),  # Remove all spaces
+        (r"\\mbox", "text"),  # Replace "\mbox" with "text"
+        (r",\\text\{and\}", ","),  # Replace ",\text{and}" with ","
+        (r"\\text\{and\}", ","),  # Replace "\text{and}" with ","
+        (r"\\text\{m\}", "\\text{}"),  # Replace "\text{m}" with "\text{}"
+    ]
+
+    # Expressions to remove from the final answer
+    # Most of these expressions omit units and formatting
+    # which the ground truth does not have
+    REMOVED_EXPRESSIONS_UNITS = [
+        "square",
+        "ways",
+        "integers",
+        "dollars",
+        "mph",
+        "inches",
+        "ft",
+        "hours",
+        "km",
+        "units",
+        "\\ldots",
+        "sue",
+        "points",
+        "feet",
+        "minutes",
+        "digits",
+        "cents",
+        "degrees",
+        "cm",
+        "gm",
+        "pounds",
+        "meters",
+        "meals",
+        "edges",
+        "students",
+        "childrentickets",
+        "multiples",
+    ]
+
+    REMOVED_EXPRESSIONS_FORMAT = [
+        "\\text{s}",
+        "\\text{.}",
+        "\\text{\ns}",
+        "\\text{}^2",
+        "\\text{}^3",
+        "\\text{\n}",
+        "\\text{}",
+        r"\mathrm{th}",
+        r"^\circ",
+        r"^{\circ}",
+        r"\;",
+        r",\!",
+        "{,}",
+        '"',
+        "\\dots",
+    ]
+
+    def normalize_expression(self, final_answer: str) -> str:
+        """
+        Function to normalize LaTeX expressions
+        :param final_answer: raw LaTeX expression
+        :return: normalized LaTeX expression
+        NOTE: Changed logic, because before the substitution randomly replaced characters in the string,
+        i.e., turned "infty" into "iny" by removing "ft"
+        """
+        for before, after in self.SUBSTITUTIONS:
+            final_answer = re.sub(before, after, final_answer)
+        for expr in self.REMOVED_EXPRESSIONS_UNITS:
+            # Safely remove units at the end, allowing optional space before the unit
+            final_answer = re.sub(rf"(.*?)\s*({re.escape(expr)})$", r"\1", final_answer)
+        for expr in self.REMOVED_EXPRESSIONS_FORMAT:
+            # Safely remove formatting expressions
+            final_answer = final_answer.replace(expr, "")
+        final_answer = re.sub(r"(.*?)(\$)(.*?)(\$)(.*)", r"$\3$", final_answer)
+        final_answer = re.sub(r"(\\text\{)(.*?)(\})", r"\2", final_answer)
+        final_answer = re.sub(r"(\\textbf\{)(.*?)(\})", r"\2", final_answer)
+        final_answer = re.sub(r"(\\overline\{)(.*?)(\})", r"\2", final_answer)
+        final_answer = re.sub(r"(\\boxed\{)(.*)(\})", r"\2", final_answer)
+        final_answer = re.sub(r"(frac)([^{])(.)", r"frac{\2}{\3}", final_answer)
+        final_answer = re.sub(r"(sqrt)([^{])", r"sqrt{\2}", final_answer)
+        final_answer = final_answer.replace("$", "")
+        # Only strip commas if it's a single numeric value with optional commas (like "1,000")
+        if re.fullmatch(r"\d{1,3}(,\d{3})*", final_answer):
+            final_answer = final_answer.replace(",", "")
+        return final_answer
+
+    def check_for_equation(self, final_answer: str) -> list:
+        """
+        Check if the final answer is an equation and split it into left hand side and right hand side
+        :param final_answer: the expression to evaluate
+        :return: list of left hand side and right hand side of the equation
+        """
+        if isinstance(final_answer, str) and "=" in final_answer:
+            return final_answer.split("=")
+        else:
+            return [final_answer]
+
+    def _safe_simplify_expression(self, expression: Basic, timeout: int = 10) -> Basic:
+        """
+        Simplify an expression with a timeout and catch recursion depth exception
+        :param expression: SymPy expression
+        :param timeout: Time limit in seconds (default: 10 seconds).
+        :return: simplified expressions
+        """
+        signal.signal(signal.SIGALRM, timeout_handler)  # Set timeout signal
+        signal.alarm(timeout)  # Set timeout duration
+
+        try:
+            factored = factor(expression)
+            simplified = simplify(factored)
+            return simplified
+        except (SympifyError, TimeoutError):
+            return S.NaN
+        finally:
+            # Ensure we never leak a pending alarm into later code paths.
+            signal.alarm(0)
+
+    def _any_symb_correct(self, response_list: Iterable[Basic], ground_truth_list: Iterable[Basic]) -> bool:
+        """
+        Check if any of the responses are correct and return true at first match
+        :param response_list: list of responses
+        :param ground_truth_list: list of ground truths
+        :return: True if any response is correct
+        """
+        for answer in response_list:
+            for ground_truth in ground_truth_list:
+                try:
+                    unsimplified_difference = answer - ground_truth
+                    # check if the difference is close to zero with numpy
+                    difference = self._safe_simplify_expression(unsimplified_difference)
+                    tolerance = 1e-12
+                    if abs(difference) < tolerance:
+                        return True
+                except ValueError:
+                    # equations cannot be evaluated against each other
+                    return False
+        return False
+
+    def _apply_safely(self, func: Callable[[Basic], Basic], list_of_expressions: list[Basic]) -> None:
+        """
+        apply safely to a list of expressions and replace the original expressions
+        :param list_of_expressions: list of sympy expressions
+        """
+        for i, expression in enumerate(list_of_expressions):
+            try:
+                list_of_expressions[i] = func(expression)
+            except RecursionError:
+                list_of_expressions[i] = S.NaN
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        """
+        Calculate the accuracy of the completion
+
+        performs several verification and simplification steps
+        to ensure that the completion is correct
+
+        the completion may either be a latex or string response
+        which sympy will parse, factor, and simplify
+
+        :param response: Completion object
+        :return: list of MetricResult
+        """
+        ground_truths = []
+        INVALID_ANSWER = S.NaN
+        timeout = 10
+        # latex parse all ingested ground truth values for math reasoning
+        for gt in response.ground_truth_list:
+            signal.signal(signal.SIGALRM, timeout_handler)  # Set timeout signal
+            signal.alarm(timeout)  # Set timeout duration
+            try:
+                gt_parsed = parse_latex(gt)  # NOTE: parses f(x)=0,\quadf(x)=x-1,\quadf(x)=-x+1 to Eq(f(x), 0) ONLY
+                ground_truths.append(gt_parsed)
+            except Exception:
+                ground_truths.append(gt)
+            finally:
+                # Ensure we never leak a pending alarm into later code paths.
+                signal.alarm(0)
+        normalized_response = self.normalize_expression(response.completion)
+        response_list = self.check_for_equation(normalized_response)
+        try:
+            symb_is_correct = self._is_symbolically_equiv(response_list, ground_truths, INVALID_ANSWER)
+        except Exception:
+            symb_is_correct = False
+
+        # check if already correct symbolically
+        if symb_is_correct:
+            return [
+                MetricResult(
+                    metric_name=self.NAME, value=float(symb_is_correct), higher_is_better=True, error=response.error
+                )
+            ]
+        else:
+            # fall back to string comparison
+            # ground truth can be list or str, we have str comparisons
+            assert isinstance(response.ground_truth, str)
+            str_is_correct = self._is_str_correct(normalized_response, response.ground_truth)
+            return [
+                MetricResult(
+                    metric_name=self.NAME, value=float(str_is_correct), higher_is_better=True, error=response.error
+                )
+            ]
+
+    def _any_str_correct(self, response_list: list, ground_truths: list) -> bool:
+        """
+        Check if any of the responses are correct and return true at first match
+        :param response_list: list of responses
+        :param ground_truths: list of ground truths
+        :return: True if any response is correct
+        """
+        for response in response_list:
+            for ground_truth in ground_truths:
+                if self._is_str_correct(response, ground_truth):
+                    return True
+        return False
+
+    def _is_str_correct(self, str1: str, str2: str) -> bool:
+        """
+        Check if two strings are equal after stripping
+        :param str1: first string
+        :param str2: second string
+        :param verbose: print the stripped strings
+        :return: True if the strings are equal
+        """
+        # if multiple equal signs in ground truth (str2)
+        # slide the response (str1) over the ground truth (str2)
+        # at the interval of every equal sign in the ground truth
+        # and check if any of the responses match
+        # this accounts for generations such as b = 1 with ground truth as x = b = 1
+        if str1.count("=") < str2.count("="):
+            return self._is_str_correct(str1, str2[str2.index("=") + 1 :])
+        if str1.count("=") > str2.count("="):
+            return self._is_str_correct(str1[str1.index("=") + 1 :], str2)
+        if str1 is None and str2 is None:
+            return True
+        if str1 is None or str2 is None:
+            return False
+        try:
+            return str1 == str2
+        except Exception:
+            return str1 == str2
+
+    def _is_symbolically_equiv(
+        self, response_list: list[str], ground_truths: list, default_invalid: Basic = S.NaN
+    ) -> bool:
+        """
+        Check if any of the responses are correct and return true at first match
+        :param response_list: list of responses
+        :param ground_truths: list of ground truths
+        :param default_invalid: default value for invalid expressions
+        :return: True if any response
+        """
+
+        try:
+            self._apply_safely(parse_latex, response_list)
+        except (LaTeXParsingError, SympifyError, TypeError):
+            response_list = [default_invalid]  # this can not occur as an answer.
+            return False
+
+        # map objects dont catch errors, so we use safe apply here
+        self._apply_safely(self._safe_simplify_expression, ground_truths)
+        self._apply_safely(self._safe_simplify_expression, response_list)
+
+        # check if any of the simplified responses match any of the simplified ground truths
+        try:
+            is_correct = self._any_symb_correct(response_list, ground_truths)
+            return is_correct
+        except ValueError:
+            return False

eval_framework/metrics/completion/niah_accuracy.py

@@ -0,0 +1,163 @@
+import re
+import unicodedata
+
+from eval_framework.metrics.base import (
+    BaseMetric,
+    MetricResult,
+)
+from eval_framework.shared.types import Completion, Error, LanguageMetricContext, extract_context_metric
+
+# Dictionary of "none" words in different languages
+NONE_DICT = {
+    "en": ["none"],
+    "ko": ["없음"],
+    "pl": ["brak"],
+    "zh": ["无"],
+    "vi": ["Không có"],
+    "ja": ["なし", "数字はありません"],
+    "ta": ["ஏதுமில்லை"],
+    "hu": ["nincs"],
+    "fr": ["aucun"],
+    "no": ["ingen"],
+    "uk": ["немає", "Нема"],
+    "ru": ["нет"],
+    "de": ["Keine vorhanden"],
+    "es": ["ninguno"],
+    "sv": ["inga"],
+    "fi": ["ei mikään"],
+    "cs": ["žádné", "žádná"],
+    "sr": ["nema"],
+    "pt": ["nenhum"],
+    "it": ["nessuno"],
+    "fa": ["هیچ کدام"],
+    "sw": ["hakuna"],
+    "nl": ["geen"],
+    "st": ["ha ho letho"],
+    "hi": ["कोई नहीं"],
+    "da": ["ingen"],
+}
+
+
+def clean_text(text: str) -> str:
+    """Clean text by removing spaces and normalizing"""
+    return text.strip().lower().replace("\u200c", "").replace(" ", "")
+
+
+class NIAHAccuracy(BaseMetric[Completion]):
+    """Metric for Needle in a Haystack tasks"""
+
+    NAME = "NIAHAccuracy"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        context = extract_context_metric(response, LanguageMetricContext)
+
+        ground_truths = [gt for gt in response.ground_truth_list if gt is not None]
+
+        try:
+            # Extract task and language from metadata
+            assert response.context is not None
+            language = context.language
+
+            # Get model's answer
+            model_answer = response.completion
+
+            # Determine which comparison function to use based on the task
+            none_values = set(v for values in NONE_DICT.values() for v in values)
+            if ground_truths[0] in none_values:
+                is_correct = self._compare_none(language, model_answer)
+            else:
+                is_correct = self._compare_numbers(language, ground_truths, model_answer)
+
+            return [
+                MetricResult(
+                    metric_name=self.NAME, value=float(is_correct), higher_is_better=True, error=response.error
+                )
+            ]
+        except Exception as e:
+            error = Error(error_class=e.__class__.__name__, message=str(e), traceback="")
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=error)]
+
+    def _compare_numbers(self, lang: str, correct_answer: list[str], model_answer: str) -> bool:
+        """Compare numbers for regular NIAH tasks"""
+        if "-" in lang:
+            inst_lang = lang.split("-")[1]
+        else:
+            inst_lang = lang
+
+        if not model_answer:
+            return False
+
+        processed_model_answer = unicodedata.normalize("NFKC", model_answer)
+
+        none_words = NONE_DICT.get(inst_lang, ["none"])
+        # Check if any word in none_words is present in the processed answer; if yes, auto-fail
+        for word in none_words:
+            if word in processed_model_answer or clean_text(word) in processed_model_answer:
+                return False
+
+        # Extract all numeric substrings from the processed answer
+        numeric_strings = re.findall(r"\d+", processed_model_answer)
+
+        # Remove numbers that consist of a single digit
+        numeric_strings = [num for num in numeric_strings if len(num) > 1]
+
+        # Remove duplicates while preserving the original order
+        numeric_strings = list(dict.fromkeys(numeric_strings))
+
+        # If no numerics are found after processing, return False
+        if not numeric_strings:
+            return False
+
+        # Convert the extracted number strings to integers
+        try:
+            extracted_numbers = [int(num) for num in numeric_strings]
+        except Exception:
+            return False
+
+        # Convert correct_answers elements to integers to ensure numeric comparison
+        try:
+            correct_converted = [int(item) for item in correct_answer]
+        except Exception:
+            return False
+
+        # Check that the number of extracted numbers matches the length of correct_answers
+        if len(extracted_numbers) != len(correct_converted):
+            return False
+
+        # Compare the extracted numbers with the correct answers
+        if set(extracted_numbers) == set(correct_converted):
+            return True
+        else:
+            return False
+
+    def _compare_none(self, lang: str, model_answer: str) -> bool:
+        """Compare for NIAH none tasks"""
+        # Lower-case all inputs for consistent, case-insensitive processing
+        if "-" in lang:
+            inst_lang = lang.split("-")[1]
+        else:
+            inst_lang = lang
+
+        processed_model_answer = clean_text(unicodedata.normalize("NFKC", model_answer))
+        none_words = [clean_text(word) for word in NONE_DICT[inst_lang]]
+
+        # Remove single digit numbers from the processed answer
+        processed_model_answer = re.sub(r"\b\d\b", "", processed_model_answer)
+
+        # Extract all multi-digit numeric substrings from the processed answer
+        numeric_strings = re.findall(r"\d\d+", processed_model_answer)
+
+        # If any multi-digit numbers are found, return False
+        if numeric_strings:
+            return False
+
+        # Check if any of the words in none_words are present
+        for word in none_words:
+            if word in processed_model_answer:
+                return True
+
+        # If none of the none_words are found, return False
+        return False

eval_framework/metrics/completion/placeholder_checker.py

@@ -0,0 +1,27 @@
+import re
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import BaseMetricContext, Completion, extract_context_metric
+
+
+class PlaceholderCheckerMetricContext(BaseMetricContext):
+    num_placeholders: int
+
+
+class PlaceholderChecker(BaseMetric[Completion]):
+    NAME = "Placeholder Check"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        context = extract_context_metric(response, PlaceholderCheckerMetricContext)
+
+        assert context.num_placeholders is not None, "Expected 'num_placeholders' in context"
+        assert isinstance(context.num_placeholders, int), (
+            f"'num_placeholders' has incorrect type: {type(context.num_placeholders)}"
+        )
+
+        placeholders = re.findall(r"\[.*?\]", response.completion)
+        value = float(len(placeholders) >= context.num_placeholders)
+        return [MetricResult(metric_name=self.NAME, value=value, higher_is_better=True, error=response.error)]