eval-framework 0.2.0__py3-none-any.whl

eval_framework/metrics/completion/math_reasoning_completion.py

@@ -0,0 +1,303 @@
+import re
+import signal
+from collections.abc import Callable, Iterable
+from typing import Any
+
+from sympy import Basic, S, SympifyError, factor, simplify
+from sympy.parsing.latex import parse_latex
+from sympy.parsing.latex.errors import LaTeXParsingError
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+def timeout_handler(signum: Any, frame: Any) -> None:
+    raise TimeoutError()
+
+
+class MathReasoningCompletion(BaseMetric[Completion]):
+    #
+    # Math Reasoning Completion (symbolic)
+    #
+    # This metric evaluates the correctness of the completion of a math reasoning task without
+    # correcting LaTeX expressions. Normalization occurs on the strings, only to remove formatting
+    # and units.
+    #
+    # The metric is designed to evaluate the correctness of the completion of a math reasoning task
+    # without correcting LaTeX expressions.
+    #
+
+    NAME = "Math Reasoning Completion (symbolic)"
+
+    # Substitutions to apply to the final answer
+    SUBSTITUTIONS = [
+        (r"\ban\b(?!\w)", ""),  # Remove "an" if not part of a word
+        (r"\ba\b(?!\w)", ""),  # Remove "a" if not part of a word
+        (r"\.\$", "$"),  # Replace ".$" with "$"
+        (r"\\\$", ""),  # Remove "\$"
+        (r"\\ ", ""),  # Remove "\ " (escaped space)
+        (r"\s+", ""),  # Remove all spaces
+        (r"\\mbox", "text"),  # Replace "\mbox" with "text"
+        (r",\\text\{and\}", ","),  # Replace ",\text{and}" with ","
+        (r"\\text\{and\}", ","),  # Replace "\text{and}" with ","
+        (r"\\text\{m\}", "\\text{}"),  # Replace "\text{m}" with "\text{}"
+    ]
+
+    # Expressions to remove from the final answer
+    # Most of these expressions omit units and formatting
+    # which the ground truth does not have
+    REMOVED_EXPRESSIONS_UNITS = [
+        "square",
+        "ways",
+        "integers",
+        "dollars",
+        "mph",
+        "inches",
+        "ft",
+        "hours",
+        "km",
+        "units",
+        "\\ldots",
+        "sue",
+        "points",
+        "feet",
+        "minutes",
+        "digits",
+        "cents",
+        "degrees",
+        "cm",
+        "gm",
+        "pounds",
+        "meters",
+        "meals",
+        "edges",
+        "students",
+        "childrentickets",
+        "multiples",
+    ]
+
+    REMOVED_EXPRESSIONS_FORMAT = [
+        "\\text{s}",
+        "\\text{.}",
+        "\\text{\ns}",
+        "\\text{}^2",
+        "\\text{}^3",
+        "\\text{\n}",
+        "\\text{}",
+        r"\mathrm{th}",
+        r"^\circ",
+        r"^{\circ}",
+        r"\;",
+        r",\!",
+        "{,}",
+        '"',
+        "\\dots",
+    ]
+
+    def normalize_expression(self, final_answer: str) -> str:
+        """
+        Function to normalize LaTeX expressions
+        :param final_answer: raw LaTeX expression
+        :return: normalized LaTeX expression
+        NOTE: Changed logic, because before the substitution randomly replaced characters in the string,
+        i.e., turned "infty" into "iny" by removing "ft"
+        """
+        for before, after in self.SUBSTITUTIONS:
+            final_answer = re.sub(before, after, final_answer)
+        for expr in self.REMOVED_EXPRESSIONS_UNITS:
+            # Safely remove units at the end, allowing optional space before the unit
+            final_answer = re.sub(rf"(.*?)\s*({re.escape(expr)})$", r"\1", final_answer)
+        for expr in self.REMOVED_EXPRESSIONS_FORMAT:
+            # Safely remove formatting expressions
+            final_answer = final_answer.replace(expr, "")
+        final_answer = re.sub(r"(.*?)(\$)(.*?)(\$)(.*)", r"$\3$", final_answer)
+        final_answer = re.sub(r"(\\text\{)(.*?)(\})", r"\2", final_answer)
+        final_answer = re.sub(r"(\\textbf\{)(.*?)(\})", r"\2", final_answer)
+        final_answer = re.sub(r"(\\overline\{)(.*?)(\})", r"\2", final_answer)
+        final_answer = re.sub(r"(\\boxed\{)(.*)(\})", r"\2", final_answer)
+        final_answer = re.sub(r"(frac)([^{])(.)", r"frac{\2}{\3}", final_answer)
+        final_answer = re.sub(r"(sqrt)([^{])", r"sqrt{\2}", final_answer)
+        final_answer = final_answer.replace("$", "")
+        # Only strip commas if it's a single numeric value with optional commas (like "1,000")
+        if re.fullmatch(r"\d{1,3}(,\d{3})*", final_answer):
+            final_answer = final_answer.replace(",", "")
+        return final_answer
+
+    def check_for_equation(self, final_answer: str) -> list:
+        """
+        Check if the final answer is an equation and split it into left hand side and right hand side
+        :param final_answer: the expression to evaluate
+        :return: list of left hand side and right hand side of the equation
+        """
+        if isinstance(final_answer, str) and "=" in final_answer:
+            return final_answer.split("=")
+        else:
+            return [final_answer]
+
+    def _safe_simplify_expression(self, expression: Basic, timeout: int = 10) -> Basic:
+        """
+        Simplify an expression with a timeout and catch recursion depth exception
+        :param expression: SymPy expression
+        :param timeout: Time limit in seconds (default: 10 seconds).
+        :return: simplified expressions
+        """
+        signal.signal(signal.SIGALRM, timeout_handler)  # Set timeout signal
+        signal.alarm(timeout)  # Set timeout duration
+
+        try:
+            factored = factor(expression)
+            simplified = simplify(factored)
+            signal.alarm(0)  # Disable timeout
+            return simplified
+        except (SympifyError, TimeoutError):
+            return S.NaN
+
+    def _any_symb_correct(self, response_list: Iterable[Basic], ground_truth_list: Iterable[Basic]) -> bool:
+        """
+        Check if any of the responses are correct and return true at first match
+        :param response_list: list of responses
+        :param ground_truth_list: list of ground truths
+        :return: True if any response is correct
+        """
+        for answer in response_list:
+            for ground_truth in ground_truth_list:
+                try:
+                    unsimplified_difference = answer - ground_truth
+                    # check if the difference is close to zero with numpy
+                    difference = self._safe_simplify_expression(unsimplified_difference)
+                    tolerance = 1e-12
+                    if abs(difference) < tolerance:
+                        return True
+                except ValueError:
+                    # equations cannot be evaluated against each other
+                    return False
+        return False
+
+    def _apply_safely(self, func: Callable[[Basic], Basic], list_of_expressions: list[Basic]) -> None:
+        """
+        apply safely to a list of expressions and replace the original expressions
+        :param list_of_expressions: list of sympy expressions
+        """
+        for i, expression in enumerate(list_of_expressions):
+            try:
+                list_of_expressions[i] = func(expression)
+            except RecursionError:
+                list_of_expressions[i] = S.NaN
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        """
+        Calculate the accuracy of the completion
+
+        performs several verification and simplification steps
+        to ensure that the completion is correct
+
+        the completion may either be a latex or string response
+        which sympy will parse, factor, and simplify
+
+        :param response: Completion object
+        :return: list of MetricResult
+        """
+        ground_truths = []
+        INVALID_ANSWER = S.NaN
+        timeout = 10
+        # latex parse all ingested ground truth values for math reasoning
+        for gt in response.ground_truth_list:
+            signal.signal(signal.SIGALRM, timeout_handler)  # Set timeout signal
+            signal.alarm(timeout)  # Set timeout duration
+            try:
+                gt_parsed = parse_latex(gt)  # NOTE: parses f(x)=0,\quadf(x)=x-1,\quadf(x)=-x+1 to Eq(f(x), 0) ONLY
+                ground_truths.append(gt_parsed)
+                signal.alarm(0)
+            except Exception:
+                ground_truths.append(gt)
+        normalized_response = self.normalize_expression(response.completion)
+        response_list = self.check_for_equation(normalized_response)
+        try:
+            symb_is_correct = self._is_symbolically_equiv(response_list, ground_truths, INVALID_ANSWER)
+        except Exception:
+            symb_is_correct = False
+
+        # check if already correct symbolically
+        if symb_is_correct:
+            return [
+                MetricResult(
+                    metric_name=self.NAME, value=float(symb_is_correct), higher_is_better=True, error=response.error
+                )
+            ]
+        else:
+            # fall back to string comparison
+            # ground truth can be list or str, we have str comparisons
+            assert isinstance(response.ground_truth, str)
+            str_is_correct = self._is_str_correct(normalized_response, response.ground_truth)
+            return [
+                MetricResult(
+                    metric_name=self.NAME, value=float(str_is_correct), higher_is_better=True, error=response.error
+                )
+            ]
+
+    def _any_str_correct(self, response_list: list, ground_truths: list) -> bool:
+        """
+        Check if any of the responses are correct and return true at first match
+        :param response_list: list of responses
+        :param ground_truths: list of ground truths
+        :return: True if any response is correct
+        """
+        for response in response_list:
+            for ground_truth in ground_truths:
+                if self._is_str_correct(response, ground_truth):
+                    return True
+        return False
+
+    def _is_str_correct(self, str1: str, str2: str) -> bool:
+        """
+        Check if two strings are equal after stripping
+        :param str1: first string
+        :param str2: second string
+        :param verbose: print the stripped strings
+        :return: True if the strings are equal
+        """
+        # if multiple equal signs in ground truth (str2)
+        # slide the response (str1) over the ground truth (str2)
+        # at the interval of every equal sign in the ground truth
+        # and check if any of the responses match
+        # this accounts for generations such as b = 1 with ground truth as x = b = 1
+        if str1.count("=") < str2.count("="):
+            return self._is_str_correct(str1, str2[str2.index("=") + 1 :])
+        if str1.count("=") > str2.count("="):
+            return self._is_str_correct(str1[str1.index("=") + 1 :], str2)
+        if str1 is None and str2 is None:
+            return True
+        if str1 is None or str2 is None:
+            return False
+        try:
+            return str1 == str2
+        except Exception:
+            return str1 == str2
+
+    def _is_symbolically_equiv(
+        self, response_list: list[str], ground_truths: list, default_invalid: Basic = S.NaN
+    ) -> bool:
+        """
+        Check if any of the responses are correct and return true at first match
+        :param response_list: list of responses
+        :param ground_truths: list of ground truths
+        :param default_invalid: default value for invalid expressions
+        :return: True if any response
+        """
+
+        try:
+            self._apply_safely(parse_latex, response_list)
+        except (LaTeXParsingError, SympifyError, TypeError):
+            response_list = [default_invalid]  # this can not occur as an answer.
+            return False
+
+        # map objects dont catch errors, so we use safe apply here
+        self._apply_safely(self._safe_simplify_expression, ground_truths)
+        self._apply_safely(self._safe_simplify_expression, response_list)
+
+        # check if any of the simplified responses match any of the simplified ground truths
+        try:
+            is_correct = self._any_symb_correct(response_list, ground_truths)
+            return is_correct
+        except ValueError:
+            return False

eval_framework/metrics/completion/niah_accuracy.py

@@ -0,0 +1,163 @@
+import re
+import unicodedata
+
+from eval_framework.metrics.base import (
+    BaseMetric,
+    MetricResult,
+)
+from eval_framework.shared.types import Completion, Error, LanguageMetricContext, extract_context_metric
+
+# Dictionary of "none" words in different languages
+NONE_DICT = {
+    "en": ["none"],
+    "ko": ["없음"],
+    "pl": ["brak"],
+    "zh": ["无"],
+    "vi": ["Không có"],
+    "ja": ["なし", "数字はありません"],
+    "ta": ["ஏதுமில்லை"],
+    "hu": ["nincs"],
+    "fr": ["aucun"],
+    "no": ["ingen"],
+    "uk": ["немає", "Нема"],
+    "ru": ["нет"],
+    "de": ["Keine vorhanden"],
+    "es": ["ninguno"],
+    "sv": ["inga"],
+    "fi": ["ei mikään"],
+    "cs": ["žádné", "žádná"],
+    "sr": ["nema"],
+    "pt": ["nenhum"],
+    "it": ["nessuno"],
+    "fa": ["هیچ کدام"],
+    "sw": ["hakuna"],
+    "nl": ["geen"],
+    "st": ["ha ho letho"],
+    "hi": ["कोई नहीं"],
+    "da": ["ingen"],
+}
+
+
+def clean_text(text: str) -> str:
+    """Clean text by removing spaces and normalizing"""
+    return text.strip().lower().replace("\u200c", "").replace(" ", "")
+
+
+class NIAHAccuracy(BaseMetric[Completion]):
+    """Metric for Needle in a Haystack tasks"""
+
+    NAME = "NIAHAccuracy"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        context = extract_context_metric(response, LanguageMetricContext)
+
+        ground_truths = [gt for gt in response.ground_truth_list if gt is not None]
+
+        try:
+            # Extract task and language from metadata
+            assert response.context is not None
+            language = context.language
+
+            # Get model's answer
+            model_answer = response.completion
+
+            # Determine which comparison function to use based on the task
+            none_values = set(v for values in NONE_DICT.values() for v in values)
+            if ground_truths[0] in none_values:
+                is_correct = self._compare_none(language, model_answer)
+            else:
+                is_correct = self._compare_numbers(language, ground_truths, model_answer)
+
+            return [
+                MetricResult(
+                    metric_name=self.NAME, value=float(is_correct), higher_is_better=True, error=response.error
+                )
+            ]
+        except Exception as e:
+            error = Error(error_class=e.__class__.__name__, message=str(e), traceback="")
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=error)]
+
+    def _compare_numbers(self, lang: str, correct_answer: list[str], model_answer: str) -> bool:
+        """Compare numbers for regular NIAH tasks"""
+        if "-" in lang:
+            inst_lang = lang.split("-")[1]
+        else:
+            inst_lang = lang
+
+        if not model_answer:
+            return False
+
+        processed_model_answer = unicodedata.normalize("NFKC", model_answer)
+
+        none_words = NONE_DICT.get(inst_lang, ["none"])
+        # Check if any word in none_words is present in the processed answer; if yes, auto-fail
+        for word in none_words:
+            if word in processed_model_answer or clean_text(word) in processed_model_answer:
+                return False
+
+        # Extract all numeric substrings from the processed answer
+        numeric_strings = re.findall(r"\d+", processed_model_answer)
+
+        # Remove numbers that consist of a single digit
+        numeric_strings = [num for num in numeric_strings if len(num) > 1]
+
+        # Remove duplicates while preserving the original order
+        numeric_strings = list(dict.fromkeys(numeric_strings))
+
+        # If no numerics are found after processing, return False
+        if not numeric_strings:
+            return False
+
+        # Convert the extracted number strings to integers
+        try:
+            extracted_numbers = [int(num) for num in numeric_strings]
+        except Exception:
+            return False
+
+        # Convert correct_answers elements to integers to ensure numeric comparison
+        try:
+            correct_converted = [int(item) for item in correct_answer]
+        except Exception:
+            return False
+
+        # Check that the number of extracted numbers matches the length of correct_answers
+        if len(extracted_numbers) != len(correct_converted):
+            return False
+
+        # Compare the extracted numbers with the correct answers
+        if set(extracted_numbers) == set(correct_converted):
+            return True
+        else:
+            return False
+
+    def _compare_none(self, lang: str, model_answer: str) -> bool:
+        """Compare for NIAH none tasks"""
+        # Lower-case all inputs for consistent, case-insensitive processing
+        if "-" in lang:
+            inst_lang = lang.split("-")[1]
+        else:
+            inst_lang = lang
+
+        processed_model_answer = clean_text(unicodedata.normalize("NFKC", model_answer))
+        none_words = [clean_text(word) for word in NONE_DICT[inst_lang]]
+
+        # Remove single digit numbers from the processed answer
+        processed_model_answer = re.sub(r"\b\d\b", "", processed_model_answer)
+
+        # Extract all multi-digit numeric substrings from the processed answer
+        numeric_strings = re.findall(r"\d\d+", processed_model_answer)
+
+        # If any multi-digit numbers are found, return False
+        if numeric_strings:
+            return False
+
+        # Check if any of the words in none_words are present
+        for word in none_words:
+            if word in processed_model_answer:
+                return True
+
+        # If none of the none_words are found, return False
+        return False

eval_framework/metrics/completion/placeholder_checker.py

@@ -0,0 +1,27 @@
+import re
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import BaseMetricContext, Completion, extract_context_metric
+
+
+class PlaceholderCheckerMetricContext(BaseMetricContext):
+    num_placeholders: int
+
+
+class PlaceholderChecker(BaseMetric[Completion]):
+    NAME = "Placeholder Check"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        context = extract_context_metric(response, PlaceholderCheckerMetricContext)
+
+        assert context.num_placeholders is not None, "Expected 'num_placeholders' in context"
+        assert isinstance(context.num_placeholders, int), (
+            f"'num_placeholders' has incorrect type: {type(context.num_placeholders)}"
+        )
+
+        placeholders = re.findall(r"\[.*?\]", response.completion)
+        value = float(len(placeholders) >= context.num_placeholders)
+        return [MetricResult(metric_name=self.NAME, value=value, higher_is_better=True, error=response.error)]

eval_framework/metrics/completion/repetition.py

@@ -0,0 +1,88 @@
+import re
+from collections import Counter
+from collections.abc import Sequence
+from typing import Final
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class WordRepetition(BaseMetric[Completion]):
+    """Word Repetition Metric
+
+    This metric checks for repetitions of words in the completion text for a
+    given window size and repetition threshold. The window size defines the
+    consecutive word count to consider a repetition, and min_repetitions
+    specifies the minimum repetition count that triggers the metric. This metric
+    returns 0.0 if no repetitions are found, and 1.0 if a sufficient number of
+    repetitions are found. For example, if the completion contains a two-word
+    sequence that repeats once (such as "hello world hello world"), this metric
+    would trigger with a window size of 2 and min_repetitions set to 1.
+    """
+
+    NAME = "WordRepetition"
+    HIGHER_IS_BETTER: Final[bool] = False
+
+    def __init__(self, window_size: int = 128, min_repetitions: int = 1) -> None:
+        """
+        Initialize the WordRepetition metric.
+
+        Args:
+            window_size (int): The number of consecutive words to consider as a
+                sequence.
+            min_repetitions (int): The minimum number of times a sequence must
+                repeat to be considered a repetition. Set to 1 to catch any
+                repetition.
+        """
+        super().__init__()
+        self.window_size = window_size
+        self.min_repetitions = min_repetitions
+
+        if self.min_repetitions < 1:
+            raise ValueError("min_repetitions must be at least 1")
+
+        if self.window_size < 1:
+            raise ValueError("window_size must be at least 1")
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [
+                MetricResult(
+                    metric_name=self.NAME,
+                    value=None,
+                    higher_is_better=self.HIGHER_IS_BETTER,
+                    error=response.error,
+                )
+            ]
+
+        has_repetition = _has_repetition(
+            text=response.completion,
+            window_size=self.window_size,
+            min_repetitions=self.min_repetitions,
+        )
+
+        return [
+            MetricResult(
+                metric_name=self.NAME,
+                value=float(has_repetition),
+                higher_is_better=self.HIGHER_IS_BETTER,
+                error=response.error,
+            )
+        ]
+
+
+def _has_repetition(text: str, window_size: int, min_repetitions: int) -> bool:
+    """Check if the text contains any word sequences of a given size that repeat"""
+    sequences = _word_sequences(_to_words(text), window_size)
+    counts = Counter(sequences)
+    return any([count > min_repetitions for count in counts.values()])
+
+
+def _to_words(text: str) -> Sequence[str]:
+    """A somewhat crude function to tokenize a string into words."""
+    return re.findall(r"\b\w+\b", text, re.UNICODE)
+
+
+def _word_sequences(text_words: Sequence[str], window_size: int) -> Sequence[Sequence[str]]:
+    """Get all contiguous sub-sequences of a given size from a word sequence."""
+    return [tuple(text_words[i : i + window_size]) for i in range(len(text_words) - window_size + 1)]

eval_framework/metrics/completion/rouge_1.py

@@ -0,0 +1,35 @@
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.metrics.completion.f1 import calculate_f1
+from eval_framework.shared.types import Completion
+
+
+class ROUGE_1(BaseMetric[Completion]):
+    """ROUGE-1"""
+
+    NAME = "ROUGE-1"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        if response.completion == "":
+            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
+        if None in response.ground_truth_list:
+            raise LogicError("When calculating ROUGE-1 ground_truth cannot be None.")
+
+        # ROUGE-1 captures word sequence similarity by focusing on unigrams
+        rouge = max([_calculate_rouge_1(response.completion, gt) for gt in response.ground_truth_list])  # type: ignore[arg-type]
+        return [MetricResult(metric_name=self.NAME, value=float(rouge), higher_is_better=True, error=response.error)]
+
+
+def _calculate_rouge_1(candidate: str, reference: str) -> float:
+    """
+    Calculate ROUGE-1 precision, recall, and F1 score between candidate and reference texts.
+    """
+
+    # Tokenize the candidate and reference summaries
+    candidate_tokens = candidate.split()
+    reference_tokens = reference.split()
+
+    return calculate_f1(reference_tokens, candidate_tokens)

eval_framework/metrics/completion/rouge_2.py

@@ -0,0 +1,45 @@
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.metrics.completion.f1 import calculate_f1
+from eval_framework.shared.types import Completion
+
+
+class ROUGE_2(BaseMetric[Completion]):
+    """ROUGE-2"""
+
+    NAME = "ROUGE-2"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        if response.completion == "":
+            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
+        if None in response.ground_truth_list:
+            raise LogicError("When calculating ROUGE-2 ground_truth cannot be None.")
+
+        # ROUGE-2 captures word sequence similarity by focusing on bigrams,
+        # which makes it sensitive to the order and co-occurrence of words to some extent.
+        rouge = max([_calculate_rouge_2(response.completion, gt) for gt in response.ground_truth_list])  # type: ignore[arg-type]
+        return [MetricResult(metric_name=self.NAME, value=float(rouge), higher_is_better=True, error=response.error)]
+
+
+def _generate_bigrams(tokens: list[str]) -> list[tuple[str, str]]:
+    """Generate bigrams from a list of tokens."""
+    return [(tokens[i], tokens[i + 1]) for i in range(len(tokens) - 1)]
+
+
+def _calculate_rouge_2(completion: str, ground_truth: str) -> float:
+    """
+    Calculate ROUGE-2 precision, recall, and F1 score between candidate and reference texts.
+    """
+
+    # Tokenize the candidate and reference summaries
+    candidate_tokens = completion.split()
+    reference_tokens = ground_truth.split()
+
+    # Generate bigrams for candidate and reference
+    candidate_bigrams = _generate_bigrams(candidate_tokens)
+    reference_bigrams = _generate_bigrams(reference_tokens)
+
+    return calculate_f1(reference_bigrams, candidate_bigrams)