eval-framework 0.2.0__py3-none-any.whl

eval_framework/metrics/completion/bleu.py

@@ -0,0 +1,76 @@
+import sacrebleu
+
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class BLEU(BaseMetric[Completion]):
+    """The Bilingual Evaluation Understudy Score, or BLEU for short, is a metric
+    for evaluating a generated sentence to a reference sentence. It counts matching
+    n-grams in the candidate translation to n-grams in the reference text, where
+    1-gram or unigram would be each token and a bigram comparison would be each
+    word pair. The comparison is made regardless of word order
+    Source: https://machinelearningmastery.com/calculate-bleu-score-for-text-python/
+    Paper: https://www.aclweb.org/anthology/P02-1040/
+    """
+
+    NAME = "BLEU"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        scores = []
+        for ground_truth in response.ground_truth_list:
+            if ground_truth == "" or ground_truth is None:
+                raise LogicError("When calculating BLEU we need a ground truth.")
+
+            sacre_formatted_completion = [response.completion]
+            sacre_formatted_ground_truth = [[ground_truth]]
+            scores.append(sacrebleu.corpus_bleu(sacre_formatted_completion, sacre_formatted_ground_truth).score)
+
+        return [
+            MetricResult(metric_name=self.NAME, value=float(max(scores)), higher_is_better=True, error=response.error)
+        ]
+
+
+class LINEWISE_BLEU(BaseMetric[Completion]):
+    """Maximum Line-level BLEU score."""
+
+    NAME = "Linewise BLEU"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        scores = []
+        for ground_truth in response.ground_truth_list:
+            for sentence in response.completion.split("\n"):
+                if sentence == "":
+                    continue
+
+                if ground_truth == "" or ground_truth is None:
+                    raise LogicError("When calculating BLEU we need a ground truth.")
+
+                sacre_formatted_completion = [sentence]
+                sacre_formatted_ground_truth = [[ground_truth]]
+                scores.append(sacrebleu.corpus_bleu(sacre_formatted_completion, sacre_formatted_ground_truth).score)
+
+        return [
+            MetricResult(
+                metric_name=self.NAME, value=float(max(scores, default=0)), higher_is_better=True, error=response.error
+            )
+        ]
+
+
+class ResponseToOriginalBLEU(BaseMetric[Completion]):
+    NAME = "Response to Original BLEU"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        score = sacrebleu.corpus_bleu([response.completion], [[response.last_user_instruction]]).score
+        # scaled to [0, 1] to make aggregation easier
+        return [MetricResult(metric_name=self.NAME, value=score / 100, higher_is_better=True, error=response.error)]

eval_framework/metrics/completion/chrf.py

@@ -0,0 +1,62 @@
+import sacrebleu
+
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class CHRF(BaseMetric[Completion]):
+    """chrF++ is a tool for automatic evaluation of machine translation output
+    based on character n-gram precision and recall enhanced with word n-grams.
+    Source: https://github.com/m-popovic/chrF
+    Paper: https://www.aclweb.org/anthology/W15-3049.pdf
+    """
+
+    NAME = "chrF"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        scores = []
+        for ground_truth in response.ground_truth_list:
+            if ground_truth == "" or ground_truth is None:
+                raise LogicError("When calculating chrF we need a ground truth.")
+
+            sacre_formatted_completion = [response.completion]
+            sacre_formatted_ground_truth = [[ground_truth]]
+            scores.append(sacrebleu.corpus_chrf(sacre_formatted_completion, sacre_formatted_ground_truth).score)
+
+        return [
+            MetricResult(metric_name=self.NAME, value=float(max(scores)), higher_is_better=True, error=response.error)
+        ]
+
+
+class LINEWISE_CHRF(BaseMetric[Completion]):
+    """Maximum Line-level chrF++ (Character n-gram F-score) score.
+    Paper: https://aclanthology.org/W15-3049/"""
+
+    NAME = "Linewise chrF"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        scores = []
+        for ground_truth in response.ground_truth_list:
+            for sentence in response.completion.split("\n"):
+                if sentence == "":
+                    continue
+
+                if ground_truth == "" or ground_truth is None:
+                    raise LogicError("When calculating chrF we need a ground truth.")
+
+                sacre_formatted_completion = [sentence]
+                sacre_formatted_ground_truth = [[ground_truth]]
+                scores.append(sacrebleu.corpus_chrf(sacre_formatted_completion, sacre_formatted_ground_truth).score)
+
+        return [
+            MetricResult(
+                metric_name=self.NAME, value=float(max(scores, default=0)), higher_is_better=True, error=response.error
+            )
+        ]

eval_framework/metrics/completion/code_assertion.py

@@ -0,0 +1,44 @@
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion, Error
+from eval_framework.tasks.utils import run_python_code
+
+
+class CodeCompletionAssertion(BaseMetric[Completion]):
+    NAME = "Code Completion Accuracy"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        # this will always be a list, if return is "" this will be an empty list
+        code = response.completion
+        output = run_python_code(code, image="python:3.12-slim")
+
+        # Split and filter out empty strings
+        output_parts = [part for part in output.split() if part.strip()]
+
+        if not output_parts:
+            last_output = ""
+        else:
+            last_output = output_parts[-1]
+
+        success = last_output == "True"
+        error = (
+            None
+            if success
+            else Error(
+                error_class="CodeCompletionAssertionError",
+                message=f"Expected 'True' but got '{last_output}'",
+                traceback=output,
+            )
+        )
+
+        return [
+            MetricResult(
+                metric_name=self.NAME,
+                value=1.0 if success else 0.0,
+                higher_is_better=True,
+                error=error,
+                code_execution_trace=output,
+            )
+        ]

eval_framework/metrics/completion/code_execution_pass_at_one.py

@@ -0,0 +1,126 @@
+import traceback
+from collections.abc import Callable
+from typing import Self
+
+from pydantic import Field
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import BaseMetricContext, Completion, Error, extract_context_metric
+from eval_framework.tasks.utils import CallableSerializer, ExecutionResult, execute_python_code_with_tests
+
+
+class CodeExecutionBaseContext(BaseMetricContext):
+    run_env: str = Field(description="Name of docker image to run unit-tests inside")
+    code_prompt: str = Field(description="Prompt to LLM for code generation")
+    test_code: str = Field(description="Python code that contains logic for unit test execution")
+    benchmark_timeout: int = Field(default=60, description="Time in seconds for the full test execution run")
+    package_downloads: dict[str, str | None] = Field(
+        description="a dictionary listing the packages and their respective names in PyPiinto the LLM sandbox"
+    )
+
+
+class CodeExecutionPassAtOneContext(CodeExecutionBaseContext):
+    snippet_merge_fn: str = Field(
+        description="logic for merging LLM generated code with test execution code;"
+        "this code will be passed into the sandbox to run the testing process"
+        "This code is serialized"
+    )
+    output_parse_fn: str = Field(
+        description="logic for parsing the output of test code execution run within the LLM sandbox"
+        "This code is serialized"
+    )
+
+
+class RealtimeCodeExectionContext(CodeExecutionBaseContext):
+    snippet_merge_fn: Callable[[str, str], str] = Field(
+        description="logic for merging LLM generated code with test execution code;"
+        "this code will be passed into the sandbox to run the testing process"
+        "This code is deserialized"
+    )
+    output_parse_fn: Callable[[str], ExecutionResult] = Field(
+        description="logic for parsing the output of test code execution run within the LLM sandbox"
+        "This code is deserialized"
+    )
+
+    @classmethod
+    def from_context(cls, context: CodeExecutionPassAtOneContext) -> Self:
+        return cls(
+            run_env=context.run_env,
+            code_prompt=context.code_prompt,
+            test_code=context.test_code,
+            benchmark_timeout=context.benchmark_timeout,
+            snippet_merge_fn=CallableSerializer.decode(context.snippet_merge_fn),
+            output_parse_fn=CallableSerializer.decode(context.output_parse_fn),
+            package_downloads=context.package_downloads,
+        )
+
+
+class CodeExecutionPassAtOne(BaseMetric[Completion]):
+    NAME = "code-execution-pass@1"
+
+    def __init__(self) -> None:
+        self.k = 1
+        # NOTE : this serializer should be the same class as initialized in the benchmark
+        self.serializer = CallableSerializer()
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+        try:
+            context = extract_context_metric(response, CodeExecutionPassAtOneContext)
+            parsed_context = RealtimeCodeExectionContext.from_context(context)
+        except Exception as e:
+            raise Exception(f"Failed to rebuild parsing functions => {e}")
+
+        n = 1  # we only support N=1 at the moment
+        try:
+            c, output = self._count_correct_samples(response.completion, parsed_context)
+        except Exception as e:
+            error = Error(error_class=e.__class__.__name__, message=str(e), traceback=traceback.format_exc())
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=error)]
+
+        pass_at_k_value = estimate_pass_at_k(n, c, self.k)
+        return [
+            MetricResult(
+                metric_name=self.NAME,
+                value=pass_at_k_value,
+                higher_is_better=True,
+                error=response.error,
+                code_execution_trace=output,
+            )
+        ]
+
+    def _count_correct_samples(self, completion: str, context: RealtimeCodeExectionContext) -> tuple[int, str]:
+        result = execute_python_code_with_tests(
+            code=completion,
+            test_code=context.test_code,
+            package_mapping=context.package_downloads,
+            merge_code_fn=context.snippet_merge_fn,
+            image=context.run_env,
+            timeout=context.benchmark_timeout,
+            parse_output_fn=context.output_parse_fn,
+        )
+        return (1 if result.success else 0), result.output
+
+
+def estimate_pass_at_k(n: int, c: int, k: int) -> float:
+    """
+    Estimates pass@k for a single problem.
+
+    Parameters:
+    n (int): Total number of generated samples.
+    c (int): Number of correct samples.
+    k (int): Number of attempts or samples considered.
+
+    Returns:
+    float: The pass@k value.
+    """
+    if n - c < k:
+        return 1.0
+
+    # Calculate the probability that at least one of the k samples is correct
+    probability = 1.0
+    for i in range(k):
+        probability *= (n - c - i) / (n - i)
+
+    return 1.0 - probability

eval_framework/metrics/completion/comet.py

@@ -0,0 +1,56 @@
+import torch
+from comet import download_model, load_from_checkpoint
+
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion, UntemplatedPrompt
+from eval_framework.utils.constants import ROOT_DIR
+
+SAVING_DIR = ROOT_DIR / "comet_model"
+
+
+class COMET(BaseMetric[Completion]):
+    """COMET is a neural, multilingual framework for evaluating machine translation quality by leveraging cross-lingual
+    pretrained language models to achieve state-of-the-art correlation with human judgments
+    Note: this requires a Hugging Face token with access to the model: https://huggingface.co/Unbabel/XCOMET-XL
+    Source: https://github.com/Unbabel/COMET
+    Paper: https://arxiv.org/abs/2009.09025
+    """
+
+    NAME = "COMET"
+
+    def __init__(self) -> None:
+        checkpoint_path = download_model("Unbabel/XCOMET-XL", saving_directory=SAVING_DIR)
+        self.model = load_from_checkpoint(checkpoint_path)
+        assert torch.cuda.is_available(), "COMET requires a GPU"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        if (
+            response.context is None
+            or not isinstance(response.context, UntemplatedPrompt)
+            or response.context.untemplated_prompt == ""
+        ):
+            raise LogicError("When calculating COMET we need an untemplated prompt.")
+
+        scores = []
+        for ground_truth in response.ground_truth_list:
+            if ground_truth == "" or ground_truth is None:
+                raise LogicError("When calculating COMET we need a ground truth.")
+
+            data = [
+                {
+                    "src": response.context.untemplated_prompt.strip(),
+                    "mt": response.completion.strip(),
+                    "ref": ground_truth.strip(),
+                },
+            ]
+            with torch.no_grad():
+                model_output = self.model.predict(data, gpus=1)
+            scores.append(model_output.system_score)
+
+        return [
+            MetricResult(metric_name=self.NAME, value=float(max(scores)), higher_is_better=True, error=response.error)
+        ]

eval_framework/metrics/completion/concordance_index.py

@@ -0,0 +1,38 @@
+import ast
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class ConcordanceIndex(BaseMetric[Completion]):
+    NAME = "ConcordanceIndex"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        ground_truths = [gt for gt in response.ground_truth_list if gt is not None]
+        if not ground_truths:
+            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
+
+        concordance_count = max([calculate_concordance_index(gt, response.completion) for gt in ground_truths])
+        return [
+            MetricResult(metric_name=self.NAME, value=concordance_count, higher_is_better=True, error=response.error)
+        ]
+
+
+def calculate_concordance_index(
+    ground_truth: str,
+    completion: str,
+) -> float:
+    ground_truth_arr = ast.literal_eval(ground_truth)
+    completion_arr = ast.literal_eval(completion)
+
+    if len(ground_truth_arr) != len(completion_arr):
+        return 0
+
+    concordance_count = 0
+    for gt, c in zip(ground_truth_arr, completion_arr):
+        concordance_count += 1 if gt == c else 0
+
+    return concordance_count / len(ground_truth_arr)

eval_framework/metrics/completion/csv_format.py

@@ -0,0 +1,102 @@
+import json
+
+from pydantic import BaseModel
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+SEPARATOR_MAP = {"comma": ",", "semicolon": ";", "space": " ", "tab": "\t"}
+
+
+class CSVFormatEvaluation(BaseModel):
+    implicit: bool = False
+    has_csv: bool = False
+    is_separator_respected: bool = False
+    is_column_count_respected: bool = False
+
+
+class CSVFormat(BaseMetric[Completion]):
+    NAME = "CSV Format"
+    KEYS = ["has_csv", "is_separator_respected", "is_column_count_respected"]
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [
+                MetricResult(metric_name=f"{self.NAME}/{k}", value=None, higher_is_better=True, error=response.error)
+                for k in self.KEYS
+            ]
+
+        if response.completion == "":
+            return [
+                MetricResult(metric_name=f"{self.NAME}/{k}", value=0.0, higher_is_better=True, error=response.error)
+                for k in self.KEYS
+            ]
+
+        grading = evaluate_csv_format(response)
+
+        results = []
+        for key in self.KEYS:
+            result = MetricResult(
+                metric_name=f"{self.NAME}/{key}",
+                value=float(getattr(grading, key)),
+                higher_is_better=True,
+                error=response.error,
+            )
+            results.append(result)
+        return results
+
+
+def extract_csv_from_text(text: str, min_rows: int = 2, min_columns: int = 2) -> tuple[list[str] | None, str | None]:
+    lines = text.split("\n")
+    delimiters = set(SEPARATOR_MAP.values())
+    best_delimiter = None
+    csv_lines: list[str] = []
+
+    # Iterate over lines to find potential delimiters and consistent substring counts
+    for i, line in enumerate(lines):
+        for delimiter in delimiters:
+            substrings = line.split(delimiter)
+            if len(substrings) < min_columns:
+                continue
+
+            current_csv_lines = [line]
+            for j in range(i + 1, len(lines)):
+                next_line = lines[j]
+                next_substrings = next_line.split(delimiter)
+                if len(next_substrings) != len(substrings):
+                    break
+                current_csv_lines.append(next_line)
+            if len(current_csv_lines) >= min_rows and len(current_csv_lines) > len(csv_lines):
+                best_delimiter = delimiter
+                csv_lines = current_csv_lines
+
+    if not csv_lines:
+        return None, None
+
+    return csv_lines, best_delimiter
+
+
+def evaluate_csv_format(response: Completion) -> CSVFormatEvaluation:
+    expected_output = json.loads(str(response.ground_truth))
+
+    expected_separator_code = expected_output["separator"]
+    csv_lines, separator = extract_csv_from_text(response.completion)
+
+    if not csv_lines:
+        return CSVFormatEvaluation(has_csv=False, implicit=not expected_separator_code)
+
+    csv_format_evaluation = CSVFormatEvaluation(has_csv=True, implicit=not expected_separator_code)
+
+    if not expected_separator_code:
+        csv_format_evaluation.is_separator_respected = separator in SEPARATOR_MAP.values()
+    else:
+        csv_format_evaluation.is_separator_respected = separator == SEPARATOR_MAP.get(expected_separator_code)
+
+    expected_column_count = len(expected_output["columns"])
+    column_counts = [len(csv_lines.split(separator)) for csv_lines in csv_lines]
+
+    csv_format_evaluation.is_column_count_respected = all(
+        column_count == expected_column_count for column_count in column_counts
+    )
+
+    return csv_format_evaluation

eval_framework/metrics/completion/cwe_accuracy.py

@@ -0,0 +1,49 @@
+import re
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion, Error
+
+
+class CWEAccuracy(BaseMetric[Completion]):
+    """Metric for Common Word Extraction tasks"""
+
+    NAME = "CWEAccuracy"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        ground_truths = [gt for gt in response.ground_truth_list if gt is not None]
+        if not ground_truths:
+            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
+
+        try:
+            # Get model's answer
+            model_answer = response.completion
+
+            # Check if all words in the correct answer are present in the model's answer
+            is_correct = self._is_answer_correct(ground_truths, model_answer)
+
+            return [
+                MetricResult(
+                    metric_name=self.NAME, value=1.0 if is_correct else 0.0, higher_is_better=True, error=response.error
+                )
+            ]
+        except Exception as e:
+            error = Error(error_class=e.__class__.__name__, message=str(e), traceback="")
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=error)]
+
+    def _is_answer_correct(self, correct_answer: list[str], model_answer: str) -> bool:
+        """Check if all words in correct_answer are present in model_answer as whole words"""
+        model_answer = model_answer.strip().lower()
+        correct_answer = [correct.strip().lower() for correct in correct_answer]
+
+        # For each word in the correct answer, check if it exists as a whole word in the model answer
+        for word in correct_answer:
+            # Create a regex pattern that matches the word as a whole word
+            # \b represents a word boundary
+            pattern = r"\b" + re.escape(word) + r"\b"
+            if not re.search(pattern, model_answer):
+                return False
+
+        return True

eval_framework/metrics/completion/exponential_similarity.py

@@ -0,0 +1,65 @@
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion, Error
+
+
+class ExponentialSimilarity(BaseMetric[Completion]):
+    NAME = "ExponentialSimilarity"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        ground_truths = [gt for gt in response.ground_truth_list if gt is not None]
+        if not ground_truths:
+            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
+
+        try:
+            # Try to calculate exponential similarity for each ground truth
+            similarities = []
+            for gt in ground_truths:
+                try:
+                    gt_float = float(gt)
+                    completion_float = float(response.completion)
+                    similarities.append(calculate_exponential_similarity(gt_float, completion_float))
+                except (ValueError, TypeError):
+                    # Skip this ground truth if conversion fails
+                    continue
+
+            # If we have any valid similarities, return the max
+            if similarities:
+                return [
+                    MetricResult(
+                        metric_name=self.NAME, value=max(similarities), higher_is_better=True, error=response.error
+                    )
+                ]
+            else:
+                # If all conversions failed, return an error
+                error = Error(
+                    error_class="ValueError",
+                    message="Could not convert ground truth or completion to float",
+                    traceback="",
+                )
+                return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=error)]
+        except Exception as e:
+            error = Error(error_class=e.__class__.__name__, message=str(e), traceback="")
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=error)]
+
+
+def calculate_exponential_similarity(p_true: float, p_pred: float) -> float:
+    """
+    Compute the exponential similarity (SpaceDigest version) between
+    the gold percentage and predicted value.
+
+    Parameters:
+    - p_true (float): The gold/reference percentage.
+    - p_pred (float): The predicted scalar.
+    - d (float): Base of the exponent. Default is 2.
+    - c (float): Coefficient in exponent. Default is 10.
+
+    Returns:
+    - float: Similarity score between 0 and 1.
+    """
+    d = 2
+    c = 10
+
+    return d ** (-c * abs(p_true / 100 - p_pred / 100))

eval_framework/metrics/completion/f1.py

@@ -0,0 +1,42 @@
+from collections import Counter
+from typing import Any
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class F1(BaseMetric[Completion]):
+    NAME = "F1"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        ground_truths = [gt for gt in response.ground_truth_list if gt is not None]
+        if not ground_truths:
+            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
+
+        hyp_tokens = response.completion.lower().split()
+        f1_scores = [calculate_f1(gt.lower().split(), hyp_tokens) for gt in ground_truths]
+        max_f1 = max(f1_scores)
+
+        return [MetricResult(metric_name=self.NAME, value=max_f1, higher_is_better=True, error=response.error)]
+
+
+def calculate_f1(ref_tokens: list[Any], hyp_tokens: list[Any]) -> float:
+    """Calculate F1 score between two texts based on token overlap."""
+    if not ref_tokens and not hyp_tokens:
+        return 1.0
+    if not ref_tokens or not hyp_tokens:
+        return 0.0
+
+    common = Counter(ref_tokens) & Counter(hyp_tokens)
+    num_same = sum(common.values())
+
+    if num_same == 0:
+        return 0.0
+
+    precision = num_same / len(hyp_tokens)
+    recall = num_same / len(ref_tokens)
+
+    return 2 * precision * recall / (precision + recall)