EuroEval 15.10.1__py3-none-any.whl → 15.12.0__py3-none-any.whl

euroeval/generation.py

@@ -6,10 +6,8 @@ import typing as t
 from pathlib import Path
 
 import more_itertools as mit
-from datasets import Dataset, DatasetDict
 from tqdm.auto import tqdm
 
-from .benchmark_modules import BenchmarkModule
 from .enums import BatchingPreference, TaskGroup
 from .exceptions import InvalidBenchmark
 from .model_cache import (
@@ -20,6 +18,9 @@ from .model_cache import (
 from .utils import clear_memory
 
 if t.TYPE_CHECKING:
+    from datasets import Dataset, DatasetDict
+
+    from .benchmark_modules import BenchmarkModule
     from .data_models import (
         BenchmarkConfig,
         DatasetConfig,
@@ -32,7 +33,7 @@ logger = logging.getLogger("euroeval")
 
 def generate(
     model: "BenchmarkModule",
-    datasets: list[DatasetDict],
+    datasets: list["DatasetDict"],
     model_config: "ModelConfig",
     dataset_config: "DatasetConfig",
     benchmark_config: "BenchmarkConfig",
@@ -100,7 +101,7 @@ def generate(
 
 
 def generate_single_iteration(
-    dataset: Dataset,
+    dataset: "Dataset",
     model: "BenchmarkModule",
     dataset_config: "DatasetConfig",
     benchmark_config: "BenchmarkConfig",
@@ -199,17 +200,35 @@ def generate_single_iteration(
         all_preds.extend(extracted_labels)
 
     if "label" in non_cached_dataset.column_names:
+        non_cached_labels = non_cached_dataset["label"]
+        if not isinstance(non_cached_labels, list):
+            non_cached_labels = list(non_cached_labels)
+        cached_labels = cached_dataset["label"]
+        if not isinstance(cached_labels, list):
+            cached_labels = list(cached_labels)
         ground_truth = [
             label.lower() if isinstance(label, str) else label
-            for label in non_cached_dataset["label"] + cached_dataset["label"]
+            for label in non_cached_labels + cached_labels
         ]
     elif "labels" in non_cached_dataset.column_names:
+        non_cached_labels = non_cached_dataset["labels"]
+        if not isinstance(non_cached_labels, list):
+            non_cached_labels = list(non_cached_labels)
+        cached_labels = cached_dataset["labels"]
+        if not isinstance(cached_labels, list):
+            cached_labels = list(cached_labels)
         ground_truth = [
             [label.lower() if isinstance(label, str) else label for label in label_list]
-            for label_list in non_cached_dataset["labels"] + cached_dataset["labels"]
+            for label_list in non_cached_labels + cached_labels
         ]
     elif "target_text" in non_cached_dataset.column_names:
-        ground_truth = non_cached_dataset["target_text"] + cached_dataset["target_text"]
+        non_cached_labels = non_cached_dataset["target_text"]
+        if not isinstance(non_cached_labels, list):
+            non_cached_labels = list(non_cached_labels)
+        cached_labels = cached_dataset["target_text"]
+        if not isinstance(cached_labels, list):
+            cached_labels = list(cached_labels)
+        ground_truth = non_cached_labels + cached_labels
     else:
         raise ValueError(
             "The dataset must have either a 'label', 'labels', or 'target_text' column"
@@ -305,7 +324,7 @@ def debug_log(
     ):
         logger.info(
             f"Input: '{input_text}'\n"
-            f"Raw outout: '{raw_output}'\n"
+            f"Raw output: '{raw_output}'\n"
             f"Prediction: '{prediction}'\n"
             f"Label: '{label}'"
         )

euroeval/human_evaluation.py

@@ -3,6 +3,7 @@
 import importlib.util
 import json
 import logging
+import typing as t
 from collections import defaultdict
 from functools import partial
 from pathlib import Path
@@ -24,13 +25,15 @@ from .task_group_utils import (
     token_classification,
 )
 from .tasks import NER
-from .types import ComputeMetricsFunction, ExtractLabelsFunction, ScoreDict
 from .utils import enforce_reproducibility
 
 if importlib.util.find_spec("gradio") is not None:
     import gradio as gr
     from gradio.components import HTML, Button, Dropdown, Markdown, Textbox
 
+if t.TYPE_CHECKING:
+    from .types import ComputeMetricsFunction, ExtractLabelsFunction, ScoreDict
+
 logger = logging.getLogger("euroeval")
 
 
@@ -86,8 +89,8 @@ class HumanEvaluator:
             }
         )
 
-        self.extract_labels_from_generation: ExtractLabelsFunction
-        self.compute_metrics: ComputeMetricsFunction
+        self.extract_labels_from_generation: "ExtractLabelsFunction"
+        self.compute_metrics: "ComputeMetricsFunction"
 
     def create_app(self) -> "gr.Blocks":
         """Create the Gradio app for human evaluation.
@@ -269,6 +272,7 @@ class HumanEvaluator:
             num_iterations=iteration + 1,
             api_base=None,
             api_version=None,
+            gpu_memory_utilization=0.9,
             debug=False,
             run_with_cli=True,
             only_allow_safetensors=False,
@@ -342,7 +346,6 @@ class HumanEvaluator:
                 self.compute_metrics = partial(
                     sequence_classification.compute_metrics,
                     dataset_config=self.dataset_config,
-                    benchmark_config=benchmark_config,
                 )
                 self.extract_labels_from_generation = partial(
                     sequence_classification.extract_labels_from_generation,
@@ -362,7 +365,6 @@ class HumanEvaluator:
                     token_classification.compute_metrics,
                     has_misc_tags=self.has_misc_tags,
                     dataset_config=self.dataset_config,
-                    benchmark_config=benchmark_config,
                 )
                 self.extract_labels_from_generation = partial(
                     token_classification.extract_labels_from_generation,
@@ -372,7 +374,6 @@ class HumanEvaluator:
                 self.compute_metrics = partial(
                     question_answering.compute_metrics,
                     dataset_config=self.dataset_config,
-                    benchmark_config=benchmark_config,
                 )
                 self.extract_labels_from_generation = (
                     question_answering.extract_labels_from_generation
@@ -641,7 +642,7 @@ class HumanEvaluator:
         # only a single iteration, so the results from the current annotation should be
         # added to the previous results.
         results_path = Path.cwd() / "euroeval_benchmark_results.jsonl"
-        results: ScoreDict = defaultdict(list)
+        results: "ScoreDict" = defaultdict(list)
         if results_path.exists():
             all_results = [
                 json.loads(line.strip())
@@ -664,15 +665,15 @@ class HumanEvaluator:
 
         # Aggregate scores
         total_dict: dict[str, float] = dict()
-        for metric_cfg in self.dataset_config.task.metrics:
+        for metric in self.dataset_config.task.metrics:
             test_score, test_se = aggregate_scores(
                 scores=results["raw"],  # type: ignore[arg-type]
-                metric_config=metric_cfg,
+                metric=metric,
             )
-            test_score, _ = metric_cfg.postprocessing_fn(test_score)
-            test_se, _ = metric_cfg.postprocessing_fn(test_se)
-            total_dict[f"test_{metric_cfg.name}"] = test_score
-            total_dict[f"test_{metric_cfg.name}_se"] = test_se
+            test_score, _ = metric.postprocessing_fn(test_score)
+            test_se, _ = metric.postprocessing_fn(test_se)
+            total_dict[f"test_{metric.name}"] = test_score
+            total_dict[f"test_{metric.name}_se"] = test_se
         results["total"] = total_dict
 
         benchmark_result = BenchmarkResult(

euroeval/languages.py

@@ -36,7 +36,7 @@ NN = Language(
 )
 ES = Language(code="es", name="Spanish", _and_separator="y", _or_separator="o")
 SV = Language(code="sv", name="Swedish", _and_separator="och", _or_separator="eller")
-
+PT = Language(code="pt", name="Portuguese", _and_separator="e", _or_separator="ou")
 
 AB = Language(code="ab", name="Abkhazian")
 AA = Language(code="aa", name="Afar")
@@ -152,7 +152,6 @@ PI = Language(code="pi", name="Pali")
 PS = Language(code="ps", name="Pashto")
 FA = Language(code="fa", name="Persian")
 PL = Language(code="pl", name="Polish")
-PT = Language(code="pt", name="Portuguese")
 PA = Language(code="pa", name="Punjabi")
 QU = Language(code="qu", name="Quechua")
 RO = Language(code="ro", name="Romanian")

euroeval/metrics.py

@@ -0,0 +1,452 @@
+"""All the metrics used in EuroEval."""
+
+import abc
+import logging
+import typing as t
+
+import evaluate
+import litellm
+from litellm.types.utils import Choices, ModelResponse
+from pydantic import BaseModel, Field
+from tqdm.auto import tqdm
+
+from .exceptions import InvalidBenchmark
+from .utils import HiddenPrints
+
+if t.TYPE_CHECKING:
+    from evaluate import EvaluationModule
+
+logger = logging.getLogger(__name__)
+
+
+class Metric(abc.ABC):
+    """Abstract base class for all metrics."""
+
+    def __init__(
+        self,
+        name: str,
+        pretty_name: str,
+        postprocessing_fn: t.Callable[[float], tuple[float, str]] | None = None,
+    ) -> None:
+        """Initialise the metric.
+
+        Args:
+            name:
+                The name of the metric in snake_case.
+            pretty_name:
+                The pretty name of the metric, used for display purposes.
+            postprocessing_fn:
+                A function to apply to the metric scores after they are computed,
+                taking the score to the postprocessed score along with its string
+                representation. Defaults to x -> (100 * x, f"{x:.2%}").
+        """
+        self.name = name
+        self.pretty_name = pretty_name
+        self.postprocessing_fn = (
+            postprocessing_fn
+            if postprocessing_fn is not None
+            else lambda x: (100 * x, f"{x:.2%}")
+        )
+
+    @abc.abstractmethod
+    def __call__(self, predictions: t.Sequence, references: t.Sequence) -> float | None:
+        """Calculate the metric score.
+
+        Args:
+            predictions:
+                The model predictions.
+            references:
+                The ground truth references.
+
+        Returns:
+            The calculated metric score, or None if the score should be ignored.
+        """
+        ...
+
+    def __hash__(self) -> int:
+        """Return a hash of the metric configuration."""
+        return hash(self.name)
+
+
+class HuggingFaceMetric(Metric):
+    """A metric which is implemented in the `evaluate` package.
+
+    Attributes:
+        name:
+            The name of the metric in snake_case.
+        pretty_name:
+            The pretty name of the metric, used for display purposes.
+        huggingface_id:
+            The Hugging Face ID of the metric.
+        results_key:
+            The name of the key used to extract the metric scores from the results
+            dictionary.
+        compute_kwargs:
+            Keyword arguments to pass to the metric's compute function. Defaults to
+            an empty dictionary.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        pretty_name: str,
+        huggingface_id: str,
+        results_key: str,
+        compute_kwargs: dict[str, t.Any] | None = None,
+        postprocessing_fn: t.Callable[[float], tuple[float, str]] | None = None,
+    ) -> None:
+        """Initialise the Hugging Face metric.
+
+        Args:
+            name:
+                The name of the metric in snake_case.
+            pretty_name:
+                The pretty name of the metric, used for display purposes.
+            huggingface_id:
+                The Hugging Face ID of the metric.
+            results_key:
+                The name of the key used to extract the metric scores from the results
+                dictionary.
+            compute_kwargs:
+                Keyword arguments to pass to the metric's compute function. Defaults to
+                an empty dictionary.
+            postprocessing_fn:
+                A function to apply to the metric scores after they are computed, taking
+                the score to the postprocessed score along with its string
+                representation. Defaults to x -> (100 * x, f"{x:.2%}").
+        """
+        super().__init__(
+            name=name, pretty_name=pretty_name, postprocessing_fn=postprocessing_fn
+        )
+        self.huggingface_id = huggingface_id
+        self.results_key = results_key
+        self.compute_kwargs: dict[str, t.Any] = (
+            dict() if compute_kwargs is None else compute_kwargs
+        )
+        self.metric: "EvaluationModule | None" = None
+
+    def __call__(self, predictions: t.Sequence, references: t.Sequence) -> float | None:
+        """Calculate the metric score.
+
+        Args:
+            predictions:
+                The model predictions.
+            references:
+                The ground truth references.
+
+        Returns:
+            The calculated metric score, or None if the score should be ignored.
+        """
+        if self.metric is None:
+            self.metric = evaluate.load(path=self.huggingface_id)
+
+        with HiddenPrints():
+            results = self.metric.compute(
+                predictions=predictions, references=references, **self.compute_kwargs
+            )
+
+        # The metric returns None if we are running on multi-GPU and the current
+        # process is not the main process
+        if results is None:
+            return None
+
+        score = results[self.results_key]
+        if isinstance(score, list):
+            score = sum(score) / len(score)
+
+        return score
+
+
+class LLMAsAJudgeMetric(Metric):
+    """Use an LLM to judge the quality of the predictions."""
+
+    def __init__(
+        self,
+        name: str,
+        pretty_name: str,
+        judge_id: str,
+        judge_kwargs: dict[str, t.Any],
+        user_prompt: str,
+        response_format: t.Type[BaseModel],
+        scoring_fn: t.Callable[[BaseModel], float],
+        condition_formatting_fn: t.Callable[[str], str] = lambda x: x,
+        system_prompt: str | None = None,
+    ) -> None:
+        """Initialise the LLM as a judge metric.
+
+        Args:
+            name:
+                The name of the metric in snake_case.
+            pretty_name:
+                The pretty name of the metric, used for display purposes.
+            judge_id:
+                The model ID of the LLM to use as a judge.
+            judge_kwargs:
+                Generation parameters for the judge model, such as temperature.
+            user_prompt:
+                The user prompt to use for the judge model. The prompt should be
+                formatted with the variables `prediction` and `condition`, to
+                include the model predictions and a description of what the prediction
+                should be judged on, respectively. If the condition is not needed,
+                it can be omitted from the prompt, but the `prediction` variable must
+                still be present.
+            response_format:
+                The response format to use for the judge model. This should be a
+                Pydantic model that defines the expected structure of the judge's
+                response.
+            scoring_fn:
+                A function that takes the judge's response and returns a score.
+            condition_formatting_fn (optional):
+                A function to format the condition string before it is included in the
+                user prompt. Defaults to a no-op function that returns the input
+                unchanged.
+            system_prompt (optional):
+                The system prompt to use for the judge model. If not provided, no system
+                prompt will be used.
+        """
+        super().__init__(name=name, pretty_name=pretty_name)
+        self.judge_id = judge_id
+        self.judge_kwargs = judge_kwargs
+        self.user_prompt = user_prompt
+        self.response_format = response_format
+        self.scoring_fn = scoring_fn
+        self.condition_formatting_fn = condition_formatting_fn
+        self.system_prompt = system_prompt
+
+    def __call__(self, predictions: t.Sequence, references: t.Sequence) -> float | None:
+        """Calculate the metric score using the judge model.
+
+        Args:
+            predictions:
+                The model predictions.
+            references:
+                The ground truth references.
+
+        Returns:
+            The calculated metric score, or None if the score should be ignored.
+
+        Raises:
+            InvalidBenchmark:
+                If the number of predictions does not match the number of references,
+                or if the user prompt requires a condition but none is provided.
+        """
+        if not predictions or not references:
+            return None
+        elif len(predictions) != len(references):
+            raise InvalidBenchmark(
+                f"The number of predictions ({len(predictions):,}) does not match the "
+                f"number of references ({len(references):,})."
+            )
+
+        # Prepare the messages for the LLM
+        conversations: list[list[dict[str, str]]] = [
+            [
+                dict(
+                    role="user",
+                    content=self._apply_user_prompt(
+                        prediction=prediction, condition=condition
+                    ),
+                )
+            ]
+            for prediction, condition in zip(predictions, references)
+        ]
+        if self.system_prompt:
+            conversations = [
+                [dict(role="system", content=self.system_prompt), *conversation]
+                for conversation in conversations
+            ]
+
+        # Get the judge generations
+        generations = [
+            litellm.completion(
+                model=self.judge_id,
+                messages=conversation,
+                response_format=self.response_format,
+                **self.judge_kwargs,
+            )
+            for conversation in tqdm(
+                iterable=conversations,
+                desc=f"Computing {self.pretty_name} scores",
+                unit="sample",
+            )
+        ]
+
+        # Extract the outputs from the generations
+        outputs: list[BaseModel] = list()
+        for generation in generations:
+            assert isinstance(generation, ModelResponse), (
+                f"The judge model did not return a valid response: {generation!r}"
+            )
+            choice = generation.choices[0]
+            assert isinstance(choice, Choices), (
+                f"The judge model did not return a valid choice: {choice!r}"
+            )
+            json_content = choice.message.content
+            assert json_content is not None, (
+                "The judge model returned a None content in the response message."
+            )
+            output = self.response_format.model_validate_json(json_data=json_content)
+            outputs.append(output)
+
+        # Calculate the scores using the scoring function
+        scores = [self.scoring_fn(output) for output in outputs]
+        if not scores:
+            logger.warning(f"No scores were calculated for {self.pretty_name}.")
+            return None
+        return sum(scores) / len(scores)
+
+    def _apply_user_prompt(self, prediction: str, condition: str | None = None) -> str:
+        """Apply the user prompt to the prediction and condition.
+
+        Args:
+            prediction:
+                The model prediction.
+            condition (optional):
+                A description of what the prediction should be judged on. If not
+                provided, it will be omitted from the prompt.
+
+        Returns:
+            The formatted user prompt with the prediction and reference.
+
+        Raises:
+            InvalidBenchmark:
+                If the user prompt requires a reference but none is provided.
+        """
+        condition_required = "{condition}" in self.user_prompt
+        if condition_required and condition is None:
+            raise InvalidBenchmark(
+                f"The user prompt for the {self.pretty_name!r} metric requires a "
+                "condition, but none was provided."
+            )
+        if condition is not None:
+            return self.user_prompt.format(
+                prediction=prediction, condition=self.condition_formatting_fn(condition)
+            )
+        return self.user_prompt.format(prediction=prediction)
+
+
+class SpeedMetric(Metric):
+    """Speed metric."""
+
+    def __init__(self, name: str, pretty_name: str) -> None:
+        """Initialise the speed metric.
+
+        Args:
+            name:
+                The name of the metric in snake_case.
+            pretty_name:
+                The pretty name of the metric, used for display purposes.
+        """
+        super().__init__(
+            name=name,
+            pretty_name=pretty_name,
+            postprocessing_fn=lambda raw_score: (raw_score, f"{raw_score:,.0f}"),
+        )
+
+    def __call__(self, _: t.Sequence, __: t.Sequence) -> float | None:
+        """Not used with the speed metric, but required for consistency."""
+        raise NotImplementedError
+
+
+mcc_metric = HuggingFaceMetric(
+    name="mcc",
+    pretty_name="Matthew's Correlation Coefficient",
+    huggingface_id="matthews_correlation",
+    results_key="matthews_correlation",
+)
+
+macro_f1_metric = HuggingFaceMetric(
+    name="macro_f1",
+    pretty_name="Macro-average F1-score",
+    huggingface_id="f1",
+    results_key="f1",
+    compute_kwargs=dict(average="macro"),
+)
+
+micro_f1_metric = HuggingFaceMetric(
+    name="micro_f1",
+    pretty_name="Micro-average F1-score with MISC tags",
+    huggingface_id="seqeval",
+    results_key="overall_f1",
+)
+
+micro_f1_no_misc_metric = HuggingFaceMetric(
+    name="micro_f1_no_misc",
+    pretty_name="Micro-average F1-score without MISC tags",
+    huggingface_id="seqeval",
+    results_key="overall_f1",
+)
+
+f1_metric = HuggingFaceMetric(
+    name="f1",
+    pretty_name="F1-score",
+    huggingface_id="squad_v2",
+    results_key="f1",
+    postprocessing_fn=lambda x: (x, f"{x:.2f}%"),
+)
+
+em_metric = HuggingFaceMetric(
+    name="em",
+    pretty_name="Exact Match",
+    huggingface_id="squad_v2",
+    results_key="exact",
+    postprocessing_fn=lambda x: (x, f"{x:.2f}%"),
+)
+
+bert_score_metric = HuggingFaceMetric(
+    name="bertscore",
+    pretty_name="BERTScore",
+    huggingface_id="bertscore",
+    results_key="f1",
+    compute_kwargs=dict(
+        model_type="microsoft/mdeberta-v3-base", device="auto", batch_size=1
+    ),
+)
+
+rouge_l_metric = HuggingFaceMetric(
+    name="rouge_l", pretty_name="ROUGE-L", huggingface_id="rouge", results_key="rougeL"
+)
+
+accuracy_metric = HuggingFaceMetric(
+    name="accuracy",
+    pretty_name="Accuracy",
+    huggingface_id="accuracy",
+    results_key="accuracy",
+)
+
+
+class Fluency(BaseModel):
+    """Response format for the fluency metric.
+
+    Attributes:
+        fluency:
+            The fluency rating, an integer between 1 and 5.
+    """
+
+    fluency: t.Annotated[int, Field(ge=1, le=5)]
+
+
+# Example LLM-as-a-judge metric, to measure the fluency of the LLM output
+fluency_metric = LLMAsAJudgeMetric(
+    name="fluency",
+    pretty_name="Fluency",
+    judge_id="gpt-4o-mini",
+    judge_kwargs=dict(temperature=0.0),
+    user_prompt="Please rate the fluency of the following text on a scale from 1 to 5, "
+    "with the following definitions:\n"
+    "- 1: Very poor fluency, many grammatical errors\n"
+    "- 2: Poor fluency, several grammatical errors\n"
+    "- 3: Average fluency, a few grammatical errors\n"
+    "- 4: Good fluency, no grammatical errors but sounds a bit off\n"
+    "- 5: Excellent fluency, no grammatical errors and sounds natural\n\n"
+    "Text: {prediction!r}\n\n"
+    "Output your rating as a JSON object with a single key 'fluency'.",
+    response_format=Fluency,
+    scoring_fn=lambda output: (output.fluency - 1) / 4.0,
+)
+
+speed_metric = SpeedMetric(name="speed", pretty_name="Tokens per second")
+
+speed_short_metric = SpeedMetric(
+    name="speed_short", pretty_name="Tokens per second on short documents"
+)

euroeval/prompt_templates/linguistic_acceptability.py

@@ -1,7 +1,7 @@
 """Templates for the Linguistic Acceptability task."""
 
 from ..data_models import PromptConfig
-from ..languages import DA, DE, EN, ES, FI, FO, FR, IS, IT, NB, NL, NN, NO, SV
+from ..languages import DA, DE, EN, ES, FI, FO, FR, IS, IT, NB, NL, NN, NO, PT, SV
 
 LA_TEMPLATES = {
     DA: PromptConfig(
@@ -36,6 +36,14 @@ LA_TEMPLATES = {
         default_instruction_prompt="Texto: {text}\n\nDetermina si el texto es "
         "gramaticalmente correcto o no. Responde con {labels_str}, y nada más.",
     ),
+    PT: PromptConfig(
+        default_prompt_label_mapping=dict(correct="sim", incorrect="não"),
+        default_prompt_prefix="Seguem-se abaixo textos e se são "
+        "gramaticalmente correctos",
+        default_prompt_template="Texto: {text}\nGramaticalmente correcto: {label}",
+        default_instruction_prompt="Texto: {text}\n\nDetermina se o texto é "
+        "gramaticalmente correcto ou não. Responde com {labels_str}, e nada mais.",
+    ),
     FI: PromptConfig(
         default_prompt_label_mapping=dict(correct="kyllä", incorrect="ei"),
         default_prompt_prefix="Seuraavat ovat lauseita ja ovatko ne "