EuroEval 15.5.0__py3-none-any.whl → 15.6.1__py3-none-any.whl

euroeval/benchmarker.py

@@ -1,5 +1,6 @@
 """Class that benchmarks language models."""
 
+import contextlib
 import json
 import logging
 import re
@@ -13,7 +14,7 @@ from time import sleep
 from torch.distributed import destroy_process_group
 
 from .benchmark_config_factory import build_benchmark_config
-from .constants import GENERATIVE_PIPELINE_TAGS
+from .constants import GENERATIVE_DATASET_TASK_GROUPS, GENERATIVE_PIPELINE_TAGS
 from .data_loading import load_data
 from .data_models import BenchmarkConfigParams, BenchmarkResult
 from .dataset_configs import get_all_dataset_configs
@@ -399,9 +400,21 @@ class Benchmarker:
                     num_finished_benchmarks += 1
                     continue
 
+                # Skip if the model is an encoder model and the task is generative
+                task_is_generative = (
+                    dataset_config.task.task_group in GENERATIVE_DATASET_TASK_GROUPS
+                )
+                if model_config.model_type == ModelType.ENCODER and task_is_generative:
+                    logger.debug(
+                        f"Skipping benchmarking {model_id} on "
+                        f"{dataset_config.pretty_name}, as it is an encoder model and "
+                        "the task is generative."
+                    )
+                    continue
+
                 # We do not re-initialise generative models as their architecture is not
                 # customised to specific datasets
-                if model_config.task in GENERATIVE_PIPELINE_TAGS:
+                if model_config.model_type == ModelType.GENERATIVE:
                     initial_logging(
                         model_config=model_config,
                         dataset_config=dataset_config,
@@ -421,8 +434,8 @@ class Benchmarker:
                             logger.info(e.message)
 
                             # Add the remaining number of benchmarks for the model to
-                            # our benchmark counter, since we're skipping the
-                            # rest of them
+                            # our benchmark counter, since we're skipping the rest of
+                            # them
                             num_finished_benchmarks += (
                                 len(dataset_configs)
                                 - dataset_configs.index(dataset_config)
@@ -447,31 +460,22 @@ class Benchmarker:
                     raise benchmark_output_or_err
 
                 elif isinstance(benchmark_output_or_err, InvalidBenchmark):
-                    if benchmark_config.raise_errors:
-                        raise benchmark_output_or_err
-                    logger.info(
-                        f"{model_id} could not be benchmarked on "
-                        f"{dataset_config.pretty_name}. Skipping. The error message "
-                        f"raised was {benchmark_output_or_err.message!r}."
-                    )
+                    logger.info(benchmark_output_or_err.message)
                     num_finished_benchmarks += 1
                     continue
 
                 elif isinstance(benchmark_output_or_err, InvalidModel):
-                    if benchmark_config.raise_errors:
-                        raise benchmark_output_or_err
                     logger.info(benchmark_output_or_err.message)
 
-                    # Add the remaining number of benchmarks for the model to
-                    # our benchmark counter, since we're skipping the
-                    # rest of them
+                    # Add the remaining number of benchmarks for the model to our
+                    # benchmark counter, since we're skipping the rest of them
                     num_finished_benchmarks += (
                         len(dataset_configs) - dataset_configs.index(dataset_config) - 1
                     )
                     break
 
                 else:
-                    record = benchmark_output_or_err
+                    record: BenchmarkResult = benchmark_output_or_err
                     current_benchmark_results.append(record)
                     if benchmark_config.save_results:
                         record.append_to_results(results_path=self.results_path)
@@ -482,6 +486,7 @@ class Benchmarker:
                     f"{total_benchmarks} benchmarks."
                 )
 
+            del loaded_model
             if benchmark_config.clear_model_cache:
                 clear_model_cache_fn(cache_dir=benchmark_config.cache_dir)
 
@@ -493,11 +498,8 @@ class Benchmarker:
         #   point and block the progress of another member of the process group. This
         #   constraint has always been present,  but this warning has only been added
         #   since PyTorch 2.4 (function operator())
-        try:
+        with contextlib.suppress(AssertionError):
             destroy_process_group()
-        except AssertionError:
-            pass
-
         return current_benchmark_results
 
     def _get_updated_benchmark_config(

euroeval/callbacks.py

@@ -5,8 +5,8 @@ from collections.abc import Sized
 
 from torch.utils.data import DataLoader
 from tqdm.auto import tqdm
-from transformers import TrainerControl, TrainerState, TrainingArguments
-from transformers.trainer_callback import ProgressCallback
+from transformers.trainer_callback import ProgressCallback, TrainerControl, TrainerState
+from transformers.training_args import TrainingArguments
 
 
 class NeverLeaveProgressCallback(ProgressCallback):

euroeval/constants.py

@@ -51,7 +51,7 @@ TASK_GROUPS_USING_LOGPROBS = [
 
 # The number of top log probabilities to return for generative models. For several APIs
 # this is the maximum number of log probabilities that can be returned
-MAX_LOGPROBS = 10
+MAX_LOGPROBS = 8
 
 
 # We make sure to remove these metric attributes after each iteration, to avoid memory

euroeval/data_models.py

@@ -54,44 +54,107 @@ class MetricConfig:
 
 
 @dataclass
-class Task:
-    """A dataset task.
+class Language:
+    """A benchmarkable language.
 
     Attributes:
+        code:
+            The ISO 639-1 language code of the language.
         name:
-            The name of the task.
-        task_group:
-            The task group of the task.
-        metrics:
-            The metrics used to evaluate the task.
+            The name of the language.
+        and_separator (optional):
+            The word 'and' in the language.
+        or_separator (optional):
+            The word 'or' in the language.
     """
 
+    code: str
     name: str
-    task_group: TaskGroup
-    metrics: list[MetricConfig]
+    _and_separator: str | None = field(repr=False, default=None)
+    _or_separator: str | None = field(repr=False, default=None)
 
     def __hash__(self) -> int:
-        """Return a hash of the task."""
-        return hash(self.name)
+        """Return a hash of the language."""
+        return hash(self.code)
+
+    @property
+    def and_separator(self) -> str:
+        """Get the word 'and' in the language.
+
+        Returns:
+            The word 'and' in the language.
+
+        Raises:
+            NotImplementedError:
+                If `and_separator` is `None`.
+        """
+        if not self._and_separator:
+            raise NotImplementedError(
+                f"Separator for the word 'and' has not been defined for {self.name}."
+            )
+        return self._and_separator
+
+    @and_separator.setter
+    def and_separator(self, value: str | None) -> None:
+        self._and_separator = value
+
+    @property
+    def or_separator(self) -> str:
+        """Get the word 'or' in the language.
+
+        Returns:
+            The word 'or' in the language.
+
+        Raises:
+            NotImplementedError:
+                If `or_separator` is `None`.
+        """
+        if not self._or_separator:
+            raise NotImplementedError(
+                f"Separator for the word 'or' has not been defined for {self.name}."
+            )
+        return self._or_separator
+
+    @or_separator.setter
+    def or_separator(self, value: str | None) -> None:
+        self._or_separator = value
 
 
 @dataclass
-class Language:
-    """A benchmarkable language.
+class Task:
+    """A dataset task.
 
     Attributes:
-        code:
-            The ISO 639-1 language code of the language.
         name:
-            The name of the language.
+            The name of the task.
+        task_group:
+            The task group of the task.
+        template_dict:
+            The template dictionary for the task, from language to prompt template.
+        metrics:
+            The metrics used to evaluate the task.
+        default_num_few_shot_examples:
+            The default number of examples to use when benchmarking the task using
+            few-shot evaluation. For a classification task, these will be drawn evenly
+            from each label.
+        default_max_generated_tokens:
+            The default maximum number of tokens to generate when benchmarking the task
+            using few-shot evaluation.
+        default_labels:
+            The default labels for datasets using this task.
     """
 
-    code: str
     name: str
+    task_group: TaskGroup
+    template_dict: dict["Language", "PromptConfig"]
+    metrics: list[MetricConfig]
+    default_num_few_shot_examples: int
+    default_max_generated_tokens: int
+    default_labels: list[str]
 
     def __hash__(self) -> int:
-        """Return a hash of the language."""
-        return hash(self.code)
+        """Return a hash of the task."""
+        return hash(self.name)
 
 
 @dataclass
@@ -303,26 +366,32 @@ class DatasetConfig:
             The mapping from label to ID.
         num_labels:
             The number of labels in the dataset.
-        prompt_template:
+        _prompt_prefix (optional):
+            The prefix to use in the few-shot prompt. Defaults to the template for the
+            task and language.
+        _prompt_template (optional):
             The template for the prompt to use when benchmarking the dataset using
-            few-shot evaluation.
-        max_generated_tokens:
-            The maximum number of tokens to generate when benchmarking the dataset
-            using few-shot evaluation.
-        prompt_prefix:
-            The prefix to use in the few-shot prompt.
-        num_few_shot_examples:
+            few-shot evaluation. Defaults to the template for the task and language.
+        _instruction_prompt (optional):
+            The prompt to use when benchmarking the dataset using instruction-based
+            evaluation. Defaults to the template for the task and language.
+        _num_few_shot_examples (optional):
             The number of examples to use when benchmarking the dataset using few-shot
             evaluation. For a classification task, these will be drawn evenly from
-            each label.
-        instruction_prompt:
-            The prompt to use when benchmarking the dataset using instruction-based
-            evaluation.
-        labels (optional):
-            The labels in the dataset. Defaults to an empty list.
-        prompt_label_mapping (optional):
+            each label. Defaults to the template for the task and language.
+        _max_generated_tokens (optional):
+            The maximum number of tokens to generate when benchmarking the dataset
+            using few-shot evaluation. Defaults to the template for the task and
+            language.
+        _labels (optional):
+            The labels in the dataset. Defaults to the template for the task and
+            language.
+        _prompt_label_mapping (optional):
             A mapping from the labels to another phrase which is used as a substitute
-            for the label in few-shot evaluation. Defaults to an empty dictionary.
+            for the label in few-shot evaluation. If "auto" then the mapping will be set
+            to a 1:1 mapping between the labels and themselves. If None then the mapping
+            will be set to the default mapping for the task and language. Defaults to
+            None.
         unofficial (optional):
             Whether the dataset is unofficial. Defaults to False.
     """
@@ -332,15 +401,95 @@ class DatasetConfig:
     huggingface_id: str
     task: Task
     languages: list[Language]
-    prompt_template: str
-    max_generated_tokens: int
-    prompt_prefix: str
-    num_few_shot_examples: int
-    instruction_prompt: str
-    labels: list[str] = field(default_factory=list)
-    prompt_label_mapping: dict[str, str] = field(default_factory=dict)
+    _prompt_prefix: str | None = None
+    _prompt_template: str | None = None
+    _instruction_prompt: str | None = None
+    _num_few_shot_examples: int | None = None
+    _max_generated_tokens: int | None = None
+    _labels: list[str] | None = None
+    _prompt_label_mapping: dict[str, str] | t.Literal["auto"] | None = None
     unofficial: bool = False
 
+    @property
+    def prompt_prefix(self) -> str:
+        """The prefix to use in the few-shot prompt."""
+        main_language = self.languages[0]
+        prompt_config = self.task.template_dict[main_language]
+        prompt_prefix = (
+            prompt_config.default_prompt_prefix
+            if self._prompt_prefix is None
+            else self._prompt_prefix
+        )
+        prompt_prefix = prompt_prefix.replace("{labels_str}", self._labels_str)
+        return prompt_prefix
+
+    @property
+    def prompt_template(self) -> str:
+        """The template used during few-shot evaluation."""
+        main_language = self.languages[0]
+        prompt_config = self.task.template_dict[main_language]
+        prompt_template = (
+            prompt_config.default_prompt_template
+            if self._prompt_template is None
+            else self._prompt_template
+        )
+        prompt_template = prompt_template.replace("{labels_str}", self._labels_str)
+        return prompt_template
+
+    @property
+    def instruction_prompt(self) -> str:
+        """The prompt to use when evaluating instruction-tuned models."""
+        main_language = self.languages[0]
+        prompt_config = self.task.template_dict[main_language]
+        instruction_prompt = (
+            prompt_config.default_instruction_prompt
+            if self._instruction_prompt is None
+            else self._instruction_prompt
+        )
+        instruction_prompt = instruction_prompt.replace(
+            "{labels_str}", self._labels_str
+        )
+        return instruction_prompt
+
+    @property
+    def num_few_shot_examples(self) -> int:
+        """The number of few-shot examples to use."""
+        return (
+            self._num_few_shot_examples
+            if self._num_few_shot_examples is not None
+            else self.task.default_num_few_shot_examples
+        )
+
+    @property
+    def max_generated_tokens(self) -> int:
+        """The maximum number of tokens to generate when evaluating a model."""
+        return (
+            self._max_generated_tokens
+            if self._max_generated_tokens is not None
+            else self.task.default_max_generated_tokens
+        )
+
+    @property
+    def labels(self) -> list[str]:
+        """The labels in the dataset."""
+        return self._labels if self._labels is not None else self.task.default_labels
+
+    @property
+    def prompt_label_mapping(self) -> dict[str, str]:
+        """Mapping from English labels to localised labels."""
+        if self._prompt_label_mapping == "auto":
+            return {label: label for label in self.labels}
+        elif self._prompt_label_mapping is not None:
+            return self._prompt_label_mapping
+
+        main_language = self.languages[0]
+        prompt_config = self.task.template_dict[main_language]
+
+        if prompt_config.default_prompt_label_mapping == "auto":
+            return {label: label for label in self.labels}
+        else:
+            return prompt_config.default_prompt_label_mapping
+
     @property
     def id2label(self) -> dict[int, str]:
         """The mapping from ID to label."""
@@ -360,6 +509,48 @@ class DatasetConfig:
         """Return a hash of the dataset configuration."""
         return hash(self.name)
 
+    @property
+    def _labels_str(self) -> str:
+        """Converts a set of labels to a natural string, in the specified language.
+
+        If the task is NER, we separate using 'and' and use the mapped labels instead of
+        the BIO NER labels.
+
+        Args:
+            language: The language to be used when converting the labels.
+
+        Returns:
+            The natural string representation of the labels in specified language.
+
+        Raises:
+            NotImplementedError:
+                If `and_separator` or `or_separator` are `None`, see `Language`.
+
+        Example:
+            >>> get_labels_str(language=DA)
+            "'a', 'b', 'c' eller 'd'"
+        """
+        main_language = self.languages[0]
+
+        if self.task.task_group == TaskGroup.TOKEN_CLASSIFICATION:
+            sep_word = main_language.and_separator
+        else:
+            sep_word = main_language.or_separator
+
+        # Convert labels to single-quoted labels - and remove duplicates
+        quoted_labels = [
+            f"'{label}'" for label in set(self.prompt_label_mapping.values())
+        ]
+
+        if not quoted_labels:
+            return ""
+        elif len(quoted_labels) == 1:
+            return quoted_labels[0]
+        elif len(quoted_labels) == 2:
+            return f"{quoted_labels[0]} {sep_word} {quoted_labels[1]}"
+        else:
+            return f"{', '.join(quoted_labels[:-1])} {sep_word} {quoted_labels[-1]}"
+
 
 @dataclass
 class ModelConfig:
@@ -476,3 +667,31 @@ class HFModelInfo:
     pipeline_tag: str
     tags: list[str]
     adapter_base_model_id: str | None
+
+
+@dataclass
+class PromptConfig:
+    """Configuration for task-specific prompting across languages.
+
+    Defines the prompt templates needed for evaluating a specific task in a given
+    language.
+
+    Attributes:
+        default_prompt_prefix:
+            The default prefix to use in the few-shot prompt.
+        default_prompt_template:
+            The default template for the prompt to use when benchmarking the dataset
+            using few-shot evaluation.
+        default_instruction_prompt:
+            The default prompt to use when benchmarking the dataset using
+            instruction-based evaluation.
+        default_prompt_label_mapping:
+            The default mapping from the labels to another phrase which is used as a
+            substitute for the label in few-shot evaluation. If set to "auto", the
+            mapping will be set to a 1:1 mapping between the labels and themselves.
+    """
+
+    default_prompt_prefix: str
+    default_prompt_template: str
+    default_instruction_prompt: str
+    default_prompt_label_mapping: dict[str, str] | t.Literal["auto"]

euroeval/dataset_configs/__init__.py

@@ -0,0 +1,61 @@
+"""All dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import get_all_languages
+from ..tasks import SPEED
+from .danish import *  # noqa: F403
+from .dutch import *  # noqa: F403
+from .english import *  # noqa: F403
+from .faroese import *  # noqa: F403
+from .french import *  # noqa: F403
+from .german import *  # noqa: F403
+from .icelandic import *  # noqa: F403
+from .italian import *  # noqa: F403
+from .norwegian import *  # noqa: F403
+from .spanish import *  # noqa: F403
+from .swedish import *  # noqa: F403
+
+
+def get_all_dataset_configs() -> dict[str, DatasetConfig]:
+    """Get a mapping of all the dataset configurations.
+
+    Returns:
+        A mapping between names of datasets and their configurations.
+    """
+    dataset_configs = [
+        cfg for cfg in globals().values() if isinstance(cfg, DatasetConfig)
+    ]
+    assert len(dataset_configs) == len({cfg.name for cfg in dataset_configs}), (
+        "There are duplicate dataset configurations. Please ensure that each dataset "
+        "has a unique name."
+    )
+    return {cfg.name: cfg for cfg in dataset_configs}
+
+
+def get_dataset_config(dataset_name: str) -> DatasetConfig:
+    """Get the dataset configuration for a dataset.
+
+    Args:
+        dataset_name:
+            The name of the dataset.
+
+    Returns:
+        The dataset configuration.
+
+    Raises:
+        ValueError:
+            If the dataset is not found.
+    """
+    dataset_configs = get_all_dataset_configs()
+    if dataset_name not in dataset_configs:
+        raise ValueError(f"No dataset config found for dataset {dataset_name}.")
+    return dataset_configs[dataset_name]
+
+
+SPEED_CONFIG = DatasetConfig(
+    name="speed",
+    pretty_name="the speed estimation benchmark",
+    huggingface_id="",
+    task=SPEED,
+    languages=list(get_all_languages().values()),
+)

euroeval/dataset_configs/danish.py

@@ -0,0 +1,120 @@
+"""All Danish dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import DA
+from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+ANGRY_TWEETS_CONFIG = DatasetConfig(
+    name="angry-tweets",
+    pretty_name="the truncated version of the Danish sentiment classification "
+    "dataset AngryTweets",
+    huggingface_id="EuroEval/angry-tweets-mini",
+    task=SENT,
+    languages=[DA],
+)
+
+SCALA_DA_CONFIG = DatasetConfig(
+    name="scala-da",
+    pretty_name="the Danish part of the linguistic acceptability dataset ScaLA",
+    huggingface_id="EuroEval/scala-da",
+    task=LA,
+    languages=[DA],
+)
+
+DANSK_CONFIG = DatasetConfig(
+    name="dansk",
+    pretty_name="the truncated version of the Danish named entity recognition "
+    "dataset DANSK",
+    huggingface_id="EuroEval/dansk-mini",
+    task=NER,
+    languages=[DA],
+)
+
+SCANDIQA_DA_CONFIG = DatasetConfig(
+    name="scandiqa-da",
+    pretty_name="the Danish part of the truncated version of the question answering "
+    "dataset ScandiQA",
+    huggingface_id="EuroEval/scandiqa-da-mini",
+    task=RC,
+    languages=[DA],
+)
+
+NORDJYLLAND_NEWS_CONFIG = DatasetConfig(
+    name="nordjylland-news",
+    pretty_name="the truncated version of the Danish summarisation dataset "
+    "Nordjylland News",
+    huggingface_id="EuroEval/nordjylland-news-mini",
+    task=SUMM,
+    languages=[DA],
+)
+
+DANSKE_TALEMAADER_CONFIG = DatasetConfig(
+    name="danske-talemaader",
+    pretty_name="the truncated version of the Danish knowledge dataset Danske "
+    "Talemåder",
+    huggingface_id="EuroEval/danske-talemaader",
+    task=KNOW,
+    languages=[DA],
+)
+
+DANISH_CITIZEN_TESTS_CONFIG = DatasetConfig(
+    name="danish-citizen-tests",
+    pretty_name="the Danish knowledge dataset Danish Citizen Tests",
+    huggingface_id="EuroEval/danish-citizen-tests-updated",
+    task=KNOW,
+    languages=[DA],
+)
+
+HELLASWAG_DA_CONFIG = DatasetConfig(
+    name="hellaswag-da",
+    pretty_name="the truncated version of the Danish common-sense reasoning dataset "
+    "HellaSwag-da, translated from the English HellaSwag dataset",
+    huggingface_id="EuroEval/hellaswag-da-mini",
+    task=COMMON_SENSE,
+    languages=[DA],
+)
+
+
+### Unofficial datasets ###
+
+DANE_CONFIG = DatasetConfig(
+    name="dane",
+    pretty_name="the truncated version of the Danish named entity recognition "
+    "dataset DaNE",
+    huggingface_id="EuroEval/dane-mini",
+    task=NER,
+    languages=[DA],
+    unofficial=True,
+)
+
+MMLU_DA_CONFIG = DatasetConfig(
+    name="mmlu-da",
+    pretty_name="the truncated version of the Danish knowledge dataset MMLU-da, "
+    "translated from the English MMLU dataset",
+    huggingface_id="EuroEval/mmlu-da-mini",
+    task=KNOW,
+    languages=[DA],
+    unofficial=True,
+)
+
+ARC_DA_CONFIG = DatasetConfig(
+    name="arc-da",
+    pretty_name="the truncated version of the Danish knowledge dataset ARC-da, "
+    "translated from the English ARC dataset",
+    huggingface_id="EuroEval/arc-da-mini",
+    task=KNOW,
+    languages=[DA],
+    unofficial=True,
+)
+
+BELEBELE_DA_CONFIG = DatasetConfig(
+    name="belebele-da",
+    pretty_name="the Danish multiple choice reading comprehension dataset BeleBele-da, "
+    "translated from the English BeleBele dataset",
+    huggingface_id="EuroEval/belebele-da-mini",
+    task=MCRC,
+    languages=[DA],
+    unofficial=True,
+)