EuroEval 15.4.2__py3-none-any.whl → 15.6.0__py3-none-any.whl

euroeval/data_models.py

@@ -10,10 +10,9 @@ from dataclasses import dataclass, field
 import pydantic
 import torch
 
-from euroeval.utils import get_package_version
-
 from .enums import Device, InferenceBackend, ModelType, TaskGroup
 from .types import ScoreDict
+from .utils import get_package_version
 
 
 @dataclass
@@ -55,44 +54,107 @@ class MetricConfig:
 
 
 @dataclass
-class Task:
-    """A dataset task.
+class Language:
+    """A benchmarkable language.
 
     Attributes:
+        code:
+            The ISO 639-1 language code of the language.
         name:
-            The name of the task.
-        task_group:
-            The task group of the task.
-        metrics:
-            The metrics used to evaluate the task.
+            The name of the language.
+        and_separator (optional):
+            The word 'and' in the language.
+        or_separator (optional):
+            The word 'or' in the language.
     """
 
+    code: str
     name: str
-    task_group: TaskGroup
-    metrics: list[MetricConfig]
+    _and_separator: str | None = field(repr=False, default=None)
+    _or_separator: str | None = field(repr=False, default=None)
 
     def __hash__(self) -> int:
-        """Return a hash of the task."""
-        return hash(self.name)
+        """Return a hash of the language."""
+        return hash(self.code)
+
+    @property
+    def and_separator(self) -> str:
+        """Get the word 'and' in the language.
+
+        Returns:
+            The word 'and' in the language.
+
+        Raises:
+            NotImplementedError:
+                If `and_separator` is `None`.
+        """
+        if not self._and_separator:
+            raise NotImplementedError(
+                f"Separator for the word 'and' has not been defined for {self.name}."
+            )
+        return self._and_separator
+
+    @and_separator.setter
+    def and_separator(self, value: str | None) -> None:
+        self._and_separator = value
+
+    @property
+    def or_separator(self) -> str:
+        """Get the word 'or' in the language.
+
+        Returns:
+            The word 'or' in the language.
+
+        Raises:
+            NotImplementedError:
+                If `or_separator` is `None`.
+        """
+        if not self._or_separator:
+            raise NotImplementedError(
+                f"Separator for the word 'or' has not been defined for {self.name}."
+            )
+        return self._or_separator
+
+    @or_separator.setter
+    def or_separator(self, value: str | None) -> None:
+        self._or_separator = value
 
 
 @dataclass
-class Language:
-    """A benchmarkable language.
+class Task:
+    """A dataset task.
 
     Attributes:
-        code:
-            The ISO 639-1 language code of the language.
         name:
-            The name of the language.
+            The name of the task.
+        task_group:
+            The task group of the task.
+        template_dict:
+            The template dictionary for the task, from language to prompt template.
+        metrics:
+            The metrics used to evaluate the task.
+        default_num_few_shot_examples:
+            The default number of examples to use when benchmarking the task using
+            few-shot evaluation. For a classification task, these will be drawn evenly
+            from each label.
+        default_max_generated_tokens:
+            The default maximum number of tokens to generate when benchmarking the task
+            using few-shot evaluation.
+        default_labels:
+            The default labels for datasets using this task.
     """
 
-    code: str
     name: str
+    task_group: TaskGroup
+    template_dict: dict["Language", "PromptConfig"]
+    metrics: list[MetricConfig]
+    default_num_few_shot_examples: int
+    default_max_generated_tokens: int
+    default_labels: list[str]
 
     def __hash__(self) -> int:
-        """Return a hash of the language."""
-        return hash(self.code)
+        """Return a hash of the task."""
+        return hash(self.name)
 
 
 @dataclass
@@ -304,26 +366,30 @@ class DatasetConfig:
             The mapping from label to ID.
         num_labels:
             The number of labels in the dataset.
-        prompt_template:
+        _prompt_prefix (optional):
+            The prefix to use in the few-shot prompt. Defaults to the template for the
+            task and language.
+        _prompt_template (optional):
             The template for the prompt to use when benchmarking the dataset using
-            few-shot evaluation.
-        max_generated_tokens:
-            The maximum number of tokens to generate when benchmarking the dataset
-            using few-shot evaluation.
-        prompt_prefix:
-            The prefix to use in the few-shot prompt.
-        num_few_shot_examples:
+            few-shot evaluation. Defaults to the template for the task and language.
+        _instruction_prompt (optional):
+            The prompt to use when benchmarking the dataset using instruction-based
+            evaluation. Defaults to the template for the task and language.
+        _num_few_shot_examples (optional):
             The number of examples to use when benchmarking the dataset using few-shot
             evaluation. For a classification task, these will be drawn evenly from
-            each label.
-        instruction_prompt:
-            The prompt to use when benchmarking the dataset using instruction-based
-            evaluation.
-        labels (optional):
-            The labels in the dataset. Defaults to an empty list.
-        prompt_label_mapping (optional):
+            each label. Defaults to the template for the task and language.
+        _max_generated_tokens (optional):
+            The maximum number of tokens to generate when benchmarking the dataset
+            using few-shot evaluation. Defaults to the template for the task and
+            language.
+        _labels (optional):
+            The labels in the dataset. Defaults to the template for the task and
+            language.
+        _prompt_label_mapping (optional):
             A mapping from the labels to another phrase which is used as a substitute
-            for the label in few-shot evaluation. Defaults to an empty dictionary.
+            for the label in few-shot evaluation. Defaults to the template for the task
+            and language.
         unofficial (optional):
             Whether the dataset is unofficial. Defaults to False.
     """
@@ -333,15 +399,93 @@ class DatasetConfig:
     huggingface_id: str
     task: Task
     languages: list[Language]
-    prompt_template: str
-    max_generated_tokens: int
-    prompt_prefix: str
-    num_few_shot_examples: int
-    instruction_prompt: str
-    labels: list[str] = field(default_factory=list)
-    prompt_label_mapping: dict[str, str] = field(default_factory=dict)
+    _prompt_prefix: str | None = None
+    _prompt_template: str | None = None
+    _instruction_prompt: str | None = None
+    _num_few_shot_examples: int | None = None
+    _max_generated_tokens: int | None = None
+    _labels: list[str] | None = None
+    _prompt_label_mapping: dict[str, str] | None = None
     unofficial: bool = False
 
+    @property
+    def prompt_prefix(self) -> str:
+        """The prefix to use in the few-shot prompt."""
+        main_language = self.languages[0]
+        prompt_config = self.task.template_dict[main_language]
+        prompt_prefix = (
+            prompt_config.default_prompt_prefix
+            if self._prompt_prefix is None
+            else self._prompt_prefix
+        )
+        prompt_prefix = prompt_prefix.replace("{labels_str}", self._labels_str)
+        return prompt_prefix
+
+    @property
+    def prompt_template(self) -> str:
+        """The template used during few-shot evaluation."""
+        main_language = self.languages[0]
+        prompt_config = self.task.template_dict[main_language]
+        prompt_template = (
+            prompt_config.default_prompt_template
+            if self._prompt_template is None
+            else self._prompt_template
+        )
+        prompt_template = prompt_template.replace("{labels_str}", self._labels_str)
+        return prompt_template
+
+    @property
+    def instruction_prompt(self) -> str:
+        """The prompt to use when evaluating instruction-tuned models."""
+        main_language = self.languages[0]
+        prompt_config = self.task.template_dict[main_language]
+        instruction_prompt = (
+            prompt_config.default_instruction_prompt
+            if self._instruction_prompt is None
+            else self._instruction_prompt
+        )
+        instruction_prompt = instruction_prompt.replace(
+            "{labels_str}", self._labels_str
+        )
+        return instruction_prompt
+
+    @property
+    def num_few_shot_examples(self) -> int:
+        """The number of few-shot examples to use."""
+        return (
+            self._num_few_shot_examples
+            if self._num_few_shot_examples is not None
+            else self.task.default_num_few_shot_examples
+        )
+
+    @property
+    def max_generated_tokens(self) -> int:
+        """The maximum number of tokens to generate when evaluating a model."""
+        return (
+            self._max_generated_tokens
+            if self._max_generated_tokens is not None
+            else self.task.default_max_generated_tokens
+        )
+
+    @property
+    def labels(self) -> list[str]:
+        """The labels in the dataset."""
+        return self._labels if self._labels is not None else self.task.default_labels
+
+    @property
+    def prompt_label_mapping(self) -> dict[str, str]:
+        """Mapping from English labels to localised labels."""
+        if self._prompt_label_mapping is not None:
+            return self._prompt_label_mapping
+
+        main_language = self.languages[0]
+        prompt_config = self.task.template_dict[main_language]
+
+        if prompt_config.default_prompt_label_mapping == "auto":
+            return {label: label for label in self.labels}
+        else:
+            return prompt_config.default_prompt_label_mapping
+
     @property
     def id2label(self) -> dict[int, str]:
         """The mapping from ID to label."""
@@ -361,6 +505,48 @@ class DatasetConfig:
         """Return a hash of the dataset configuration."""
         return hash(self.name)
 
+    @property
+    def _labels_str(self) -> str:
+        """Converts a set of labels to a natural string, in the specified language.
+
+        If the task is NER, we separate using 'and' and use the mapped labels instead of
+        the BIO NER labels.
+
+        Args:
+            language: The language to be used when converting the labels.
+
+        Returns:
+            The natural string representation of the labels in specified language.
+
+        Raises:
+            NotImplementedError:
+                If `and_separator` or `or_separator` are `None`, see `Language`.
+
+        Example:
+            >>> get_labels_str(language=DA)
+            "'a', 'b', 'c' eller 'd'"
+        """
+        main_language = self.languages[0]
+
+        if self.task.task_group == TaskGroup.TOKEN_CLASSIFICATION:
+            sep_word = main_language.and_separator
+        else:
+            sep_word = main_language.or_separator
+
+        # Convert labels to single-quoted labels - and remove duplicates
+        quoted_labels = [
+            f"'{label}'" for label in set(self.prompt_label_mapping.values())
+        ]
+
+        if not quoted_labels:
+            return ""
+        elif len(quoted_labels) == 1:
+            return quoted_labels[0]
+        elif len(quoted_labels) == 2:
+            return f"{quoted_labels[0]} {sep_word} {quoted_labels[1]}"
+        else:
+            return f"{', '.join(quoted_labels[:-1])} {sep_word} {quoted_labels[-1]}"
+
 
 @dataclass
 class ModelConfig:
@@ -477,3 +663,31 @@ class HFModelInfo:
     pipeline_tag: str
     tags: list[str]
     adapter_base_model_id: str | None
+
+
+@dataclass
+class PromptConfig:
+    """Configuration for task-specific prompting across languages.
+
+    Defines the prompt templates needed for evaluating a specific task in a given
+    language.
+
+    Attributes:
+        default_prompt_prefix:
+            The default prefix to use in the few-shot prompt.
+        default_prompt_template:
+            The default template for the prompt to use when benchmarking the dataset
+            using few-shot evaluation.
+        default_instruction_prompt:
+            The default prompt to use when benchmarking the dataset using
+            instruction-based evaluation.
+        default_prompt_label_mapping:
+            The default mapping from the labels to another phrase which is used as a
+            substitute for the label in few-shot evaluation. If set to "auto", the
+            mapping will be set to a 1:1 mapping between the labels and themselves.
+    """
+
+    default_prompt_prefix: str
+    default_prompt_template: str
+    default_instruction_prompt: str
+    default_prompt_label_mapping: dict[str, str] | t.Literal["auto"]

euroeval/dataset_configs/__init__.py

@@ -0,0 +1,61 @@
+"""All dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import get_all_languages
+from ..tasks import SPEED
+from .danish import *  # noqa: F403
+from .dutch import *  # noqa: F403
+from .english import *  # noqa: F403
+from .faroese import *  # noqa: F403
+from .french import *  # noqa: F403
+from .german import *  # noqa: F403
+from .icelandic import *  # noqa: F403
+from .italian import *  # noqa: F403
+from .norwegian import *  # noqa: F403
+from .spanish import *  # noqa: F403
+from .swedish import *  # noqa: F403
+
+
+def get_all_dataset_configs() -> dict[str, DatasetConfig]:
+    """Get a mapping of all the dataset configurations.
+
+    Returns:
+        A mapping between names of datasets and their configurations.
+    """
+    dataset_configs = [
+        cfg for cfg in globals().values() if isinstance(cfg, DatasetConfig)
+    ]
+    assert len(dataset_configs) == len({cfg.name for cfg in dataset_configs}), (
+        "There are duplicate dataset configurations. Please ensure that each dataset "
+        "has a unique name."
+    )
+    return {cfg.name: cfg for cfg in dataset_configs}
+
+
+def get_dataset_config(dataset_name: str) -> DatasetConfig:
+    """Get the dataset configuration for a dataset.
+
+    Args:
+        dataset_name:
+            The name of the dataset.
+
+    Returns:
+        The dataset configuration.
+
+    Raises:
+        ValueError:
+            If the dataset is not found.
+    """
+    dataset_configs = get_all_dataset_configs()
+    if dataset_name not in dataset_configs:
+        raise ValueError(f"No dataset config found for dataset {dataset_name}.")
+    return dataset_configs[dataset_name]
+
+
+SPEED_CONFIG = DatasetConfig(
+    name="speed",
+    pretty_name="the speed estimation benchmark",
+    huggingface_id="",
+    task=SPEED,
+    languages=list(get_all_languages().values()),
+)

euroeval/dataset_configs/danish.py

@@ -0,0 +1,120 @@
+"""All Danish dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import DA
+from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+ANGRY_TWEETS_CONFIG = DatasetConfig(
+    name="angry-tweets",
+    pretty_name="the truncated version of the Danish sentiment classification "
+    "dataset AngryTweets",
+    huggingface_id="EuroEval/angry-tweets-mini",
+    task=SENT,
+    languages=[DA],
+)
+
+SCALA_DA_CONFIG = DatasetConfig(
+    name="scala-da",
+    pretty_name="the Danish part of the linguistic acceptability dataset ScaLA",
+    huggingface_id="EuroEval/scala-da",
+    task=LA,
+    languages=[DA],
+)
+
+DANSK_CONFIG = DatasetConfig(
+    name="dansk",
+    pretty_name="the truncated version of the Danish named entity recognition "
+    "dataset DANSK",
+    huggingface_id="EuroEval/dansk-mini",
+    task=NER,
+    languages=[DA],
+)
+
+SCANDIQA_DA_CONFIG = DatasetConfig(
+    name="scandiqa-da",
+    pretty_name="the Danish part of the truncated version of the question answering "
+    "dataset ScandiQA",
+    huggingface_id="EuroEval/scandiqa-da-mini",
+    task=RC,
+    languages=[DA],
+)
+
+NORDJYLLAND_NEWS_CONFIG = DatasetConfig(
+    name="nordjylland-news",
+    pretty_name="the truncated version of the Danish summarisation dataset "
+    "Nordjylland News",
+    huggingface_id="EuroEval/nordjylland-news-mini",
+    task=SUMM,
+    languages=[DA],
+)
+
+DANSKE_TALEMAADER_CONFIG = DatasetConfig(
+    name="danske-talemaader",
+    pretty_name="the truncated version of the Danish knowledge dataset Danske "
+    "Talemåder",
+    huggingface_id="EuroEval/danske-talemaader",
+    task=KNOW,
+    languages=[DA],
+)
+
+DANISH_CITIZEN_TESTS_CONFIG = DatasetConfig(
+    name="danish-citizen-tests",
+    pretty_name="the Danish knowledge dataset Danish Citizen Tests",
+    huggingface_id="EuroEval/danish-citizen-tests-updated",
+    task=KNOW,
+    languages=[DA],
+)
+
+HELLASWAG_DA_CONFIG = DatasetConfig(
+    name="hellaswag-da",
+    pretty_name="the truncated version of the Danish common-sense reasoning dataset "
+    "HellaSwag-da, translated from the English HellaSwag dataset",
+    huggingface_id="EuroEval/hellaswag-da-mini",
+    task=COMMON_SENSE,
+    languages=[DA],
+)
+
+
+### Unofficial datasets ###
+
+DANE_CONFIG = DatasetConfig(
+    name="dane",
+    pretty_name="the truncated version of the Danish named entity recognition "
+    "dataset DaNE",
+    huggingface_id="EuroEval/dane-mini",
+    task=NER,
+    languages=[DA],
+    unofficial=True,
+)
+
+MMLU_DA_CONFIG = DatasetConfig(
+    name="mmlu-da",
+    pretty_name="the truncated version of the Danish knowledge dataset MMLU-da, "
+    "translated from the English MMLU dataset",
+    huggingface_id="EuroEval/mmlu-da-mini",
+    task=KNOW,
+    languages=[DA],
+    unofficial=True,
+)
+
+ARC_DA_CONFIG = DatasetConfig(
+    name="arc-da",
+    pretty_name="the truncated version of the Danish knowledge dataset ARC-da, "
+    "translated from the English ARC dataset",
+    huggingface_id="EuroEval/arc-da-mini",
+    task=KNOW,
+    languages=[DA],
+    unofficial=True,
+)
+
+BELEBELE_DA_CONFIG = DatasetConfig(
+    name="belebele-da",
+    pretty_name="the Danish multiple choice reading comprehension dataset BeleBele-da, "
+    "translated from the English BeleBele dataset",
+    huggingface_id="EuroEval/belebele-da-mini",
+    task=MCRC,
+    languages=[DA],
+    unofficial=True,
+)

euroeval/dataset_configs/dutch.py

@@ -0,0 +1,123 @@
+"""All Dutch dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import NL
+from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+DUTCH_SOCIAL_CONFIG = DatasetConfig(
+    name="dutch-social",
+    pretty_name="the truncated version of the Dutch sentiment classification "
+    "dataset Dutch Social",
+    huggingface_id="EuroEval/dutch-social-mini",
+    task=SENT,
+    languages=[NL],
+)
+
+SCALA_NL_CONFIG = DatasetConfig(
+    name="scala-nl",
+    pretty_name="the Dutch part of the linguistic acceptability dataset ScaLA",
+    huggingface_id="EuroEval/scala-nl",
+    task=LA,
+    languages=[NL],
+)
+
+CONLL_NL_CONFIG = DatasetConfig(
+    name="conll-nl",
+    pretty_name="the Dutch part of the truncated version of the named entity "
+    "recognition dataset CoNLL 2002",
+    huggingface_id="EuroEval/conll-nl-mini",
+    task=NER,
+    languages=[NL],
+)
+
+SQUAD_NL_CONFIG = DatasetConfig(
+    name="squad-nl",
+    pretty_name="the truncated version of the Dutch reading comprehension dataset "
+    "SQuAD-nl, translated from the English SQuAD dataset",
+    huggingface_id="EuroEval/squad-nl-v2-mini",
+    task=RC,
+    languages=[NL],
+)
+
+WIKI_LINGUA_NL_CONFIG = DatasetConfig(
+    name="wiki-lingua-nl",
+    pretty_name="the Dutch part of the truncated version of the summarisation dataset "
+    "WikiLingua",
+    huggingface_id="EuroEval/wiki-lingua-nl-mini",
+    task=SUMM,
+    languages=[NL],
+)
+
+MMLU_NL_CONFIG = DatasetConfig(
+    name="mmlu-nl",
+    pretty_name="the truncated version of the Dutch knowledge dataset MMLU-nl, "
+    "translated from the English MMLU dataset",
+    huggingface_id="EuroEval/mmlu-nl-mini",
+    task=KNOW,
+    languages=[NL],
+)
+
+HELLASWAG_NL_CONFIG = DatasetConfig(
+    name="hellaswag-nl",
+    pretty_name="the truncated version of the Dutch common-sense reasoning dataset "
+    "HellaSwag-nl, translated from the English HellaSwag dataset",
+    huggingface_id="EuroEval/hellaswag-nl-mini",
+    task=COMMON_SENSE,
+    languages=[NL],
+)
+
+
+### Unofficial datasets ###
+
+DBRD_CONFIG = DatasetConfig(
+    name="dbrd",
+    pretty_name="the truncated version of the Dutch sentiment classification "
+    "dataset DBRD",
+    huggingface_id="EuroEval/dbrd-mini",
+    task=SENT,
+    languages=[NL],
+    _labels=["negative", "positive"],
+    _prompt_label_mapping=dict(positive="positief", negative="negatief"),
+    unofficial=True,
+)
+
+DUTCH_COLA_CONFIG = DatasetConfig(
+    name="dutch-cola",
+    pretty_name="the truncated version of the Dutch linguistic acceptability dataset "
+    "Dutch CoLA",
+    huggingface_id="EuroEval/dutch-cola",
+    task=LA,
+    languages=[NL],
+    unofficial=True,
+)
+
+DUTCH_COLA_FULL_CONFIG = DatasetConfig(
+    name="dutch-cola-full",
+    pretty_name="the Dutch linguistic acceptability dataset Dutch CoLA",
+    huggingface_id="EuroEval/dutch-cola-full",
+    task=LA,
+    languages=[NL],
+    unofficial=True,
+)
+
+ARC_NL_CONFIG = DatasetConfig(
+    name="arc-nl",
+    pretty_name="the truncated version of the Dutch knowledge dataset ARC-nl, "
+    "translated from the English ARC dataset",
+    huggingface_id="EuroEval/arc-nl-mini",
+    task=KNOW,
+    languages=[NL],
+    unofficial=True,
+)
+
+BELEBELE_NL_CONFIG = DatasetConfig(
+    name="belebele-nl",
+    pretty_name="the Dutch multiple choice reading comprehension dataset BeleBele-nl, "
+    "translated from the English BeleBele dataset",
+    huggingface_id="EuroEval/belebele-nl-mini",
+    task=MCRC,
+    languages=[NL],
+    unofficial=True,
+)