EuroEval 15.6.1__py3-none-any.whl → 15.7.1__py3-none-any.whl

euroeval/data_loading.py

@@ -39,32 +39,9 @@ def load_data(
         HuggingFaceHubDown:
             If the Hugging Face Hub is down.
     """
-    num_attempts = 5
-    for _ in range(num_attempts):
-        try:
-            dataset = load_dataset(
-                path=dataset_config.huggingface_id,
-                cache_dir=benchmark_config.cache_dir,
-                token=unscramble("HjccJFhIozVymqXDVqTUTXKvYhZMTbfIjMxG_"),
-            )
-            break
-        except (FileNotFoundError, DatasetsError, ConnectionError, ReadTimeout):
-            logger.warning(
-                f"Failed to load dataset {dataset_config.huggingface_id!r}. Retrying..."
-            )
-            time.sleep(1)
-            continue
-        except HfHubHTTPError:
-            raise HuggingFaceHubDown()
-    else:
-        raise InvalidBenchmark(
-            f"Failed to load dataset {dataset_config.huggingface_id!r} after "
-            f"{num_attempts} attempts."
-        )
-
-    assert isinstance(dataset, DatasetDict)  # type: ignore[used-before-def]
-
-    dataset = DatasetDict({key: dataset[key] for key in ["train", "val", "test"]})
+    dataset = load_raw_data(
+        dataset_config=dataset_config, cache_dir=benchmark_config.cache_dir
+    )
 
     if not benchmark_config.evaluate_test_split:
         dataset["test"] = dataset["val"]
@@ -101,3 +78,48 @@ def load_data(
         for idx in range(benchmark_config.num_iterations)
     ]
     return datasets
+
+
+def load_raw_data(dataset_config: "DatasetConfig", cache_dir: str) -> DatasetDict:
+    """Load the raw dataset.
+
+    Args:
+        dataset_config:
+            The configuration for the dataset.
+        cache_dir:
+            The directory to cache the dataset.
+
+    Returns:
+        The dataset.
+    """
+    num_attempts = 5
+    for _ in range(num_attempts):
+        try:
+            dataset = load_dataset(
+                path=dataset_config.huggingface_id,
+                cache_dir=cache_dir,
+                token=unscramble("HjccJFhIozVymqXDVqTUTXKvYhZMTbfIjMxG_"),
+            )
+            break
+        except (FileNotFoundError, DatasetsError, ConnectionError, ReadTimeout):
+            logger.warning(
+                f"Failed to load dataset {dataset_config.huggingface_id!r}. Retrying..."
+            )
+            time.sleep(1)
+            continue
+        except HfHubHTTPError:
+            raise HuggingFaceHubDown()
+    else:
+        raise InvalidBenchmark(
+            f"Failed to load dataset {dataset_config.huggingface_id!r} after "
+            f"{num_attempts} attempts."
+        )
+    assert isinstance(dataset, DatasetDict)  # type: ignore[used-before-def]
+    required_keys = ["train", "val", "test"]
+    missing_keys = [key for key in required_keys if key not in dataset]
+    if missing_keys:
+        raise InvalidBenchmark(
+            "The dataset is missing the following required splits: "
+            f"{', '.join(missing_keys)}"
+        )
+    return DatasetDict({key: dataset[key] for key in required_keys})

euroeval/data_models.py

@@ -521,14 +521,6 @@ class DatasetConfig:
 
         Returns:
             The natural string representation of the labels in specified language.
-
-        Raises:
-            NotImplementedError:
-                If `and_separator` or `or_separator` are `None`, see `Language`.
-
-        Example:
-            >>> get_labels_str(language=DA)
-            "'a', 'b', 'c' eller 'd'"
         """
         main_language = self.languages[0]
 
@@ -539,7 +531,9 @@ class DatasetConfig:
 
         # Convert labels to single-quoted labels - and remove duplicates
         quoted_labels = [
-            f"'{label}'" for label in set(self.prompt_label_mapping.values())
+            f"'{self.prompt_label_mapping[label]}'"
+            for label in set(self.labels)
+            if label in self.prompt_label_mapping
         ]
 
         if not quoted_labels:

euroeval/dataset_configs/dutch.py

@@ -6,13 +6,14 @@ from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
-DUTCH_SOCIAL_CONFIG = DatasetConfig(
-    name="dutch-social",
+DBRD_CONFIG = DatasetConfig(
+    name="dbrd",
     pretty_name="the truncated version of the Dutch sentiment classification "
-    "dataset Dutch Social",
-    huggingface_id="EuroEval/dutch-social-mini",
+    "dataset DBRD",
+    huggingface_id="EuroEval/dbrd-mini",
     task=SENT,
     languages=[NL],
+    _labels=["negative", "positive"],
 )
 
 SCALA_NL_CONFIG = DatasetConfig(
@@ -71,18 +72,6 @@ HELLASWAG_NL_CONFIG = DatasetConfig(
 
 ### Unofficial datasets ###
 
-DBRD_CONFIG = DatasetConfig(
-    name="dbrd",
-    pretty_name="the truncated version of the Dutch sentiment classification "
-    "dataset DBRD",
-    huggingface_id="EuroEval/dbrd-mini",
-    task=SENT,
-    languages=[NL],
-    _labels=["negative", "positive"],
-    _prompt_label_mapping=dict(positive="positief", negative="negatief"),
-    unofficial=True,
-)
-
 DUTCH_COLA_CONFIG = DatasetConfig(
     name="dutch-cola",
     pretty_name="the truncated version of the Dutch linguistic acceptability dataset "

euroeval/dataset_configs/finnish.py

@@ -0,0 +1,60 @@
+"""All Finnish dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import FI
+from ..tasks import COMMON_SENSE, LA, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+SCANDISENT_FI_CONFIG = DatasetConfig(
+    name="scandisent-fi",
+    pretty_name="the truncated version of the Finnish part of the binary sentiment "
+    "classification dataset ScandiSent",
+    huggingface_id="EuroEval/scandisent-fi-mini",
+    task=SENT,
+    languages=[FI],
+    _labels=["negative", "positive"],
+)
+
+TURKU_NER_FI_CONFIG = DatasetConfig(
+    name="turku-ner-fi",
+    pretty_name="the Finnish part of the named entity recognition dataset Turku NER",
+    huggingface_id="EuroEval/turku-ner-fi-mini",
+    task=NER,
+    languages=[FI],
+)
+
+TYDIQA_FI_CONFIG = DatasetConfig(
+    name="tydiqa-fi",
+    pretty_name="the Finnish part of the TydiQA reading comprehension dataset",
+    huggingface_id="EuroEval/tydiqa-fi-mini",
+    task=RC,
+    languages=[FI],
+)
+
+XLSUM_FI_CONFIG = DatasetConfig(
+    name="xlsum-fi",
+    pretty_name="the Finnish summarisation dataset XL-Sum",
+    huggingface_id="EuroEval/xlsum-fi-mini",
+    task=SUMM,
+    languages=[FI],
+)
+
+HELLASWAG_FI_CONFIG = DatasetConfig(
+    name="hellaswag-fi",
+    pretty_name="the truncated version of the Finnish common-sense reasoning dataset "
+    "HellaSwag-fi, translated from the English HellaSwag dataset",
+    huggingface_id="EuroEval/hellaswag-fi-mini",
+    task=COMMON_SENSE,
+    languages=[FI],
+)
+
+SCALA_FI_CONFIG = DatasetConfig(
+    name="scala-fi",
+    pretty_name="the Finnish part of the linguistic acceptability dataset ScaLA",
+    huggingface_id="EuroEval/scala-fi",
+    task=LA,
+    languages=[FI],
+)
+
+### Unofficial datasets ###

euroeval/generation_utils.py

@@ -0,0 +1,346 @@
+"""Utility functions related to generative models."""
+
+import itertools as it
+import json
+import logging
+import random
+import typing as t
+
+from .enums import TaskGroup
+from .exceptions import InvalidBenchmark
+from .utils import log_once
+
+if t.TYPE_CHECKING:
+    from datasets import DatasetDict
+    from transformers.tokenization_utils import PreTrainedTokenizer
+
+    from .data_models import DatasetConfig, ModelConfig
+
+logger = logging.getLogger("euroeval")
+
+
+def extract_few_shot_examples(
+    dataset: "DatasetDict", dataset_config: "DatasetConfig", itr_idx: int
+) -> list[dict[str, t.Any]]:
+    """Extract few-shot examples from a dataset.
+
+    This will always extract the examples from the training split.
+
+    We ensure that the few-shot examples are unique by picking them one at a time.
+
+    Args:
+        dataset:
+            The dataset to extract the few-shot examples from.
+        dataset_config:
+            The dataset configuration.
+        itr_idx:
+            The index of the dataset in the iterator.
+
+    Returns:
+        The few-shot examples.
+    """
+    random_seed = 4242 + itr_idx
+    num_few_shots = dataset_config.num_few_shot_examples
+    few_shot_examples: list[dict[str, t.Any]] = list()
+    shuffled_train = dataset["train"].shuffle(seed=random_seed)
+
+    match dataset_config.task.task_group:
+        case (
+            TaskGroup.SEQUENCE_CLASSIFICATION | TaskGroup.MULTIPLE_CHOICE_CLASSIFICATION
+        ):
+            # Locate the maximum number of tokens that constitutes a short example
+            for max_num_tokens in [512, 1024, 2048, 4096, 8192]:
+                train_with_short_examples = dataset["train"].filter(
+                    lambda example: len(example["text"]) < max_num_tokens
+                )
+                num_short_examples = len(train_with_short_examples)
+                if num_short_examples >= dataset_config.num_few_shot_examples:
+                    break
+            else:
+                raise InvalidBenchmark(
+                    "Could not find enough short examples for few-shot learning."
+                )
+
+            shuffled_train = train_with_short_examples.shuffle(seed=random_seed)
+            labels = it.cycle(dataset_config.labels)
+            while len(few_shot_examples) < num_few_shots and len(shuffled_train) > 0:
+                label = next(labels)
+                possible_examples = shuffled_train.filter(
+                    lambda x: x["label"].lower() == label.lower()
+                )
+                if len(possible_examples) == 0:
+                    continue
+                example = possible_examples.select(range(1))[0]
+                few_shot_examples.append(example)
+                shuffled_train = shuffled_train.filter(
+                    lambda x: x["text"] != example["text"]
+                )
+
+        case TaskGroup.TEXT_TO_TEXT:
+            while len(few_shot_examples) < num_few_shots and len(shuffled_train) > 0:
+                example = shuffled_train.select(range(1))[0]
+                few_shot_examples.append(example)
+                shuffled_train = shuffled_train.filter(
+                    lambda x: x["text"] != example["text"]
+                )
+
+        case TaskGroup.TOKEN_CLASSIFICATION:
+            labels = it.cycle(
+                [
+                    label.lower()
+                    for label in dataset_config.labels
+                    if label.lower().startswith("b-")
+                ]
+            )
+            while len(few_shot_examples) < num_few_shots and len(shuffled_train) > 0:
+                label = next(labels)
+                possible_examples = shuffled_train.filter(
+                    lambda x: label in [tag.lower() for tag in x["labels"]]
+                )
+                if len(possible_examples) == 0:
+                    continue
+                example = possible_examples.select(range(1))[0]
+                few_shot_examples.append(example)
+                shuffled_train = shuffled_train.filter(
+                    lambda x: x["tokens"] != example["tokens"]
+                )
+
+        case TaskGroup.QUESTION_ANSWERING:
+            # Locate the maximum number of tokens that constitutes a short example
+            for max_num_tokens in [512, 1024, 2048, 4096, 8192]:
+                train_with_short_examples = dataset["train"].filter(
+                    lambda example: len(example["context"]) < max_num_tokens
+                )
+                num_short_examples = len(train_with_short_examples)
+                if num_short_examples >= dataset_config.num_few_shot_examples:
+                    break
+            else:
+                raise InvalidBenchmark(
+                    "Could not find enough short examples for few-shot learning."
+                )
+
+            shuffled_train = train_with_short_examples.shuffle(seed=random_seed)
+            while len(few_shot_examples) < num_few_shots and len(shuffled_train) > 0:
+                example = shuffled_train.select(range(1))[0]
+                few_shot_examples.append(example)
+                shuffled_train = shuffled_train.filter(
+                    lambda x: x["context"] != example["context"]
+                )
+
+        case _:
+            raise NotImplementedError(
+                f"Unsupported task group: {dataset_config.task.task_group}."
+            )
+
+    random.seed(random_seed)
+    random.shuffle(few_shot_examples)
+    return few_shot_examples
+
+
+def apply_prompt(
+    examples: dict[str, t.Any],
+    few_shot_examples: list[dict[str, t.Any]],
+    model_config: "ModelConfig",
+    dataset_config: "DatasetConfig",
+    instruction_model: bool,
+    always_populate_text_field: bool,
+    tokenizer: "PreTrainedTokenizer | None",
+) -> dict[str, t.Any]:
+    """Apply prompt template to an example, potentially with few-shot examples.
+
+    Args:
+        examples:
+            The examples to apply the few-shot examples to.
+        few_shot_examples:
+            The few-shot examples to apply.
+        dataset_config:
+            The dataset configuration.
+        instruction_model:
+            Whether the model is instruction-tuned.
+        always_populate_text_field:
+            Whether to always populate the 'text' field in the examples, as opposed to
+            the 'messages' field.
+        tokenizer:
+            The tokenizer to use for the model. If None, the tokenizer is not used.
+
+    Returns:
+        The example with the few-shot examples applied.
+    """
+    # Sanity check
+    if instruction_model and always_populate_text_field and tokenizer is None:
+        raise ValueError(
+            "The `tokenizer` argument must be provided when the model is instruction "
+            "tuned and when we are not just returning the raw messages."
+        )
+
+    def create_prompt(**kwargs: str) -> tuple[str, str]:
+        """Create a prompt from the given keyword arguments.
+
+        Args:
+            kwargs:
+                The keyword arguments to use in the prompt.
+
+        Returns:
+            A pair (prompt, label), where "label" is an empty string if the model is
+            not instruction tuned (as in this case it is included in the prompt).
+        """
+        label_key = "label" if "label" in kwargs else "target_text"
+        label = kwargs.pop(label_key)
+        assert label is not None, (
+            f"Found a None label for the prompt: {kwargs}. This should not happen."
+        )
+        label_mapping = dataset_config.prompt_label_mapping
+        label = label_mapping.get(label, label)
+        if instruction_model:
+            prompt = dataset_config.instruction_prompt.format(**kwargs)
+            return prompt, label
+        else:
+            kwargs[label_key] = label
+            return dataset_config.prompt_template.format(**kwargs), ""
+
+    match dataset_config.task.task_group:
+        case (
+            TaskGroup.SEQUENCE_CLASSIFICATION | TaskGroup.MULTIPLE_CHOICE_CLASSIFICATION
+        ):
+            few_shot_sections = [
+                create_prompt(
+                    text=example["text"].replace("\n", " ").strip(),
+                    label=example["label"].replace("\n", " ").strip(),
+                )
+                for example in few_shot_examples
+            ]
+            new_sections = [
+                create_prompt(text=text.replace("\n", " ").strip(), label="")
+                for text in examples["text"]
+            ]
+
+        case TaskGroup.TEXT_TO_TEXT:
+            few_shot_sections = [
+                create_prompt(
+                    text=example["text"].replace("\n", " ").strip(),
+                    target_text=example["target_text"].replace("\n", " ").strip(),
+                )
+                for example in few_shot_examples
+            ]
+            new_sections = [
+                create_prompt(text=text.replace("\n", " ").strip(), target_text="")
+                for text in examples["text"]
+            ]
+
+        case TaskGroup.TOKEN_CLASSIFICATION:
+
+            def create_label(example: dict) -> str:
+                prompt_labels = dataset_config.prompt_label_mapping.values()
+                labels: dict[str, list[str]] = {
+                    prompt_label: list() for prompt_label in prompt_labels
+                }
+                for token, label in zip(example["tokens"], example["labels"]):
+                    label = label.lower()
+                    if label == "o":
+                        continue
+                    prompt_label = dataset_config.prompt_label_mapping[label]
+                    if label.startswith("b-"):
+                        labels[prompt_label].append(token)
+                    elif label.startswith("i-"):
+                        labels[prompt_label][-1] += " " + token
+                return json.dumps(labels, ensure_ascii=False)
+
+            few_shot_sections = [
+                create_prompt(
+                    text=" ".join(example["tokens"]).replace("\n", " ").strip(),
+                    label=create_label(example=example),
+                )
+                for example in few_shot_examples
+            ]
+            new_sections = [
+                create_prompt(
+                    text=" ".join(tokens).replace("\n", " ").strip(), label=""
+                )
+                for tokens in examples["tokens"]
+            ]
+
+        case TaskGroup.QUESTION_ANSWERING:
+            few_shot_sections = [
+                create_prompt(
+                    text=example["context"].replace("\n", " ").strip(),
+                    question=example["question"].replace("\n", " ").strip(),
+                    label=example["answers"]["text"][0].replace("\n", " "),
+                )
+                for example in few_shot_examples
+            ]
+            new_sections = [
+                create_prompt(
+                    text=context.replace("\n", " ").strip(),
+                    question=question.replace("\n", " ").strip(),
+                    label="",
+                )
+                for context, question in zip(examples["context"], examples["question"])
+            ]
+
+        case _:
+            raise NotImplementedError(
+                f"Unsupported task group: {dataset_config.task.task_group}."
+            )
+
+    if instruction_model:
+        few_shot_messages = [
+            dict(role=role, content=content)
+            for prompt, label in few_shot_sections
+            for role, content in [("user", prompt), ("assistant", label)]
+        ]
+
+        messages_list = [
+            few_shot_messages + [dict(role="user", content=prompt)]
+            for prompt, _ in new_sections
+        ]
+
+        if not always_populate_text_field:
+            examples["messages"] = messages_list
+
+        else:
+            assert tokenizer is not None
+
+            # Pick the chat template that matches the language of the dataset, if such a
+            # template exists
+            chat_template: str | None = None
+            if isinstance(tokenizer.chat_template, dict):
+                language_codes = [
+                    language.code for language in dataset_config.languages
+                ]
+                for name, candidate_template in tokenizer.chat_template.items():
+                    if name.lower() in language_codes:
+                        chat_template = candidate_template
+                        log_once(
+                            f"Using the {name!r} chat template for the tokenizer for "
+                            f"model {model_config.model_id!r}.",
+                            level=logging.DEBUG,
+                        )
+                        break
+
+            texts = [
+                tokenizer.apply_chat_template(
+                    conversation=messages,
+                    tokenize=False,
+                    add_generation_prompt=True,
+                    chat_template=chat_template,
+                )
+                for messages in messages_list
+            ]
+
+            examples["text"] = texts
+
+    else:
+        prompt_prefix = ""
+        if dataset_config.prompt_prefix:
+            prompt_prefix = dataset_config.prompt_prefix + "\n\n"
+
+        few_shot_prompt = "\n\n".join([prompt for prompt, _ in few_shot_sections])
+        if few_shot_prompt:
+            few_shot_prompt += "\n\n"
+
+        examples["text"] = [
+            prompt_prefix + few_shot_prompt + new_prompt
+            for new_prompt, _ in new_sections
+        ]
+
+    return examples

euroeval/prompt_templates/linguistic_acceptability.py

@@ -1,7 +1,7 @@
 """Templates for the Linguistic Acceptability task."""
 
 from ..data_models import PromptConfig
-from ..languages import DA, DE, EN, ES, FO, FR, IS, IT, NB, NL, NN, NO, SV
+from ..languages import DA, DE, EN, ES, FI, FO, FR, IS, IT, NB, NL, NN, NO, SV
 
 LA_TEMPLATES = {
     DA: PromptConfig(
@@ -36,6 +36,14 @@ LA_TEMPLATES = {
         default_instruction_prompt="Texto: {text}\n\nDetermina si el texto es "
         "gramaticalmente correcto o no. Responde con {labels_str}, y nada más.",
     ),
+    FI: PromptConfig(
+        default_prompt_label_mapping=dict(correct="kyllä", incorrect="ei"),
+        default_prompt_prefix="Seuraavat ovat lauseita ja ovatko ne "
+        "kieliopillisesti oikein.",
+        default_prompt_template="Lause: {text}\nKieliopillisesti oikein: {label}",
+        default_instruction_prompt="Lause: {text}\n\nMääritä onko lause "
+        "oikein vai ei. Vastaa {labels_str}, ja ei mitään muuta.",
+    ),
     FO: PromptConfig(
         default_prompt_label_mapping=dict(correct="ja", incorrect="nei"),
         default_prompt_prefix="Hetta eru nakrir setningar og um teir eru mállæruliga "

euroeval/prompt_templates/multiple_choice.py

@@ -1,7 +1,7 @@
 """Templates for all multiple choice tasks."""
 
 from ..data_models import PromptConfig
-from ..languages import DA, DE, EN, ES, FR, IS, IT, NB, NL, NN, NO, SV
+from ..languages import DA, DE, EN, ES, FI, FR, IS, IT, NB, NL, NN, NO, SV
 
 # TODO: Missing Faroese
 MULTIPLE_CHOICE_TEMPLATES = {
@@ -36,6 +36,13 @@ MULTIPLE_CHOICE_TEMPLATES = {
         "usando solo {labels_str}, y nada más.",
         default_prompt_label_mapping="auto",
     ),
+    FI: PromptConfig(
+        default_prompt_prefix="Seuraavat ovat monivalintakysymyksiä (vastauksineen).",
+        default_prompt_template="Kysymys: {text}\nVastaus: {label}",
+        default_instruction_prompt="Kysymys: {text}\n\nVastaa yllä olevaan kysymykseen "
+        "käyttämällä {labels_str}, äläkä mitään muuta.",
+        default_prompt_label_mapping="auto",
+    ),
     FR: PromptConfig(
         default_prompt_prefix="Les questions suivantes sont des questions à choix "
         "multiples (avec réponses).",

euroeval/prompt_templates/named_entity_recognition.py

@@ -1,7 +1,7 @@
 """Templates for the Named Entity Recognition task."""
 
 from ..data_models import PromptConfig
-from ..languages import DA, DE, EN, ES, FO, FR, IS, IT, NB, NL, NN, NO, SV
+from ..languages import DA, DE, EN, ES, FI, FO, FR, IS, IT, NB, NL, NN, NO, SV
 
 NER_TEMPLATES = {
     DA: PromptConfig(
@@ -80,6 +80,25 @@ NER_TEMPLATES = {
         "claves {labels_str}. Los valores deben ser listas de las "
         "entidades nombradas de ese tipo, exactamente como aparecen en la oración.",
     ),
+    FI: PromptConfig(
+        default_prompt_label_mapping={
+            "b-per": "henkilö",
+            "i-per": "henkilö",
+            "b-loc": "paikka",
+            "i-loc": "paikka",
+            "b-org": "organisaatio",
+            "i-org": "organisaatio",
+            "b-misc": "muut",
+            "i-misc": "muut",
+        },
+        default_prompt_prefix="Seuraavassa on lauseita ja JSON-sanakirjoja, jotka "
+        "sisältävät annetussa lauseessa esiintyvät nimetyt entiteetit.",
+        default_prompt_template="Lause: {text}\nNimetyt entiteetit: {label}",
+        default_instruction_prompt="Lause: {text}\n\nTunnista lauseessa olevat "
+        "entiteetit. Tulosta ne JSON-sanakirjana, jonka avaimet ovat {labels_str}. "
+        "Arvojen tulee olla listoja kyseisen tyypin nimetyistä entiteeteistä "
+        "täsmälleen siinä muodossa kuin ne esiintyvät lauseessa.",
+    ),
     FO: PromptConfig(
         default_prompt_label_mapping={
             "b-per": "persónur",

euroeval/prompt_templates/reading_comprehension.py

@@ -1,7 +1,7 @@
 """Templates for the Reading Comprehension task."""
 
 from ..data_models import PromptConfig
-from ..languages import DA, DE, EN, ES, FO, FR, IS, IT, NB, NL, NN, NO, SV
+from ..languages import DA, DE, EN, ES, FI, FO, FR, IS, IT, NB, NL, NN, NO, SV
 
 RC_TEMPLATES = {
     DA: PromptConfig(
@@ -39,6 +39,16 @@ RC_TEMPLATES = {
         "sobre el texto anterior en máximo 3 palabras.\n\nPregunta: {question}",
         default_prompt_label_mapping=dict(),
     ),
+    FI: PromptConfig(
+        default_prompt_prefix="Seuraavassa on tekstejä ja niihin liittyviä kysymyksiä "
+        "ja vastauksia.",
+        default_prompt_template="Teksti: {text}\nKysymys: {question} "
+        "\nVastaa enintään 3 sanalla: {label}",
+        default_instruction_prompt="Teksti: {text}\n\nVastaa seuraavaan "
+        "kysymykseen yllä olevasta tekstistä enintään 3 sanalla.\n\n"
+        "Kysymys: {question}",
+        default_prompt_label_mapping=dict(),
+    ),
     FO: PromptConfig(
         default_prompt_prefix="Hetta eru tekstir saman við spurningum og svar.",
         default_prompt_template="Tekstur: {text}\nSpurningur: {question}\nSvara við í "