ScandEval 16.12.0__py3-none-any.whl → 16.13.0__py3-none-any.whl

scandeval/dataset_configs/__init__.py

@@ -1,11 +1,17 @@
 """All dataset configurations used in EuroEval."""
 
+import collections.abc as c
+import logging
 from pathlib import Path
 
+from ..custom_dataset_configs import (
+    load_custom_datasets_module,
+    try_get_dataset_config_from_repo,
+)
 from ..data_models import DatasetConfig
 from ..languages import get_all_languages
+from ..logging_utils import log_once
 from ..tasks import SPEED
-from ..utils import load_custom_datasets_module
 from .albanian import *  # noqa: F403
 from .bosnian import *  # noqa: F403
 from .bulgarian import *  # noqa: F403
@@ -38,20 +44,62 @@ from .swedish import *  # noqa: F403
 from .ukrainian import *  # noqa: F403
 
 
-def get_all_dataset_configs(custom_datasets_file: Path) -> dict[str, DatasetConfig]:
+def get_all_dataset_configs(
+    custom_datasets_file: Path,
+    dataset_ids: c.Sequence[str],
+    api_key: str | None,
+    cache_dir: Path,
+) -> dict[str, DatasetConfig]:
     """Get a mapping of all the dataset configurations.
 
     Args:
         custom_datasets_file:
             A path to a Python file containing custom dataset configurations.
+        dataset_ids:
+            The IDs of the datasets to include in the mapping.
+        api_key:
+            The Hugging Face API key to use to check if the repositories have custom
+            dataset configs.
+        cache_dir:
+            The directory to store the cache in.
 
     Returns:
         A mapping between names of datasets and their configurations.
     """
     globals_dict = globals()
+
+    # If any of the dataset IDs are referring to Hugging Face dataset IDs, then we check
+    # if the repositories have custom dataset configs and if they do, we add them to the
+    # globals dict.
+    for dataset_id in dataset_ids:
+        dataset_config_or_none = try_get_dataset_config_from_repo(
+            dataset_id=dataset_id, api_key=api_key, cache_dir=cache_dir
+        )
+        if dataset_config_or_none is not None:
+            globals_dict[dataset_id] = dataset_config_or_none
+            msg = f"Loaded external dataset {dataset_id}"
+            split_strings = []
+            if dataset_config_or_none.train_split is not None:
+                split_strings.append(
+                    f"train split '{dataset_config_or_none.train_split}'"
+                )
+            if dataset_config_or_none.val_split is not None:
+                split_strings.append(f"val split '{dataset_config_or_none.val_split}'")
+            if dataset_config_or_none.test_split is not None:
+                split_strings.append(
+                    f"test split '{dataset_config_or_none.test_split}'"
+                )
+            if split_strings:
+                msg += f" with {', '.join(split_strings[:-1])} and {split_strings[-1]}"
+            msg += "."
+            log_once(msg, level=logging.INFO)
+
+    # Add the custom datasets from the custom datasets file to the globals dict
     module = load_custom_datasets_module(custom_datasets_file=custom_datasets_file)
     if module is not None:
         globals_dict |= vars(module)
+
+    # Extract the dataset configs from the globals dict
     dataset_configs = [
         cfg
         for cfg in globals_dict.values()
@@ -61,37 +109,15 @@ def get_all_dataset_configs(custom_datasets_file: Path) -> dict[str, DatasetConf
         "There are duplicate dataset configurations. Please ensure that each dataset "
         "has a unique name."
     )
+
     mapping = {cfg.name: cfg for cfg in dataset_configs}
     return mapping
 
 
-def get_dataset_config(dataset_name: str, custom_datasets_file: Path) -> DatasetConfig:
-    """Get the dataset configuration for a dataset.
-
-    Args:
-        dataset_name:
-            The name of the dataset.
-        custom_datasets_file:
-            A path to a Python file containing custom dataset configurations.
-
-    Returns:
-        The dataset configuration.
-
-    Raises:
-        ValueError:
-            If the dataset is not found.
-    """
-    dataset_configs = get_all_dataset_configs(custom_datasets_file=custom_datasets_file)
-    if dataset_name not in dataset_configs:
-        raise ValueError(f"No dataset config found for dataset {dataset_name}.")
-    return dataset_configs[dataset_name]
-
-
 SPEED_CONFIG = DatasetConfig(
     name="speed",
     pretty_name="",
     source="",
     task=SPEED,
     languages=list(get_all_languages().values()),
-    _logging_string="the speed estimation benchmark",
 )

scandeval/dataset_configs/albanian.py

@@ -60,5 +60,5 @@ WINOGRANDE_SQ_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-sq",
     task=COMMON_SENSE,
     languages=[ALBANIAN],
-    _labels=["a", "b"],
+    labels=["a", "b"],
 )

scandeval/dataset_configs/belarusian.py

@@ -0,0 +1,47 @@
+"""All Belarusian dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import BELARUSIAN
+from ..tasks import COMMON_SENSE, LA, NER, RC, SENT
+
+### Official datasets ###
+
+BESLS_CONFIG = DatasetConfig(
+    name="besls",
+    pretty_name="BeSLS",
+    source="EuroEval/besls",
+    task=SENT,
+    languages=[BELARUSIAN],
+)
+
+SCALA_BE_CONFIG = DatasetConfig(
+    name="scala-be",
+    pretty_name="ScaLA-be",
+    source="EuroEval/scala-be",
+    task=LA,
+    languages=[BELARUSIAN],
+)
+
+WIKIANN_BE_CONFIG = DatasetConfig(
+    name="wikiann-be",
+    pretty_name="WikiANN-be",
+    source="EuroEval/wikiann-be-mini",
+    task=NER,
+    languages=[BELARUSIAN],
+)
+
+MULTI_WIKI_QA_BE_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-be",
+    pretty_name="MultiWikiQA-be",
+    source="EuroEval/multi-wiki-qa-be-mini",
+    task=RC,
+    languages=[BELARUSIAN],
+)
+
+BE_WSC_CONFIG = DatasetConfig(
+    name="be-wsc",
+    pretty_name="BE-WSC",
+    source="EuroEval/be-wsc",
+    task=COMMON_SENSE,
+    languages=[BELARUSIAN],
+)

scandeval/dataset_configs/bulgarian.py

@@ -52,5 +52,5 @@ WINOGRANDE_BG_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-bg",
     task=COMMON_SENSE,
     languages=[BULGARIAN],
-    _labels=["a", "b"],
+    labels=["a", "b"],
 )

scandeval/dataset_configs/catalan.py

@@ -60,5 +60,5 @@ WINOGRANDE_CA_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-ca",
     task=COMMON_SENSE,
     languages=[CATALAN],
-    _labels=["a", "b"],
+    labels=["a", "b"],
 )

scandeval/dataset_configs/croatian.py

@@ -52,5 +52,5 @@ WINOGRANDE_HR_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-hr",
     task=COMMON_SENSE,
     languages=[CROATIAN],
-    _labels=["a", "b"],
+    labels=["a", "b"],
 )

scandeval/dataset_configs/danish.py

@@ -76,7 +76,8 @@ VALEU_DA_CONFIG = DatasetConfig(
     source="EuroEval/european-values-da",
     task=EUROPEAN_VALUES,
     languages=[DANISH],
-    splits=["test"],
+    train_split=None,
+    val_split=None,
     bootstrap_samples=False,
 )
 
@@ -143,6 +144,6 @@ WINOGRANDE_DA_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-da",
     task=COMMON_SENSE,
     languages=[DANISH],
-    _labels=["a", "b"],
+    labels=["a", "b"],
     unofficial=True,
 )

scandeval/dataset_configs/dutch.py

@@ -24,7 +24,7 @@ DBRD_CONFIG = DatasetConfig(
     source="EuroEval/dbrd-mini",
     task=SENT,
     languages=[DUTCH],
-    _labels=["negative", "positive"],
+    labels=["negative", "positive"],
 )
 
 SCALA_NL_CONFIG = DatasetConfig(
@@ -89,9 +89,10 @@ VALEU_NL_CONFIG = DatasetConfig(
     source="EuroEval/european-values-nl",
     task=EUROPEAN_VALUES,
     languages=[DUTCH],
-    splits=["test"],
+    train_split=None,
+    val_split=None,
     bootstrap_samples=False,
-    _instruction_prompt="{text}",
+    instruction_prompt="{text}",
 )
 
 MBBQ_NL_CONFIG = DatasetConfig(
@@ -100,7 +101,7 @@ MBBQ_NL_CONFIG = DatasetConfig(
     source="EuroEval/mbbq-nl",
     task=MCSTEREO,
     languages=[DUTCH],
-    splits=["val", "test"],
+    train_split=None,
 )
 
 
@@ -158,7 +159,7 @@ COPA_NL_CONFIG = DatasetConfig(
     task=COMMON_SENSE,
     languages=[DUTCH],
     unofficial=True,
-    _labels=["a", "b"],
+    labels=["a", "b"],
 )
 
 GOLDENSWAG_NL_CONFIG = DatasetConfig(
@@ -176,6 +177,6 @@ WINOGRANDE_NL_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-nl",
     task=COMMON_SENSE,
     languages=[DUTCH],
-    _labels=["a", "b"],
+    labels=["a", "b"],
     unofficial=True,
 )

scandeval/dataset_configs/english.py

@@ -68,9 +68,10 @@ VALEU_EN_CONFIG = DatasetConfig(
     source="EuroEval/european-values-en",
     task=EUROPEAN_VALUES,
     languages=[ENGLISH],
-    splits=["test"],
+    train_split=None,
+    val_split=None,
     bootstrap_samples=False,
-    _instruction_prompt="{text}",
+    instruction_prompt="{text}",
 )
 
 
@@ -127,6 +128,6 @@ WINOGRANDE_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-en",
     task=COMMON_SENSE,
     languages=[ENGLISH],
-    _labels=["a", "b"],
+    labels=["a", "b"],
     unofficial=True,
 )

scandeval/dataset_configs/estonian.py

@@ -60,13 +60,13 @@ WINOGRANDE_ET_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-et",
     task=COMMON_SENSE,
     languages=[ESTONIAN],
-    _prompt_prefix="Sulle esitatakse lüngaga (_) tekstülesanded, "
+    prompt_prefix="Sulle esitatakse lüngaga (_) tekstülesanded, "
     "igal ülesandel on kaks vastusevarianti (a ja b).",
-    _prompt_template="Tekstülesanne: {text}\nVastus: {label}",
-    _instruction_prompt="Tekstülesanne: {text}\n\n"
+    prompt_template="Tekstülesanne: {text}\nVastus: {label}",
+    instruction_prompt="Tekstülesanne: {text}\n\n"
     "Sinu ülesanne on valida lünka sobiv vastusevariant. "
     "Vasta ainult {labels_str}. Muud vastused ei ole lubatud.",
-    _labels=["a", "b"],
+    labels=["a", "b"],
 )
 
 VALEU_ET_CONFIG = DatasetConfig(
@@ -75,9 +75,10 @@ VALEU_ET_CONFIG = DatasetConfig(
     source="EuroEval/european-values-et",
     task=EUROPEAN_VALUES,
     languages=[ESTONIAN],
-    splits=["test"],
+    train_split=None,
+    val_split=None,
     bootstrap_samples=False,
-    _instruction_prompt="{text}",
+    instruction_prompt="{text}",
 )
 
 ### Unofficial datasets ###
@@ -97,7 +98,7 @@ EXAM_ET_CONFIG = DatasetConfig(
     source="EuroEval/exam-et",
     task=KNOW,
     languages=[ESTONIAN],
-    _labels=["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o"],
+    labels=["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o"],
     unofficial=True,
 )
 

scandeval/dataset_configs/faroese.py

@@ -12,7 +12,7 @@ FOSENT_CONFIG = DatasetConfig(
     source="EuroEval/fosent",
     task=SENT,
     languages=[FAROESE],
-    _num_few_shot_examples=5,
+    num_few_shot_examples=5,
 )
 
 SCALA_FO_CONFIG = DatasetConfig(

scandeval/dataset_configs/finnish.py

@@ -12,7 +12,7 @@ SCANDISENT_FI_CONFIG = DatasetConfig(
     source="EuroEval/scandisent-fi-mini",
     task=SENT,
     languages=[FINNISH],
-    _labels=["negative", "positive"],
+    labels=["negative", "positive"],
 )
 
 TURKU_NER_FI_CONFIG = DatasetConfig(
@@ -61,9 +61,10 @@ VALEU_FI_CONFIG = DatasetConfig(
     source="EuroEval/european-values-fi",
     task=EUROPEAN_VALUES,
     languages=[FINNISH],
-    splits=["test"],
+    train_split=None,
+    val_split=None,
     bootstrap_samples=False,
-    _instruction_prompt="{text}",
+    instruction_prompt="{text}",
 )
 
 
@@ -102,6 +103,6 @@ WINOGRANDE_FI_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-fi",
     task=COMMON_SENSE,
     languages=[FINNISH],
-    _labels=["a", "b"],
+    labels=["a", "b"],
     unofficial=True,
 )

scandeval/dataset_configs/french.py

@@ -12,8 +12,8 @@ ALLOCINE_CONFIG = DatasetConfig(
     source="EuroEval/allocine-mini",
     task=SENT,
     languages=[FRENCH],
-    _labels=["negative", "positive"],
-    _prompt_label_mapping=dict(positive="positif", negative="négatif"),
+    labels=["negative", "positive"],
+    prompt_label_mapping=dict(positive="positif", negative="négatif"),
 )
 
 SCALA_FR_CONFIG = DatasetConfig(
@@ -70,9 +70,10 @@ VALEU_FR_CONFIG = DatasetConfig(
     source="EuroEval/european-values-fr",
     task=EUROPEAN_VALUES,
     languages=[FRENCH],
-    splits=["test"],
+    train_split=None,
+    val_split=None,
     bootstrap_samples=False,
-    _instruction_prompt="{text}",
+    instruction_prompt="{text}",
 )
 
 
@@ -111,6 +112,6 @@ WINOGRANDE_FR_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-fr",
     task=COMMON_SENSE,
     languages=[FRENCH],
-    _labels=["a", "b"],
+    labels=["a", "b"],
     unofficial=True,
 )

scandeval/dataset_configs/german.py

@@ -68,9 +68,10 @@ VALEU_DE_CONFIG = DatasetConfig(
     source="EuroEval/european-values-de",
     task=EUROPEAN_VALUES,
     languages=[GERMAN],
-    splits=["test"],
+    train_split=None,
+    val_split=None,
     bootstrap_samples=False,
-    _instruction_prompt="{text}",
+    instruction_prompt="{text}",
 )
 
 
@@ -127,6 +128,6 @@ WINOGRANDE_DE_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-de",
     task=COMMON_SENSE,
     languages=[GERMAN],
-    _labels=["a", "b"],
+    labels=["a", "b"],
     unofficial=True,
 )

scandeval/dataset_configs/greek.py

@@ -12,7 +12,7 @@ GREEK_SA_CONFIG = DatasetConfig(
     source="EuroEval/greek-sa-mini",
     task=SENT,
     languages=[GREEK],
-    _labels=["negative", "positive"],
+    labels=["negative", "positive"],
 )
 
 SCALA_EL_CONFIG = DatasetConfig(

scandeval/dataset_configs/hungarian.py

@@ -60,5 +60,5 @@ WINOGRANDE_HU_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-hu",
     task=COMMON_SENSE,
     languages=[HUNGARIAN],
-    _labels=["a", "b"],
+    labels=["a", "b"],
 )

scandeval/dataset_configs/icelandic.py

@@ -60,7 +60,7 @@ WINOGRANDE_IS_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-is",
     task=COMMON_SENSE,
     languages=[ICELANDIC],
-    _labels=["a", "b"],
+    labels=["a", "b"],
 )
 
 VALEU_IS_CONFIG = DatasetConfig(
@@ -69,9 +69,10 @@ VALEU_IS_CONFIG = DatasetConfig(
     source="EuroEval/european-values-is",
     task=EUROPEAN_VALUES,
     languages=[ICELANDIC],
-    splits=["test"],
+    train_split=None,
+    val_split=None,
     bootstrap_samples=False,
-    _instruction_prompt="{text}",
+    instruction_prompt="{text}",
 )
 
 

scandeval/dataset_configs/italian.py

@@ -68,9 +68,10 @@ VALEU_IT_CONFIG = DatasetConfig(
     source="EuroEval/european-values-it",
     task=EUROPEAN_VALUES,
     languages=[ITALIAN],
-    splits=["test"],
+    train_split=None,
+    val_split=None,
     bootstrap_samples=False,
-    _instruction_prompt="{text}",
+    instruction_prompt="{text}",
 )
 
 
@@ -118,6 +119,6 @@ WINOGRANDE_IT_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-it",
     task=COMMON_SENSE,
     languages=[ITALIAN],
-    _labels=["a", "b"],
+    labels=["a", "b"],
     unofficial=True,
 )

scandeval/dataset_configs/latvian.py

@@ -61,7 +61,7 @@ COPA_LV_CONFIG = DatasetConfig(
     source="EuroEval/copa-lv",
     task=COMMON_SENSE,
     languages=[LATVIAN],
-    _labels=["a", "b"],
+    labels=["a", "b"],
 )
 
 
@@ -82,6 +82,6 @@ WINOGRANDE_LV_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-lv",
     task=COMMON_SENSE,
     languages=[LATVIAN],
-    _labels=["a", "b"],
+    labels=["a", "b"],
     unofficial=True,
 )

scandeval/dataset_configs/lithuanian.py

@@ -60,7 +60,7 @@ WINOGRANDE_LT_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-lt",
     task=COMMON_SENSE,
     languages=[LITHUANIAN],
-    _labels=["a", "b"],
+    labels=["a", "b"],
 )
 
 ### Unofficial datasets ###

scandeval/dataset_configs/norwegian.py

@@ -52,7 +52,7 @@ NORQUAD_CONFIG = DatasetConfig(
     source="EuroEval/norquad-mini",
     task=RC,
     languages=[NORWEGIAN_BOKMÅL, NORWEGIAN_NYNORSK, NORWEGIAN],
-    _num_few_shot_examples=2,
+    num_few_shot_examples=2,
 )
 
 NO_SAMMENDRAG_CONFIG = DatasetConfig(
@@ -85,7 +85,7 @@ NOR_COMMON_SENSE_QA_CONFIG = DatasetConfig(
     source="EuroEval/nor-common-sense-qa",
     task=COMMON_SENSE,
     languages=[NORWEGIAN_BOKMÅL, NORWEGIAN_NYNORSK, NORWEGIAN],
-    _labels=["a", "b", "c", "d", "e"],
+    labels=["a", "b", "c", "d", "e"],
 )
 
 VALEU_NO_CONFIG = DatasetConfig(
@@ -94,9 +94,10 @@ VALEU_NO_CONFIG = DatasetConfig(
     source="EuroEval/european-values-no",
     task=EUROPEAN_VALUES,
     languages=[NORWEGIAN_BOKMÅL, NORWEGIAN_NYNORSK, NORWEGIAN],
-    splits=["test"],
+    train_split=None,
+    val_split=None,
     bootstrap_samples=False,
-    _instruction_prompt="{text}",
+    instruction_prompt="{text}",
 )
 
 
@@ -207,6 +208,6 @@ WINOGRANDE_NO_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-no",
     task=COMMON_SENSE,
     languages=[NORWEGIAN_BOKMÅL, NORWEGIAN_NYNORSK, NORWEGIAN],
-    _labels=["a", "b"],
+    labels=["a", "b"],
     unofficial=True,
 )

scandeval/dataset_configs/polish.py

@@ -60,7 +60,7 @@ WINOGRANDE_PL_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-pl",
     task=COMMON_SENSE,
     languages=[POLISH],
-    _labels=["a", "b"],
+    labels=["a", "b"],
 )
 
 VALEU_PL_CONFIG = DatasetConfig(
@@ -69,9 +69,10 @@ VALEU_PL_CONFIG = DatasetConfig(
     source="EuroEval/european-values-pl",
     task=EUROPEAN_VALUES,
     languages=[POLISH],
-    splits=["test"],
+    train_split=None,
+    val_split=None,
     bootstrap_samples=False,
-    _instruction_prompt="{text}",
+    instruction_prompt="{text}",
 )
 
 

scandeval/dataset_configs/portuguese.py

@@ -12,7 +12,7 @@ SST2_PT_CONFIG = DatasetConfig(
     source="EuroEval/sst2-pt-mini",
     task=SENT,
     languages=[PORTUGUESE, EUROPEAN_PORTUGUESE],
-    _labels=["positive", "negative"],
+    labels=["positive", "negative"],
 )
 
 SCALA_PT = DatasetConfig(
@@ -69,9 +69,10 @@ VALEU_PT_CONFIG = DatasetConfig(
     source="EuroEval/european-values-pt",
     task=EUROPEAN_VALUES,
     languages=[PORTUGUESE, EUROPEAN_PORTUGUESE],
-    splits=["test"],
+    train_split=None,
+    val_split=None,
     bootstrap_samples=False,
-    _instruction_prompt="{text}",
+    instruction_prompt="{text}",
 )
 
 
@@ -92,6 +93,6 @@ WINOGRANDE_PT_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-pt",
     task=COMMON_SENSE,
     languages=[PORTUGUESE, EUROPEAN_PORTUGUESE],
-    _labels=["a", "b"],
+    labels=["a", "b"],
     unofficial=True,
 )

scandeval/dataset_configs/romanian.py

@@ -12,7 +12,7 @@ ROSENT_CONFIG = DatasetConfig(
     source="EuroEval/ro-sent-mini",
     task=SENT,
     languages=[ROMANIAN],
-    _labels=["positive", "negative"],
+    labels=["positive", "negative"],
 )
 
 SCALA_RO_CONFIG = DatasetConfig(
@@ -61,5 +61,5 @@ WINOGRANDE_RO_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-ro",
     task=COMMON_SENSE,
     languages=[ROMANIAN],
-    _labels=["a", "b"],
+    labels=["a", "b"],
 )

scandeval/dataset_configs/serbian.py

@@ -60,5 +60,5 @@ WINOGRANDE_SR_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-sr",
     task=COMMON_SENSE,
     languages=[SERBIAN],
-    _labels=["a", "b"],
+    labels=["a", "b"],
 )

scandeval/dataset_configs/slovene.py

@@ -52,5 +52,5 @@ WINOGRANDE_SL_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-sl",
     task=COMMON_SENSE,
     languages=[SLOVENE],
-    _labels=["a", "b"],
+    labels=["a", "b"],
 )

scandeval/dataset_configs/spanish.py

@@ -68,9 +68,10 @@ VALEU_ES_CONFIG = DatasetConfig(
     source="EuroEval/european-values-es",
     task=EUROPEAN_VALUES,
     languages=[SPANISH],
-    splits=["test"],
+    train_split=None,
+    val_split=None,
     bootstrap_samples=False,
-    _instruction_prompt="{text}",
+    instruction_prompt="{text}",
 )
 
 
@@ -127,6 +128,6 @@ WINOGRANDE_ES_CONFIG = DatasetConfig(
     source="EuroEval/winogrande-es",
     task=COMMON_SENSE,
     languages=[SPANISH],
-    _labels=["a", "b"],
+    labels=["a", "b"],
     unofficial=True,
 )