EuroEval 15.12.0__py3-none-any.whl → 16.7.1__py3-none-any.whl

euroeval/dataset_configs/__init__.py

@@ -3,19 +3,33 @@
 from ..data_models import DatasetConfig
 from ..languages import get_all_languages
 from ..tasks import SPEED
+from ..utils import load_custom_datasets_module
+from .bosnian import *  # noqa: F403
+from .bulgarian import *  # noqa: F403
+from .croatian import *  # noqa: F403
+from .czech import *  # noqa: F403
 from .danish import *  # noqa: F403
 from .dutch import *  # noqa: F403
 from .english import *  # noqa: F403
+from .estonian import *  # noqa: F403
 from .faroese import *  # noqa: F403
 from .finnish import *  # noqa: F403
 from .french import *  # noqa: F403
 from .german import *  # noqa: F403
+from .greek import *  # noqa: F403
 from .icelandic import *  # noqa: F403
 from .italian import *  # noqa: F403
+from .latvian import *  # noqa: F403
+from .lithuanian import *  # noqa: F403
 from .norwegian import *  # noqa: F403
+from .polish import *  # noqa: F403
 from .portuguese import *  # noqa: F403
+from .serbian import *  # noqa: F403
+from .slovak import *  # noqa: F403
+from .slovene import *  # noqa: F403
 from .spanish import *  # noqa: F403
 from .swedish import *  # noqa: F403
+from .ukrainian import *  # noqa: F403
 
 
 def get_all_dataset_configs() -> dict[str, DatasetConfig]:
@@ -24,14 +38,21 @@ def get_all_dataset_configs() -> dict[str, DatasetConfig]:
     Returns:
         A mapping between names of datasets and their configurations.
     """
+    globals_dict = globals()
+    module = load_custom_datasets_module()
+    if module is not None:
+        globals_dict |= vars(module)
     dataset_configs = [
-        cfg for cfg in globals().values() if isinstance(cfg, DatasetConfig)
+        cfg
+        for cfg in globals_dict.values()
+        if isinstance(cfg, DatasetConfig) and cfg.task != SPEED
     ]
     assert len(dataset_configs) == len({cfg.name for cfg in dataset_configs}), (
         "There are duplicate dataset configurations. Please ensure that each dataset "
         "has a unique name."
     )
-    return {cfg.name: cfg for cfg in dataset_configs}
+    mapping = {cfg.name: cfg for cfg in dataset_configs}
+    return mapping
 
 
 def get_dataset_config(dataset_name: str) -> DatasetConfig:
@@ -56,8 +77,9 @@ def get_dataset_config(dataset_name: str) -> DatasetConfig:
 
 SPEED_CONFIG = DatasetConfig(
     name="speed",
-    pretty_name="the speed estimation benchmark",
-    huggingface_id="",
+    pretty_name="",
+    source="",
     task=SPEED,
     languages=list(get_all_languages().values()),
+    _logging_string="the speed estimation benchmark",
 )

euroeval/dataset_configs/bosnian.py

@@ -0,0 +1,39 @@
+"""All Bosnian dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import BOSNIAN
+from ..tasks import NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+MMS_BS_CONFIG = DatasetConfig(
+    name="mms-bs",
+    pretty_name="MMS-bs",
+    source="EuroEval/mms-bs-mini",
+    task=SENT,
+    languages=[BOSNIAN],
+)
+
+WIKIANN_BS_CONFIG = DatasetConfig(
+    name="wikiann-bs",
+    pretty_name="WikiANN-bs",
+    source="EuroEval/wikiann-bs-mini",
+    task=NER,
+    languages=[BOSNIAN],
+)
+
+MULTI_WIKI_QA_BS_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-bs",
+    pretty_name="MultiWikiQA-bs",
+    source="EuroEval/multi-wiki-qa-bs-mini",
+    task=RC,
+    languages=[BOSNIAN],
+)
+
+LR_SUM_BS_CONFIG = DatasetConfig(
+    name="lr-sum-bs",
+    pretty_name="LRSum-bs",
+    source="EuroEval/lr-sum-bs-mini",
+    task=SUMM,
+    languages=[BOSNIAN],
+)

euroeval/dataset_configs/bulgarian.py

@@ -0,0 +1,56 @@
+"""All Bulgarian dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import BULGARIAN
+from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT
+
+### Official datasets ###
+
+CINEXIO_CONFIG = DatasetConfig(
+    name="cinexio",
+    pretty_name="Cinexio",
+    source="EuroEval/cinexio-mini",
+    task=SENT,
+    languages=[BULGARIAN],
+)
+
+SCALA_BG_CONFIG = DatasetConfig(
+    name="scala-bg",
+    pretty_name="ScaLA-bg",
+    source="EuroEval/scala-bg",
+    task=LA,
+    languages=[BULGARIAN],
+)
+
+BG_NER_BSNLP_CONFIG = DatasetConfig(
+    name="bg-ner-bsnlp",
+    pretty_name="BG-NER-BSNLp",
+    source="EuroEval/bg-ner-bsnlp-mini",
+    task=NER,
+    languages=[BULGARIAN],
+)
+
+MULTI_WIKI_QA_BG_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-bg",
+    pretty_name="MultiWikiQA-bg",
+    source="EuroEval/multi-wiki-qa-bg-mini",
+    task=RC,
+    languages=[BULGARIAN],
+)
+
+EXAMS_BG_CONFIG = DatasetConfig(
+    name="exams-bg",
+    pretty_name="Exams-bg",
+    source="EuroEval/exams-bg-mini",
+    task=KNOW,
+    languages=[BULGARIAN],
+)
+
+WINOGRANDE_BG_CONFIG = DatasetConfig(
+    name="winogrande-bg",
+    pretty_name="Winogrande-bg",
+    source="EuroEval/winogrande-bg",
+    task=COMMON_SENSE,
+    languages=[BULGARIAN],
+    _labels=["a", "b"],
+)

euroeval/dataset_configs/croatian.py

@@ -0,0 +1,56 @@
+"""All Croatian dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import CROATIAN
+from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT
+
+### Official datasets ###
+
+MMS_HR_CONFIG = DatasetConfig(
+    name="mms-hr",
+    pretty_name="MMS-hr",
+    source="EuroEval/mms-hr-mini",
+    task=SENT,
+    languages=[CROATIAN],
+)
+
+SCALA_HR_CONFIG = DatasetConfig(
+    name="scala-hr",
+    pretty_name="ScaLA-hr",
+    source="EuroEval/scala-hr",
+    task=LA,
+    languages=[CROATIAN],
+)
+
+WIKIANN_HR_CONFIG = DatasetConfig(
+    name="wikiann-hr",
+    pretty_name="WikiANN-hr",
+    source="EuroEval/wikiann-hr-mini",
+    task=NER,
+    languages=[CROATIAN],
+)
+
+MULTI_WIKI_QA_HR_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-hr",
+    pretty_name="MultiWikiQA-hr",
+    source="EuroEval/multi-wiki-qa-hr-mini",
+    task=RC,
+    languages=[CROATIAN],
+)
+
+MMLU_HR_CONFIG = DatasetConfig(
+    name="mmlu-hr",
+    pretty_name="MMLU-hr",
+    source="EuroEval/mmlu-hr-mini",
+    task=KNOW,
+    languages=[CROATIAN],
+)
+
+WINOGRANDE_HR_CONFIG = DatasetConfig(
+    name="winogrande-hr",
+    pretty_name="Winogrande-hr",
+    source="EuroEval/winogrande-hr",
+    task=COMMON_SENSE,
+    languages=[CROATIAN],
+    _labels=["a", "b"],
+)

euroeval/dataset_configs/czech.py

@@ -0,0 +1,75 @@
+"""All Czech dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import CZECH
+from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+CSFD_SENTIMENT_CONFIG = DatasetConfig(
+    name="csfd-sentiment",
+    pretty_name="CSFD Sentiment",
+    source="EuroEval/csfd-sentiment-mini",
+    task=SENT,
+    languages=[CZECH],
+)
+
+CS_GEC_CONFIG = DatasetConfig(
+    name="cs-gec",
+    pretty_name="CS-GEC",
+    source="EuroEval/cs-gec-mini",
+    task=LA,
+    languages=[CZECH],
+)
+
+PONER_CONFIG = DatasetConfig(
+    name="poner",
+    pretty_name="PoNER",
+    source="EuroEval/poner-mini",
+    task=NER,
+    languages=[CZECH],
+)
+
+SQAD_CONFIG = DatasetConfig(
+    name="sqad",
+    pretty_name="SQAD",
+    source="EuroEval/sqad-mini",
+    task=RC,
+    languages=[CZECH],
+)
+
+CZECH_NEWS_CONFIG = DatasetConfig(
+    name="czech-news",
+    pretty_name="Czech News",
+    source="EuroEval/czech-news-mini",
+    task=SUMM,
+    languages=[CZECH],
+)
+
+UMIMETO_QA_CONFIG = DatasetConfig(
+    name="umimeto-qa",
+    pretty_name="Umimeto QA",
+    source="EuroEval/umimeto-qa",
+    task=KNOW,
+    languages=[CZECH],
+)
+
+HELLASWAG_CS_CONFIG = DatasetConfig(
+    name="hellaswag-cs",
+    pretty_name="HellaSwag-cs",
+    source="EuroEval/hellaswag-cs-mini",
+    task=COMMON_SENSE,
+    languages=[CZECH],
+)
+
+
+### Unofficial datasets ###
+
+SCALA_CS_CONFIG = DatasetConfig(
+    name="scala-cs",
+    pretty_name="ScaLA-cs",
+    source="EuroEval/scala-cs",
+    task=LA,
+    languages=[CZECH],
+    unofficial=True,
+)

euroeval/dataset_configs/danish.py

@@ -1,79 +1,83 @@
 """All Danish dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import DA
-from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+from ..languages import DANISH
+from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 ANGRY_TWEETS_CONFIG = DatasetConfig(
     name="angry-tweets",
-    pretty_name="the truncated version of the Danish sentiment classification "
-    "dataset AngryTweets",
-    huggingface_id="EuroEval/angry-tweets-mini",
+    pretty_name="AngryTweets",
+    source="EuroEval/angry-tweets-mini",
     task=SENT,
-    languages=[DA],
+    languages=[DANISH],
 )
 
 SCALA_DA_CONFIG = DatasetConfig(
     name="scala-da",
-    pretty_name="the Danish part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-da",
+    pretty_name="ScaLA-da",
+    source="EuroEval/scala-da",
     task=LA,
-    languages=[DA],
+    languages=[DANISH],
 )
 
 DANSK_CONFIG = DatasetConfig(
     name="dansk",
-    pretty_name="the truncated version of the Danish named entity recognition "
-    "dataset DANSK",
-    huggingface_id="EuroEval/dansk-mini",
+    pretty_name="DANSK",
+    source="EuroEval/dansk-mini",
     task=NER,
-    languages=[DA],
+    languages=[DANISH],
 )
 
-SCANDIQA_DA_CONFIG = DatasetConfig(
-    name="scandiqa-da",
-    pretty_name="the Danish part of the truncated version of the question answering "
-    "dataset ScandiQA",
-    huggingface_id="EuroEval/scandiqa-da-mini",
+MULTI_WIKI_QA_DA_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-da",
+    pretty_name="MultiWikiQA-da",
+    source="EuroEval/multi-wiki-qa-da-mini",
     task=RC,
-    languages=[DA],
+    languages=[DANISH],
 )
 
 NORDJYLLAND_NEWS_CONFIG = DatasetConfig(
     name="nordjylland-news",
-    pretty_name="the truncated version of the Danish summarisation dataset "
-    "Nordjylland News",
-    huggingface_id="EuroEval/nordjylland-news-mini",
+    pretty_name="Nordjylland News",
+    source="EuroEval/nordjylland-news-mini",
     task=SUMM,
-    languages=[DA],
+    languages=[DANISH],
 )
 
 DANSKE_TALEMAADER_CONFIG = DatasetConfig(
     name="danske-talemaader",
-    pretty_name="the truncated version of the Danish knowledge dataset Danske "
-    "Talemåder",
-    huggingface_id="EuroEval/danske-talemaader",
+    pretty_name="Danske Talemåder",
+    source="EuroEval/danske-talemaader",
     task=KNOW,
-    languages=[DA],
+    languages=[DANISH],
 )
 
 DANISH_CITIZEN_TESTS_CONFIG = DatasetConfig(
     name="danish-citizen-tests",
-    pretty_name="the Danish knowledge dataset Danish Citizen Tests",
-    huggingface_id="EuroEval/danish-citizen-tests-updated",
+    pretty_name="Danish Citizen Tests",
+    source="EuroEval/danish-citizen-tests-updated",
     task=KNOW,
-    languages=[DA],
+    languages=[DANISH],
 )
 
 HELLASWAG_DA_CONFIG = DatasetConfig(
     name="hellaswag-da",
-    pretty_name="the truncated version of the Danish common-sense reasoning dataset "
-    "HellaSwag-da, translated from the English HellaSwag dataset",
-    huggingface_id="EuroEval/hellaswag-da-mini",
+    pretty_name="HellaSwag-da",
+    source="EuroEval/hellaswag-da-mini",
     task=COMMON_SENSE,
-    languages=[DA],
+    languages=[DANISH],
+)
+
+VALEU_DA_CONFIG = DatasetConfig(
+    name="valeu-da",
+    pretty_name="ValEU-da",
+    source="EuroEval/european-values-da",
+    task=EUROPEAN_VALUES,
+    languages=[DANISH],
+    splits=["test"],
+    bootstrap_samples=False,
 )
 
 
@@ -81,40 +85,64 @@ HELLASWAG_DA_CONFIG = DatasetConfig(
 
 DANE_CONFIG = DatasetConfig(
     name="dane",
-    pretty_name="the truncated version of the Danish named entity recognition "
-    "dataset DaNE",
-    huggingface_id="EuroEval/dane-mini",
+    pretty_name="DaNE",
+    source="EuroEval/dane-mini",
     task=NER,
-    languages=[DA],
+    languages=[DANISH],
     unofficial=True,
 )
 
 MMLU_DA_CONFIG = DatasetConfig(
     name="mmlu-da",
-    pretty_name="the truncated version of the Danish knowledge dataset MMLU-da, "
-    "translated from the English MMLU dataset",
-    huggingface_id="EuroEval/mmlu-da-mini",
+    pretty_name="MMLU-da",
+    source="EuroEval/mmlu-da-mini",
     task=KNOW,
-    languages=[DA],
+    languages=[DANISH],
     unofficial=True,
 )
 
 ARC_DA_CONFIG = DatasetConfig(
     name="arc-da",
-    pretty_name="the truncated version of the Danish knowledge dataset ARC-da, "
-    "translated from the English ARC dataset",
-    huggingface_id="EuroEval/arc-da-mini",
+    pretty_name="ARC-da",
+    source="EuroEval/arc-da-mini",
     task=KNOW,
-    languages=[DA],
+    languages=[DANISH],
     unofficial=True,
 )
 
 BELEBELE_DA_CONFIG = DatasetConfig(
     name="belebele-da",
-    pretty_name="the Danish multiple choice reading comprehension dataset BeleBele-da, "
-    "translated from the English BeleBele dataset",
-    huggingface_id="EuroEval/belebele-da-mini",
+    pretty_name="Belebele-da",
+    source="EuroEval/belebele-da-mini",
     task=MCRC,
-    languages=[DA],
+    languages=[DANISH],
+    unofficial=True,
+)
+
+SCANDIQA_DA_CONFIG = DatasetConfig(
+    name="scandiqa-da",
+    pretty_name="ScandiQA-da",
+    source="EuroEval/scandiqa-da-mini",
+    task=RC,
+    languages=[DANISH],
+    unofficial=True,
+)
+
+GOLDENSWAG_DA_CONFIG = DatasetConfig(
+    name="goldenswag-da",
+    pretty_name="GoldenSwag-da",
+    source="EuroEval/goldenswag-da-mini",
+    task=COMMON_SENSE,
+    languages=[DANISH],
+    unofficial=True,
+)
+
+WINOGRANDE_DA_CONFIG = DatasetConfig(
+    name="winogrande-da",
+    pretty_name="Winogrande-da",
+    source="EuroEval/winogrande-da",
+    task=COMMON_SENSE,
+    languages=[DANISH],
+    _labels=["a", "b"],
     unofficial=True,
 )

euroeval/dataset_configs/dutch.py

@@ -1,72 +1,77 @@
 """All Dutch dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import NL
-from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+from ..languages import DUTCH
+from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 DBRD_CONFIG = DatasetConfig(
     name="dbrd",
-    pretty_name="the truncated version of the Dutch sentiment classification "
-    "dataset DBRD",
-    huggingface_id="EuroEval/dbrd-mini",
+    pretty_name="DBRD",
+    source="EuroEval/dbrd-mini",
     task=SENT,
-    languages=[NL],
+    languages=[DUTCH],
     _labels=["negative", "positive"],
 )
 
 SCALA_NL_CONFIG = DatasetConfig(
     name="scala-nl",
-    pretty_name="the Dutch part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-nl",
+    pretty_name="ScaLA-nl",
+    source="EuroEval/scala-nl",
     task=LA,
-    languages=[NL],
+    languages=[DUTCH],
 )
 
 CONLL_NL_CONFIG = DatasetConfig(
     name="conll-nl",
-    pretty_name="the Dutch part of the truncated version of the named entity "
-    "recognition dataset CoNLL 2002",
-    huggingface_id="EuroEval/conll-nl-mini",
+    pretty_name="CoNLL-nl",
+    source="EuroEval/conll-nl-mini",
     task=NER,
-    languages=[NL],
+    languages=[DUTCH],
 )
 
 SQUAD_NL_CONFIG = DatasetConfig(
     name="squad-nl",
-    pretty_name="the truncated version of the Dutch reading comprehension dataset "
-    "SQuAD-nl, translated from the English SQuAD dataset",
-    huggingface_id="EuroEval/squad-nl-v2-mini",
+    pretty_name="SQuAD-nl",
+    source="EuroEval/squad-nl-v2-mini",
     task=RC,
-    languages=[NL],
+    languages=[DUTCH],
 )
 
 WIKI_LINGUA_NL_CONFIG = DatasetConfig(
     name="wiki-lingua-nl",
-    pretty_name="the Dutch part of the truncated version of the summarisation dataset "
-    "WikiLingua",
-    huggingface_id="EuroEval/wiki-lingua-nl-mini",
+    pretty_name="WikiLingua-nl",
+    source="EuroEval/wiki-lingua-nl-mini",
     task=SUMM,
-    languages=[NL],
+    languages=[DUTCH],
 )
 
 MMLU_NL_CONFIG = DatasetConfig(
     name="mmlu-nl",
-    pretty_name="the truncated version of the Dutch knowledge dataset MMLU-nl, "
-    "translated from the English MMLU dataset",
-    huggingface_id="EuroEval/mmlu-nl-mini",
+    pretty_name="MMLU-nl",
+    source="EuroEval/mmlu-nl-mini",
     task=KNOW,
-    languages=[NL],
+    languages=[DUTCH],
 )
 
 HELLASWAG_NL_CONFIG = DatasetConfig(
     name="hellaswag-nl",
-    pretty_name="the truncated version of the Dutch common-sense reasoning dataset "
-    "HellaSwag-nl, translated from the English HellaSwag dataset",
-    huggingface_id="EuroEval/hellaswag-nl-mini",
+    pretty_name="HellaSwag-nl",
+    source="EuroEval/hellaswag-nl-mini",
     task=COMMON_SENSE,
-    languages=[NL],
+    languages=[DUTCH],
+)
+
+VALEU_NL_CONFIG = DatasetConfig(
+    name="valeu-nl",
+    pretty_name="VaLEU-nl",
+    source="EuroEval/european-values-nl",
+    task=EUROPEAN_VALUES,
+    languages=[DUTCH],
+    splits=["test"],
+    bootstrap_samples=False,
+    _instruction_prompt="{text}",
 )
 
 
@@ -74,39 +79,64 @@ HELLASWAG_NL_CONFIG = DatasetConfig(
 
 DUTCH_COLA_CONFIG = DatasetConfig(
     name="dutch-cola",
-    pretty_name="the truncated version of the Dutch linguistic acceptability dataset "
-    "Dutch CoLA",
-    huggingface_id="EuroEval/dutch-cola",
+    pretty_name="Dutch CoLA",
+    source="EuroEval/dutch-cola",
     task=LA,
-    languages=[NL],
+    languages=[DUTCH],
     unofficial=True,
 )
 
 DUTCH_COLA_FULL_CONFIG = DatasetConfig(
     name="dutch-cola-full",
-    pretty_name="the Dutch linguistic acceptability dataset Dutch CoLA",
-    huggingface_id="EuroEval/dutch-cola-full",
+    pretty_name="Dutch CoLA Full",
+    source="EuroEval/dutch-cola-full",
     task=LA,
-    languages=[NL],
+    languages=[DUTCH],
     unofficial=True,
 )
 
 ARC_NL_CONFIG = DatasetConfig(
     name="arc-nl",
-    pretty_name="the truncated version of the Dutch knowledge dataset ARC-nl, "
-    "translated from the English ARC dataset",
-    huggingface_id="EuroEval/arc-nl-mini",
+    pretty_name="ARC-nl",
+    source="EuroEval/arc-nl-mini",
     task=KNOW,
-    languages=[NL],
+    languages=[DUTCH],
     unofficial=True,
 )
 
 BELEBELE_NL_CONFIG = DatasetConfig(
     name="belebele-nl",
-    pretty_name="the Dutch multiple choice reading comprehension dataset BeleBele-nl, "
-    "translated from the English BeleBele dataset",
-    huggingface_id="EuroEval/belebele-nl-mini",
+    pretty_name="Belebele-nl",
+    source="EuroEval/belebele-nl-mini",
     task=MCRC,
-    languages=[NL],
+    languages=[DUTCH],
+    unofficial=True,
+)
+
+MULTI_WIKI_QA_NL_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-nl",
+    pretty_name="MultiWikiQA-nl",
+    source="EuroEval/multi-wiki-qa-nl-mini",
+    task=RC,
+    languages=[DUTCH],
+    unofficial=True,
+)
+
+GOLDENSWAG_NL_CONFIG = DatasetConfig(
+    name="goldenswag-nl",
+    pretty_name="GoldenSwag-nl",
+    source="EuroEval/goldenswag-nl-mini",
+    task=COMMON_SENSE,
+    languages=[DUTCH],
+    unofficial=True,
+)
+
+WINOGRANDE_NL_CONFIG = DatasetConfig(
+    name="winogrande-nl",
+    pretty_name="Winogrande-nl",
+    source="EuroEval/winogrande-nl",
+    task=COMMON_SENSE,
+    languages=[DUTCH],
+    _labels=["a", "b"],
     unofficial=True,
 )