EuroEval 15.12.0__py3-none-any.whl → 16.7.1__py3-none-any.whl

euroeval/dataset_configs/english.py

@@ -1,97 +1,132 @@
 """All English dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import EN
-from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+from ..languages import ENGLISH
+from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 SST5_CONFIG = DatasetConfig(
     name="sst5",
-    pretty_name="the truncated version of the English sentiment classification "
-    "dataset SST5",
-    huggingface_id="EuroEval/sst5-mini",
+    pretty_name="SST-5",
+    source="EuroEval/sst5-mini",
     task=SENT,
-    languages=[EN],
+    languages=[ENGLISH],
 )
 
 SCALA_EN_CONFIG = DatasetConfig(
     name="scala-en",
-    pretty_name="the English part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-en",
+    pretty_name="ScaLA-en",
+    source="EuroEval/scala-en",
     task=LA,
-    languages=[EN],
+    languages=[ENGLISH],
 )
 
 CONLL_EN_CONFIG = DatasetConfig(
     name="conll-en",
-    pretty_name="the truncated version of the English named entity recognition "
-    "dataset CoNLL 2003",
-    huggingface_id="EuroEval/conll-en-mini",
+    pretty_name="CoNLL-en",
+    source="EuroEval/conll-en-mini",
     task=NER,
-    languages=[EN],
+    languages=[ENGLISH],
 )
 
 SQUAD_CONFIG = DatasetConfig(
     name="squad",
-    pretty_name="the truncated version of the English question answering dataset SQuAD",
-    huggingface_id="EuroEval/squad-mini",
+    pretty_name="SQuAD",
+    source="EuroEval/squad-mini",
     task=RC,
-    languages=[EN],
+    languages=[ENGLISH],
 )
 
 CNN_DAILYMAIL_CONFIG = DatasetConfig(
     name="cnn-dailymail",
-    pretty_name="the truncated version of the English summarisation dataset "
-    "CNN-DailyMail",
-    huggingface_id="EuroEval/cnn-dailymail-mini",
+    pretty_name="CNN/DailyMail",
+    source="EuroEval/cnn-dailymail-mini",
     task=SUMM,
-    languages=[EN],
+    languages=[ENGLISH],
 )
 
 LIFE_IN_THE_UK_CONFIG = DatasetConfig(
     name="life-in-the-uk",
-    pretty_name="the English knowledge dataset Life in the UK",
-    huggingface_id="EuroEval/life-in-the-uk",
+    pretty_name="Life in the UK",
+    source="EuroEval/life-in-the-uk",
     task=KNOW,
-    languages=[EN],
+    languages=[ENGLISH],
 )
 
 HELLASWAG_CONFIG = DatasetConfig(
     name="hellaswag",
-    pretty_name="the truncated version of the English common-sense reasoning "
-    "dataset HellaSwag",
-    huggingface_id="EuroEval/hellaswag-mini",
+    pretty_name="HellaSwag",
+    source="EuroEval/hellaswag-mini",
     task=COMMON_SENSE,
-    languages=[EN],
+    languages=[ENGLISH],
+)
+
+VALEU_EN_CONFIG = DatasetConfig(
+    name="valeu-en",
+    pretty_name="VaLEU-en",
+    source="EuroEval/european-values-en",
+    task=EUROPEAN_VALUES,
+    languages=[ENGLISH],
+    splits=["test"],
+    bootstrap_samples=False,
+    _instruction_prompt="{text}",
 )
 
 
 ### Unofficial datasets ###
 
+XQUAD_EN_CONFIG = DatasetConfig(
+    name="xquad-en",
+    pretty_name="XQuAD-en",
+    source="EuroEval/xquad-en",
+    task=RC,
+    languages=[ENGLISH],
+    unofficial=True,
+)
+
 ARC_CONFIG = DatasetConfig(
     name="arc",
-    pretty_name="the truncated version of the English knowledge dataset ARC",
-    huggingface_id="EuroEval/arc-mini",
+    pretty_name="ARC",
+    source="EuroEval/arc-mini",
     task=KNOW,
-    languages=[EN],
+    languages=[ENGLISH],
     unofficial=True,
 )
 
 BELEBELE_CONFIG = DatasetConfig(
     name="belebele-en",
-    pretty_name="the English multiple choice reading comprehension dataset BeleBele",
-    huggingface_id="EuroEval/belebele-mini",
+    pretty_name="Belebele-en",
+    source="EuroEval/belebele-mini",
     task=MCRC,
-    languages=[EN],
+    languages=[ENGLISH],
     unofficial=True,
 )
 
 MMLU_CONFIG = DatasetConfig(
     name="mmlu",
-    pretty_name="the truncated version of the English knowledge dataset MMLU",
-    huggingface_id="EuroEval/mmlu-mini",
+    pretty_name="MMLU",
+    source="EuroEval/mmlu-mini",
     task=KNOW,
-    languages=[EN],
+    languages=[ENGLISH],
+    unofficial=True,
+)
+
+MULTI_WIKI_QA_EN_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-en",
+    pretty_name="MultiWikiQA-en",
+    source="EuroEval/multi-wiki-qa-en-mini",
+    task=RC,
+    languages=[ENGLISH],
+    unofficial=True,
+)
+
+WINOGRANDE_CONFIG = DatasetConfig(
+    name="winogrande",
+    pretty_name="Winogrande-en",
+    source="EuroEval/winogrande-en",
+    task=COMMON_SENSE,
+    languages=[ENGLISH],
+    _labels=["a", "b"],
     unofficial=True,
 )

euroeval/dataset_configs/estonian.py

@@ -0,0 +1,111 @@
+"""All Estonian dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import ESTONIAN
+from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+ESTONIAN_VALENCE_CONFIG = DatasetConfig(
+    name="estonian-valence",
+    pretty_name="Estonian Valence",
+    source="EuroEval/estonian-valence",
+    task=SENT,
+    languages=[ESTONIAN],
+)
+
+GRAMMAR_ET_CONFIG = DatasetConfig(
+    name="grammar-et",
+    pretty_name="Grammar-et",
+    source="EuroEval/grammar-et",
+    task=LA,
+    languages=[ESTONIAN],
+)
+
+ESTNER_CONFIG = DatasetConfig(
+    name="estner",
+    pretty_name="EstNER",
+    source="EuroEval/estner-mini",
+    task=NER,
+    languages=[ESTONIAN],
+)
+
+MULTI_WIKI_QA_ET_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-et",
+    pretty_name="MultiWikiQA-et",
+    source="EuroEval/multi-wiki-qa-et-mini",
+    task=RC,
+    languages=[ESTONIAN],
+)
+
+ERR_NEWS_CONFIG = DatasetConfig(
+    name="err-news",
+    pretty_name="ERR News",
+    source="EuroEval/err-news-mini",
+    task=SUMM,
+    languages=[ESTONIAN],
+)
+
+TRIVIA_ET_CONFIG = DatasetConfig(
+    name="trivia-et",
+    pretty_name="Trivia-et",
+    source="EuroEval/trivia-et",
+    task=KNOW,
+    languages=[ESTONIAN],
+)
+
+WINOGRANDE_ET_CONFIG = DatasetConfig(
+    name="winogrande-et",
+    pretty_name="Winogrande-et",
+    source="EuroEval/winogrande-et",
+    task=COMMON_SENSE,
+    languages=[ESTONIAN],
+    _prompt_prefix="Sulle esitatakse lüngaga (_) tekstülesanded, "
+    "igal ülesandel on kaks vastusevarianti (a ja b).",
+    _prompt_template="Tekstülesanne: {text}\nVastus: {label}",
+    _instruction_prompt="Tekstülesanne: {text}\n\n"
+    "Sinu ülesanne on valida lünka sobiv vastusevariant. "
+    "Vasta ainult {labels_str}. Muud vastused ei ole lubatud.",
+    _labels=["a", "b"],
+)
+
+VALEU_ET_CONFIG = DatasetConfig(
+    name="valeu-et",
+    pretty_name="VaLEU-et",
+    source="EuroEval/european-values-et",
+    task=EUROPEAN_VALUES,
+    languages=[ESTONIAN],
+    splits=["test"],
+    bootstrap_samples=False,
+    _instruction_prompt="{text}",
+)
+
+### Unofficial datasets ###
+
+SCALA_ET_CONFIG = DatasetConfig(
+    name="scala-et",
+    pretty_name="ScaLA-et",
+    source="EuroEval/scala-et",
+    task=LA,
+    languages=[ESTONIAN],
+    unofficial=True,
+)
+
+EXAM_ET_CONFIG = DatasetConfig(
+    name="exam-et",
+    pretty_name="Exam-et",
+    source="EuroEval/exam-et",
+    task=KNOW,
+    languages=[ESTONIAN],
+    _labels=["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o"],
+    unofficial=True,
+)
+
+MMLU_ET_CONFIG = DatasetConfig(
+    name="mmlu-et",
+    pretty_name="MMLU-et",
+    source="EuroEval/mmlu-et-mini",
+    task=KNOW,
+    languages=[ESTONIAN],
+    unofficial=True,
+)

euroeval/dataset_configs/faroese.py

@@ -1,43 +1,42 @@
 """All Faroese dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import FO
+from ..languages import FAROESE
 from ..tasks import LA, NER, RC, SENT
 
 ### Official datasets ###
 
 FOSENT_CONFIG = DatasetConfig(
     name="fosent",
-    pretty_name="the Faroese sentiment classification dataset FoSent",
-    huggingface_id="EuroEval/fosent",
+    pretty_name="FoSent",
+    source="EuroEval/fosent",
     task=SENT,
-    languages=[FO],
+    languages=[FAROESE],
     _num_few_shot_examples=5,
 )
 
 SCALA_FO_CONFIG = DatasetConfig(
     name="scala-fo",
-    pretty_name="the Faroese part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-fo",
+    pretty_name="ScaLA-fo",
+    source="EuroEval/scala-fo",
     task=LA,
-    languages=[FO],
+    languages=[FAROESE],
 )
 
 FONE_CONFIG = DatasetConfig(
     name="fone",
-    pretty_name="the truncated version of the Faroese named entity recognition "
-    "dataset FoNE",
-    huggingface_id="EuroEval/fone-mini",
+    pretty_name="FoNE",
+    source="EuroEval/fone-mini",
     task=NER,
-    languages=[FO],
+    languages=[FAROESE],
 )
 
 FOQA_CONFIG = DatasetConfig(
     name="foqa",
-    pretty_name="the Faroese reading comprehension dataset FoQA",
-    huggingface_id="EuroEval/foqa",
+    pretty_name="FoQA",
+    source="EuroEval/foqa",
     task=RC,
-    languages=[FO],
+    languages=[FAROESE],
 )
 
 
@@ -45,10 +44,18 @@ FOQA_CONFIG = DatasetConfig(
 
 WIKIANN_FO_CONFIG = DatasetConfig(
     name="wikiann-fo",
-    pretty_name="the truncated version of the Faroese part of the named entity "
-    "recognition dataset WikiANN",
-    huggingface_id="EuroEval/wikiann-fo-mini",
+    pretty_name="WikiANN-fo",
+    source="EuroEval/wikiann-fo-mini",
     task=NER,
-    languages=[FO],
+    languages=[FAROESE],
+    unofficial=True,
+)
+
+MULTI_WIKI_QA_FO_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-fo",
+    pretty_name="MultiWikiQA-fo",
+    source="EuroEval/multi-wiki-qa-fo-mini",
+    task=RC,
+    languages=[FAROESE],
     unofficial=True,
 )

euroeval/dataset_configs/finnish.py

@@ -1,70 +1,107 @@
 """All Finnish dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import FI
-from ..tasks import COMMON_SENSE, LA, MCRC, NER, RC, SENT, SUMM
+from ..languages import FINNISH
+from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 SCANDISENT_FI_CONFIG = DatasetConfig(
     name="scandisent-fi",
-    pretty_name="the truncated version of the Finnish part of the binary sentiment "
-    "classification dataset ScandiSent",
-    huggingface_id="EuroEval/scandisent-fi-mini",
+    pretty_name="ScandiSent-fi",
+    source="EuroEval/scandisent-fi-mini",
     task=SENT,
-    languages=[FI],
+    languages=[FINNISH],
     _labels=["negative", "positive"],
 )
 
 TURKU_NER_FI_CONFIG = DatasetConfig(
     name="turku-ner-fi",
-    pretty_name="the Finnish part of the named entity recognition dataset Turku NER",
-    huggingface_id="EuroEval/turku-ner-fi-mini",
+    pretty_name="Turku NER-fi",
+    source="EuroEval/turku-ner-fi-mini",
     task=NER,
-    languages=[FI],
+    languages=[FINNISH],
 )
 
 TYDIQA_FI_CONFIG = DatasetConfig(
     name="tydiqa-fi",
-    pretty_name="the Finnish part of the TydiQA reading comprehension dataset",
-    huggingface_id="EuroEval/tydiqa-fi-mini",
+    pretty_name="TyDiQA-fi",
+    source="EuroEval/tydiqa-fi-mini",
     task=RC,
-    languages=[FI],
+    languages=[FINNISH],
 )
 
 XLSUM_FI_CONFIG = DatasetConfig(
     name="xlsum-fi",
-    pretty_name="the Finnish summarisation dataset XL-Sum",
-    huggingface_id="EuroEval/xlsum-fi-mini",
+    pretty_name="XLSum-fi",
+    source="EuroEval/xlsum-fi-mini",
     task=SUMM,
-    languages=[FI],
+    languages=[FINNISH],
 )
 
 HELLASWAG_FI_CONFIG = DatasetConfig(
     name="hellaswag-fi",
-    pretty_name="the truncated version of the Finnish common-sense reasoning dataset "
-    "HellaSwag-fi, translated from the English HellaSwag dataset",
-    huggingface_id="EuroEval/hellaswag-fi-mini",
+    pretty_name="HellaSwag-fi",
+    source="EuroEval/hellaswag-fi-mini",
     task=COMMON_SENSE,
-    languages=[FI],
+    languages=[FINNISH],
 )
 
 SCALA_FI_CONFIG = DatasetConfig(
     name="scala-fi",
-    pretty_name="the Finnish part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-fi",
+    pretty_name="ScaLA-fi",
+    source="EuroEval/scala-fi",
     task=LA,
-    languages=[FI],
+    languages=[FINNISH],
 )
 
+VALEU_FI_CONFIG = DatasetConfig(
+    name="valeu-fi",
+    pretty_name="VaLEU-fi",
+    source="EuroEval/european-values-fi",
+    task=EUROPEAN_VALUES,
+    languages=[FINNISH],
+    splits=["test"],
+    bootstrap_samples=False,
+    _instruction_prompt="{text}",
+)
+
+
 ### Unofficial datasets ###
 
 BELEBELE_FI_CONFIG = DatasetConfig(
     name="belebele-fi",
-    pretty_name="the Finnish multiple choice reading comprehension dataset "
-    "BeleBele-fi, translated from the English BeleBele dataset",
-    huggingface_id="EuroEval/belebele-fi-mini",
+    pretty_name="Belebele-fi",
+    source="EuroEval/belebele-fi-mini",
     task=MCRC,
-    languages=[FI],
+    languages=[FINNISH],
+    unofficial=True,
+)
+
+MULTI_WIKI_QA_FI_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-fi",
+    pretty_name="MultiWikiQA-fi",
+    source="EuroEval/multi-wiki-qa-fi-mini",
+    task=RC,
+    languages=[FINNISH],
+    unofficial=True,
+)
+
+GOLDENSWAG_FI_CONFIG = DatasetConfig(
+    name="goldenswag-fi",
+    pretty_name="GoldenSwag-fi",
+    source="EuroEval/goldenswag-fi-mini",
+    task=COMMON_SENSE,
+    languages=[FINNISH],
+    unofficial=True,
+)
+
+WINOGRANDE_FI_CONFIG = DatasetConfig(
+    name="winogrande-fi",
+    pretty_name="Winogrande-fi",
+    source="EuroEval/winogrande-fi",
+    task=COMMON_SENSE,
+    languages=[FINNISH],
+    _labels=["a", "b"],
     unofficial=True,
 )

euroeval/dataset_configs/french.py

@@ -1,72 +1,78 @@
 """All French dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import FR
-from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+from ..languages import FRENCH
+from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 ALLOCINE_CONFIG = DatasetConfig(
     name="allocine",
-    pretty_name="the truncated version of the French sentiment classification "
-    "dataset AlloCiné",
-    huggingface_id="EuroEval/allocine-mini",
+    pretty_name="AlloCiné",
+    source="EuroEval/allocine-mini",
     task=SENT,
-    languages=[FR],
+    languages=[FRENCH],
     _labels=["negative", "positive"],
     _prompt_label_mapping=dict(positive="positif", negative="négatif"),
 )
 
 SCALA_FR_CONFIG = DatasetConfig(
     name="scala-fr",
-    pretty_name="the French part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-fr",
+    pretty_name="ScaLA-fr",
+    source="EuroEval/scala-fr",
     task=LA,
-    languages=[FR],
+    languages=[FRENCH],
 )
 
 ELTEC_CONFIG = DatasetConfig(
     name="eltec",
-    pretty_name="the truncated version of the French named entity recognition "
-    "dataset ELTeC",
-    huggingface_id="EuroEval/eltec-mini",
+    pretty_name="ELTeC",
+    source="EuroEval/eltec-mini",
     task=NER,
-    languages=[FR],
+    languages=[FRENCH],
 )
 
 FQUAD_CONFIG = DatasetConfig(
     name="fquad",
-    pretty_name="the truncated version of the French reading comprehension dataset "
-    "FQuAD",
-    huggingface_id="EuroEval/fquad-mini",
+    pretty_name="FQuAD",
+    source="EuroEval/fquad-mini",
     task=RC,
-    languages=[FR],
+    languages=[FRENCH],
 )
 
 ORANGE_SUM_CONFIG = DatasetConfig(
     name="orange-sum",
-    pretty_name="the truncated version of the French summarisation dataset OrangeSum",
-    huggingface_id="EuroEval/orange-sum-mini",
+    pretty_name="OrangeSum",
+    source="EuroEval/orange-sum-mini",
     task=SUMM,
-    languages=[FR],
+    languages=[FRENCH],
 )
 
 MMLU_FR_CONFIG = DatasetConfig(
     name="mmlu-fr",
-    pretty_name="the truncated version of the French knowledge dataset MMLU-fr, "
-    "translated from the English MMLU dataset",
-    huggingface_id="EuroEval/mmlu-fr-mini",
+    pretty_name="MMLU-fr",
+    source="EuroEval/mmlu-fr-mini",
     task=KNOW,
-    languages=[FR],
+    languages=[FRENCH],
 )
 
 HELLASWAG_FR_CONFIG = DatasetConfig(
     name="hellaswag-fr",
-    pretty_name="the truncated version of the French common-sense reasoning dataset "
-    "HellaSwag-fr, translated from the English HellaSwag dataset",
-    huggingface_id="EuroEval/hellaswag-fr-mini",
+    pretty_name="HellaSwag-fr",
+    source="EuroEval/hellaswag-fr-mini",
     task=COMMON_SENSE,
-    languages=[FR],
+    languages=[FRENCH],
+)
+
+VALEU_FR_CONFIG = DatasetConfig(
+    name="valeu-fr",
+    pretty_name="VaLEU-fr",
+    source="EuroEval/european-values-fr",
+    task=EUROPEAN_VALUES,
+    languages=[FRENCH],
+    splits=["test"],
+    bootstrap_samples=False,
+    _instruction_prompt="{text}",
 )
 
 
@@ -74,10 +80,37 @@ HELLASWAG_FR_CONFIG = DatasetConfig(
 
 BELEBELE_FR_CONFIG = DatasetConfig(
     name="belebele-fr",
-    pretty_name="the French multiple choice reading comprehension dataset BeleBele-fr, "
-    "translated from the English BeleBele dataset",
-    huggingface_id="EuroEval/belebele-fr-mini",
+    pretty_name="Belebele-fr",
+    source="EuroEval/belebele-fr-mini",
     task=MCRC,
-    languages=[FR],
+    languages=[FRENCH],
+    unofficial=True,
+)
+
+MULTI_WIKI_QA_FR_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-fr",
+    pretty_name="MultiWikiQA-fr",
+    source="EuroEval/multi-wiki-qa-fr-mini",
+    task=RC,
+    languages=[FRENCH],
+    unofficial=True,
+)
+
+GOLDENSWAG_FR_CONFIG = DatasetConfig(
+    name="goldenswag-fr",
+    pretty_name="GoldenSwag-fr",
+    source="EuroEval/goldenswag-fr-mini",
+    task=COMMON_SENSE,
+    languages=[FRENCH],
+    unofficial=True,
+)
+
+WINOGRANDE_FR_CONFIG = DatasetConfig(
+    name="winogrande-fr",
+    pretty_name="Winogrande-fr",
+    source="EuroEval/winogrande-fr",
+    task=COMMON_SENSE,
+    languages=[FRENCH],
+    _labels=["a", "b"],
     unofficial=True,
 )