EuroEval 16.4.0__py3-none-any.whl → 16.5.0__py3-none-any.whl

euroeval/dataset_configs/english.py

@@ -1,77 +1,73 @@
 """All English dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import EN
+from ..languages import ENGLISH
 from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 SST5_CONFIG = DatasetConfig(
     name="sst5",
-    pretty_name="the truncated version of the English sentiment classification "
-    "dataset SST5",
-    huggingface_id="EuroEval/sst5-mini",
+    pretty_name="SST-5",
+    source="EuroEval/sst5-mini",
     task=SENT,
-    languages=[EN],
+    languages=[ENGLISH],
 )
 
 SCALA_EN_CONFIG = DatasetConfig(
     name="scala-en",
-    pretty_name="the English part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-en",
+    pretty_name="ScaLA-en",
+    source="EuroEval/scala-en",
     task=LA,
-    languages=[EN],
+    languages=[ENGLISH],
 )
 
 CONLL_EN_CONFIG = DatasetConfig(
     name="conll-en",
-    pretty_name="the truncated version of the English named entity recognition "
-    "dataset CoNLL 2003",
-    huggingface_id="EuroEval/conll-en-mini",
+    pretty_name="CoNLL-en",
+    source="EuroEval/conll-en-mini",
     task=NER,
-    languages=[EN],
+    languages=[ENGLISH],
 )
 
 SQUAD_CONFIG = DatasetConfig(
     name="squad",
-    pretty_name="the truncated version of the English question answering dataset SQuAD",
-    huggingface_id="EuroEval/squad-mini",
+    pretty_name="SQuAD",
+    source="EuroEval/squad-mini",
     task=RC,
-    languages=[EN],
+    languages=[ENGLISH],
 )
 
 CNN_DAILYMAIL_CONFIG = DatasetConfig(
     name="cnn-dailymail",
-    pretty_name="the truncated version of the English summarisation dataset "
-    "CNN-DailyMail",
-    huggingface_id="EuroEval/cnn-dailymail-mini",
+    pretty_name="CNN/DailyMail",
+    source="EuroEval/cnn-dailymail-mini",
     task=SUMM,
-    languages=[EN],
+    languages=[ENGLISH],
 )
 
 LIFE_IN_THE_UK_CONFIG = DatasetConfig(
     name="life-in-the-uk",
-    pretty_name="the English knowledge dataset Life in the UK",
-    huggingface_id="EuroEval/life-in-the-uk",
+    pretty_name="Life in the UK",
+    source="EuroEval/life-in-the-uk",
     task=KNOW,
-    languages=[EN],
+    languages=[ENGLISH],
 )
 
 HELLASWAG_CONFIG = DatasetConfig(
     name="hellaswag",
-    pretty_name="the truncated version of the English common-sense reasoning "
-    "dataset HellaSwag",
-    huggingface_id="EuroEval/hellaswag-mini",
+    pretty_name="HellaSwag",
+    source="EuroEval/hellaswag-mini",
     task=COMMON_SENSE,
-    languages=[EN],
+    languages=[ENGLISH],
 )
 
-EUROPEAN_VALUES_EN_CONFIG = DatasetConfig(
-    name="european-values-en",
-    pretty_name="the English version of the European values evaluation dataset",
-    huggingface_id="EuroEval/european-values-en",
+VALEU_EN_CONFIG = DatasetConfig(
+    name="valeu-en",
+    pretty_name="VaLEU-en",
+    source="EuroEval/european-values-en",
     task=EUROPEAN_VALUES,
-    languages=[EN],
+    languages=[ENGLISH],
     splits=["test"],
     bootstrap_samples=False,
     _instruction_prompt="{text}",
@@ -82,82 +78,55 @@ EUROPEAN_VALUES_EN_CONFIG = DatasetConfig(
 
 XQUAD_EN_CONFIG = DatasetConfig(
     name="xquad-en",
-    pretty_name="the English version of the reading comprehension dataset XQuAD",
-    huggingface_id="EuroEval/xquad-en",
+    pretty_name="XQuAD-en",
+    source="EuroEval/xquad-en",
     task=RC,
-    languages=[EN],
+    languages=[ENGLISH],
     unofficial=True,
 )
 
 ARC_CONFIG = DatasetConfig(
     name="arc",
-    pretty_name="the truncated version of the English knowledge dataset ARC",
-    huggingface_id="EuroEval/arc-mini",
+    pretty_name="ARC",
+    source="EuroEval/arc-mini",
     task=KNOW,
-    languages=[EN],
+    languages=[ENGLISH],
     unofficial=True,
 )
 
 BELEBELE_CONFIG = DatasetConfig(
     name="belebele-en",
-    pretty_name="the English multiple choice reading comprehension dataset BeleBele",
-    huggingface_id="EuroEval/belebele-mini",
+    pretty_name="Belebele-en",
+    source="EuroEval/belebele-mini",
     task=MCRC,
-    languages=[EN],
+    languages=[ENGLISH],
     unofficial=True,
 )
 
 MMLU_CONFIG = DatasetConfig(
     name="mmlu",
-    pretty_name="the truncated version of the English knowledge dataset MMLU",
-    huggingface_id="EuroEval/mmlu-mini",
+    pretty_name="MMLU",
+    source="EuroEval/mmlu-mini",
     task=KNOW,
-    languages=[EN],
+    languages=[ENGLISH],
     unofficial=True,
 )
 
 MULTI_WIKI_QA_EN_CONFIG = DatasetConfig(
     name="multi-wiki-qa-en",
-    pretty_name="the truncated version of the English part of the reading "
-    "comprehension dataset MultiWikiQA",
-    huggingface_id="EuroEval/multi-wiki-qa-en-mini",
+    pretty_name="MultiWikiQA-en",
+    source="EuroEval/multi-wiki-qa-en-mini",
     task=RC,
-    languages=[EN],
+    languages=[ENGLISH],
     unofficial=True,
 )
 
 WINOGRANDE_CONFIG = DatasetConfig(
     name="winogrande",
-    pretty_name="the English common-sense reasoning dataset Winogrande",
-    huggingface_id="EuroEval/winogrande-en",
+    pretty_name="Winogrande-en",
+    source="EuroEval/winogrande-en",
     task=COMMON_SENSE,
-    languages=[EN],
+    languages=[ENGLISH],
     _labels=["a", "b"],
     unofficial=True,
 )
-
-EUROPEAN_VALUES_SITUATIONAL_EN_CONFIG = DatasetConfig(
-    name="european-values-situational-en",
-    pretty_name="the English version of the European values evaluation dataset, where "
-    "the questions are phrased in a situational way",
-    huggingface_id="EuroEval/european-values-situational-en",
-    task=EUROPEAN_VALUES,
-    languages=[EN],
-    splits=["test"],
-    bootstrap_samples=False,
-    _instruction_prompt="{text}",
-    unofficial=True,
-)
-
-EUROPEAN_VALUES_COMPLETIONS_EN_CONFIG = DatasetConfig(
-    name="european-values-completions-en",
-    pretty_name="the English version of the European values evaluation dataset, where "
-    "the questions are phrased as sentence completions",
-    huggingface_id="EuroEval/european-values-completions-en",
-    task=EUROPEAN_VALUES,
-    languages=[EN],
-    splits=["test"],
-    bootstrap_samples=False,
-    _instruction_prompt="{text}",
-    unofficial=True,
-)

euroeval/dataset_configs/estonian.py

@@ -1,66 +1,65 @@
 """All Estonian dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import ET
+from ..languages import ESTONIAN
 from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 ESTONIAN_VALENCE_CONFIG = DatasetConfig(
     name="estonian-valence",
-    pretty_name="the Estonian sentiment classification dataset Estonian Valence",
-    huggingface_id="EuroEval/estonian-valence",
+    pretty_name="Estonian Valence",
+    source="EuroEval/estonian-valence",
     task=SENT,
-    languages=[ET],
+    languages=[ESTONIAN],
 )
 
 GRAMMAR_ET_CONFIG = DatasetConfig(
     name="grammar-et",
-    pretty_name="the Estonian linguistic acceptability dataset Grammar-et",
-    huggingface_id="EuroEval/grammar-et",
+    pretty_name="Grammar-et",
+    source="EuroEval/grammar-et",
     task=LA,
-    languages=[ET],
+    languages=[ESTONIAN],
 )
 
 ESTNER_CONFIG = DatasetConfig(
     name="estner",
-    pretty_name="the Estonian named entity recognition dataset EstNER",
-    huggingface_id="EuroEval/estner-mini",
+    pretty_name="EstNER",
+    source="EuroEval/estner-mini",
     task=NER,
-    languages=[ET],
+    languages=[ESTONIAN],
 )
 
 MULTI_WIKI_QA_ET_CONFIG = DatasetConfig(
     name="multi-wiki-qa-et",
-    pretty_name="the truncated version of the Estonian part of the reading "
-    "comprehension dataset MultiWikiQA",
-    huggingface_id="EuroEval/multi-wiki-qa-et-mini",
+    pretty_name="MultiWikiQA-et",
+    source="EuroEval/multi-wiki-qa-et-mini",
     task=RC,
-    languages=[ET],
+    languages=[ESTONIAN],
 )
 
 ERR_NEWS_CONFIG = DatasetConfig(
     name="err-news",
-    pretty_name="the Estonian summarisation dataset ErrNews",
-    huggingface_id="EuroEval/err-news-mini",
+    pretty_name="ERR News",
+    source="EuroEval/err-news-mini",
     task=SUMM,
-    languages=[ET],
+    languages=[ESTONIAN],
 )
 
 TRIVIA_ET_CONFIG = DatasetConfig(
     name="trivia-et",
-    pretty_name="the Estonian knowledge dataset Trivia-et",
-    huggingface_id="EuroEval/trivia-et",
+    pretty_name="Trivia-et",
+    source="EuroEval/trivia-et",
     task=KNOW,
-    languages=[ET],
+    languages=[ESTONIAN],
 )
 
 WINOGRANDE_ET_CONFIG = DatasetConfig(
     name="winogrande-et",
-    pretty_name="the Estonian common-sense reasoning dataset Winogrande-et",
-    huggingface_id="EuroEval/winogrande-et",
+    pretty_name="Winogrande-et",
+    source="EuroEval/winogrande-et",
     task=COMMON_SENSE,
-    languages=[ET],
+    languages=[ESTONIAN],
     _prompt_prefix="Sulle esitatakse lüngaga (_) tekstülesanded, "
     "igal ülesandel on kaks vastusevarianti (a ja b).",
     _prompt_template="Tekstülesanne: {text}\nVastus: {label}",
@@ -70,12 +69,12 @@ WINOGRANDE_ET_CONFIG = DatasetConfig(
     _labels=["a", "b"],
 )
 
-EUROPEAN_VALUES_ET_CONFIG = DatasetConfig(
-    name="european-values-et",
-    pretty_name="the Estonian version of the European values evaluation dataset",
-    huggingface_id="EuroEval/european-values-et",
+VALEU_ET_CONFIG = DatasetConfig(
+    name="valeu-et",
+    pretty_name="VaLEU-et",
+    source="EuroEval/european-values-et",
     task=EUROPEAN_VALUES,
-    languages=[ET],
+    languages=[ESTONIAN],
     splits=["test"],
     bootstrap_samples=False,
     _instruction_prompt="{text}",
@@ -85,29 +84,28 @@ EUROPEAN_VALUES_ET_CONFIG = DatasetConfig(
 
 SCALA_ET_CONFIG = DatasetConfig(
     name="scala-et",
-    pretty_name="the Estonian part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-et",
+    pretty_name="ScaLA-et",
+    source="EuroEval/scala-et",
     task=LA,
-    languages=[ET],
+    languages=[ESTONIAN],
     unofficial=True,
 )
 
 EXAM_ET_CONFIG = DatasetConfig(
     name="exam-et",
-    pretty_name="the Estonian knowledge dataset Exam-et",
-    huggingface_id="EuroEval/exam-et",
+    pretty_name="Exam-et",
+    source="EuroEval/exam-et",
     task=KNOW,
-    languages=[ET],
+    languages=[ESTONIAN],
     _labels=["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o"],
     unofficial=True,
 )
 
 MMLU_ET_CONFIG = DatasetConfig(
     name="mmlu-et",
-    pretty_name="the truncated version of the Estonian knowledge dataset MMLU-et, "
-    "translated from the English MMLU dataset",
-    huggingface_id="EuroEval/mmlu-et-mini",
+    pretty_name="MMLU-et",
+    source="EuroEval/mmlu-et-mini",
     task=KNOW,
-    languages=[ET],
+    languages=[ESTONIAN],
     unofficial=True,
 )

euroeval/dataset_configs/faroese.py

@@ -1,102 +1,61 @@
 """All Faroese dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import FO
+from ..languages import FAROESE
 from ..tasks import LA, NER, RC, SENT
 
 ### Official datasets ###
 
 FOSENT_CONFIG = DatasetConfig(
     name="fosent",
-    pretty_name="the Faroese sentiment classification dataset FoSent",
-    huggingface_id="EuroEval/fosent",
+    pretty_name="FoSent",
+    source="EuroEval/fosent",
     task=SENT,
-    languages=[FO],
+    languages=[FAROESE],
     _num_few_shot_examples=5,
 )
 
 SCALA_FO_CONFIG = DatasetConfig(
     name="scala-fo",
-    pretty_name="the Faroese part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-fo",
+    pretty_name="ScaLA-fo",
+    source="EuroEval/scala-fo",
     task=LA,
-    languages=[FO],
+    languages=[FAROESE],
 )
 
 FONE_CONFIG = DatasetConfig(
     name="fone",
-    pretty_name="the truncated version of the Faroese named entity recognition "
-    "dataset FoNE",
-    huggingface_id="EuroEval/fone-mini",
+    pretty_name="FoNE",
+    source="EuroEval/fone-mini",
     task=NER,
-    languages=[FO],
+    languages=[FAROESE],
 )
 
 FOQA_CONFIG = DatasetConfig(
     name="foqa",
-    pretty_name="the Faroese reading comprehension dataset FoQA",
-    huggingface_id="EuroEval/foqa",
+    pretty_name="FoQA",
+    source="EuroEval/foqa",
     task=RC,
-    languages=[FO],
+    languages=[FAROESE],
 )
 
-# TODO: No Faroese version of the European values dataset exists yet
-# EUROPEAN_VALUES_FO_CONFIG = DatasetConfig(
-#     name="european-values-fo",
-#     pretty_name="the Faroese version of the European values evaluation dataset",
-#     huggingface_id="EuroEval/european-values-fo",
-#     task=EUROPEAN_VALUES,
-#     languages=[FO],
-#     splits=["test"],
-#     bootstrap_samples=False,
-#     _instruction_prompt="{text}",
-# )
-#
-# EUROPEAN_VALUES_SITUATIONAL_FO_CONFIG = DatasetConfig(
-#     name="european-values-situational-fo",
-#     pretty_name="the Faroese version of the European values evaluation dataset, "
-#     "where the questions are phrased in a situational way",
-#     huggingface_id="EuroEval/european-values-situational-fo",
-#     task=EUROPEAN_VALUES,
-#     languages=[FO],
-#     splits=["test"],
-#     bootstrap_samples=False,
-#     _instruction_prompt="{text}",
-#     unofficial=True,
-# )
-#
-# EUROPEAN_VALUES_COMPLETIONS_FO_CONFIG = DatasetConfig(
-#     name="european-values-completions-fo",
-#     pretty_name="the Faroese version of the European values evaluation dataset, "
-#     "where the questions are phrased as sentence completions",
-#     huggingface_id="EuroEval/european-values-completions-fo",
-#     task=EUROPEAN_VALUES,
-#     languages=[FO],
-#     splits=["test"],
-#     bootstrap_samples=False,
-#     _instruction_prompt="{text}",
-#     unofficial=True,
-# )
-
 
 ### Unofficial datasets ###
 
 WIKIANN_FO_CONFIG = DatasetConfig(
     name="wikiann-fo",
-    pretty_name="the truncated version of the Faroese part of the named entity "
-    "recognition dataset WikiANN",
-    huggingface_id="EuroEval/wikiann-fo-mini",
+    pretty_name="WikiANN-fo",
+    source="EuroEval/wikiann-fo-mini",
     task=NER,
-    languages=[FO],
+    languages=[FAROESE],
     unofficial=True,
 )
 
 MULTI_WIKI_QA_FO_CONFIG = DatasetConfig(
     name="multi-wiki-qa-fo",
-    pretty_name="the truncated version of the Faroese part of the reading "
-    "comprehension dataset MultiWikiQA",
-    huggingface_id="EuroEval/multi-wiki-qa-fo-mini",
+    pretty_name="MultiWikiQA-fo",
+    source="EuroEval/multi-wiki-qa-fo-mini",
     task=RC,
-    languages=[FO],
+    languages=[FAROESE],
     unofficial=True,
 )

euroeval/dataset_configs/finnish.py

@@ -1,68 +1,66 @@
 """All Finnish dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import FI
+from ..languages import FINNISH
 from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 SCANDISENT_FI_CONFIG = DatasetConfig(
     name="scandisent-fi",
-    pretty_name="the truncated version of the Finnish part of the binary sentiment "
-    "classification dataset ScandiSent",
-    huggingface_id="EuroEval/scandisent-fi-mini",
+    pretty_name="ScandiSent-fi",
+    source="EuroEval/scandisent-fi-mini",
     task=SENT,
-    languages=[FI],
+    languages=[FINNISH],
     _labels=["negative", "positive"],
 )
 
 TURKU_NER_FI_CONFIG = DatasetConfig(
     name="turku-ner-fi",
-    pretty_name="the Finnish part of the named entity recognition dataset Turku NER",
-    huggingface_id="EuroEval/turku-ner-fi-mini",
+    pretty_name="Turku NER-fi",
+    source="EuroEval/turku-ner-fi-mini",
     task=NER,
-    languages=[FI],
+    languages=[FINNISH],
 )
 
 TYDIQA_FI_CONFIG = DatasetConfig(
     name="tydiqa-fi",
-    pretty_name="the Finnish part of the TydiQA reading comprehension dataset",
-    huggingface_id="EuroEval/tydiqa-fi-mini",
+    pretty_name="TyDiQA-fi",
+    source="EuroEval/tydiqa-fi-mini",
     task=RC,
-    languages=[FI],
+    languages=[FINNISH],
 )
 
 XLSUM_FI_CONFIG = DatasetConfig(
     name="xlsum-fi",
-    pretty_name="the Finnish summarisation dataset XL-Sum",
-    huggingface_id="EuroEval/xlsum-fi-mini",
+    pretty_name="XLSum-fi",
+    source="EuroEval/xlsum-fi-mini",
     task=SUMM,
-    languages=[FI],
+    languages=[FINNISH],
 )
 
 HELLASWAG_FI_CONFIG = DatasetConfig(
     name="hellaswag-fi",
-    pretty_name="the truncated version of the Finnish common-sense reasoning dataset "
-    "HellaSwag-fi, translated from the English HellaSwag dataset",
-    huggingface_id="EuroEval/hellaswag-fi-mini",
+    pretty_name="HellaSwag-fi",
+    source="EuroEval/hellaswag-fi-mini",
     task=COMMON_SENSE,
-    languages=[FI],
+    languages=[FINNISH],
 )
 
 SCALA_FI_CONFIG = DatasetConfig(
     name="scala-fi",
-    pretty_name="the Finnish part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-fi",
+    pretty_name="ScaLA-fi",
+    source="EuroEval/scala-fi",
     task=LA,
-    languages=[FI],
+    languages=[FINNISH],
 )
 
-EUROPEAN_VALUES_FI_CONFIG = DatasetConfig(
-    name="european-values-fi",
-    pretty_name="the Finnish version of the European values evaluation dataset",
-    huggingface_id="EuroEval/european-values-fi",
+VALEU_FI_CONFIG = DatasetConfig(
+    name="valeu-fi",
+    pretty_name="VaLEU-fi",
+    source="EuroEval/european-values-fi",
     task=EUROPEAN_VALUES,
-    languages=[FI],
+    languages=[FINNISH],
     splits=["test"],
     bootstrap_samples=False,
     _instruction_prompt="{text}",
@@ -73,67 +71,37 @@ EUROPEAN_VALUES_FI_CONFIG = DatasetConfig(
 
 BELEBELE_FI_CONFIG = DatasetConfig(
     name="belebele-fi",
-    pretty_name="the Finnish multiple choice reading comprehension dataset "
-    "BeleBele-fi, translated from the English BeleBele dataset",
-    huggingface_id="EuroEval/belebele-fi-mini",
+    pretty_name="Belebele-fi",
+    source="EuroEval/belebele-fi-mini",
     task=MCRC,
-    languages=[FI],
+    languages=[FINNISH],
     unofficial=True,
 )
 
 MULTI_WIKI_QA_FI_CONFIG = DatasetConfig(
     name="multi-wiki-qa-fi",
-    pretty_name="the truncated version of the Finnish part of the reading "
-    "comprehension dataset MultiWikiQA",
-    huggingface_id="EuroEval/multi-wiki-qa-fi-mini",
+    pretty_name="MultiWikiQA-fi",
+    source="EuroEval/multi-wiki-qa-fi-mini",
     task=RC,
-    languages=[FI],
+    languages=[FINNISH],
     unofficial=True,
 )
 
 GOLDENSWAG_FI_CONFIG = DatasetConfig(
     name="goldenswag-fi",
-    pretty_name="the truncated version of the Finnish common-sense reasoning "
-    "dataset GoldenSwag-fi, translated from the English GoldenSwag dataset",
-    huggingface_id="EuroEval/goldenswag-fi-mini",
+    pretty_name="GoldenSwag-fi",
+    source="EuroEval/goldenswag-fi-mini",
     task=COMMON_SENSE,
-    languages=[FI],
+    languages=[FINNISH],
     unofficial=True,
 )
 
 WINOGRANDE_FI_CONFIG = DatasetConfig(
     name="winogrande-fi",
-    pretty_name="the Finnish common-sense reasoning dataset Winogrande-fi, translated "
-    "from the English Winogrande dataset",
-    huggingface_id="EuroEval/winogrande-fi",
+    pretty_name="Winogrande-fi",
+    source="EuroEval/winogrande-fi",
     task=COMMON_SENSE,
-    languages=[FI],
+    languages=[FINNISH],
     _labels=["a", "b"],
     unofficial=True,
 )
-
-EUROPEAN_VALUES_SITUATIONAL_FI_CONFIG = DatasetConfig(
-    name="european-values-situational-fi",
-    pretty_name="the Finnish version of the European values evaluation dataset, where "
-    "the questions are phrased in a situational way",
-    huggingface_id="EuroEval/european-values-situational-fi",
-    task=EUROPEAN_VALUES,
-    languages=[FI],
-    splits=["test"],
-    bootstrap_samples=False,
-    _instruction_prompt="{text}",
-    unofficial=True,
-)
-
-EUROPEAN_VALUES_COMPLETIONS_FI_CONFIG = DatasetConfig(
-    name="european-values-completions-fi",
-    pretty_name="the Finnish version of the European values evaluation dataset, where "
-    "the questions are phrased as sentence completions",
-    huggingface_id="EuroEval/european-values-completions-fi",
-    task=EUROPEAN_VALUES,
-    languages=[FI],
-    splits=["test"],
-    bootstrap_samples=False,
-    _instruction_prompt="{text}",
-    unofficial=True,
-)