EuroEval 15.12.0__py3-none-any.whl → 16.7.1__py3-none-any.whl

euroeval/dataset_configs/german.py

@@ -1,91 +1,132 @@
 """All German dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import DE
-from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+from ..languages import GERMAN
+from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 SB10K_CONFIG = DatasetConfig(
     name="sb10k",
-    pretty_name="the truncated version of the German sentiment classification "
-    "dataset SB10k",
-    huggingface_id="EuroEval/sb10k-mini",
+    pretty_name="SB10K",
+    source="EuroEval/sb10k-mini",
     task=SENT,
-    languages=[DE],
+    languages=[GERMAN],
 )
 
 SCALA_DE_CONFIG = DatasetConfig(
     name="scala-de",
-    pretty_name="the German part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-de",
+    pretty_name="ScaLA-de",
+    source="EuroEval/scala-de",
     task=LA,
-    languages=[DE],
+    languages=[GERMAN],
 )
 
 GERMEVAL_CONFIG = DatasetConfig(
     name="germeval",
-    pretty_name="the truncated version of the German named entity recognition "
-    "dataset GermEval",
-    huggingface_id="EuroEval/germeval-mini",
+    pretty_name="GermEval",
+    source="EuroEval/germeval-mini",
     task=NER,
-    languages=[DE],
+    languages=[GERMAN],
 )
 
 GERMANQUAD_CONFIG = DatasetConfig(
     name="germanquad",
-    pretty_name="the truncated version of the German reading comprehension dataset "
-    "GermanQuAD",
-    huggingface_id="EuroEval/germanquad-mini",
+    pretty_name="GermanQuAD",
+    source="EuroEval/germanquad-mini",
     task=RC,
-    languages=[DE],
+    languages=[GERMAN],
 )
 
 MLSUM_DE_CONFIG = DatasetConfig(
     name="mlsum-de",
-    pretty_name="the truncated version of the German summarisation dataset MLSum-de",
-    huggingface_id="EuroEval/mlsum-mini",
+    pretty_name="MLSUM-de",
+    source="EuroEval/mlsum-mini",
     task=SUMM,
-    languages=[DE],
+    languages=[GERMAN],
 )
 
 MMLU_DE_CONFIG = DatasetConfig(
     name="mmlu-de",
-    pretty_name="the truncated version of the German knowledge dataset MMLU-de, "
-    "translated from the English MMLU dataset",
-    huggingface_id="EuroEval/mmlu-de-mini",
+    pretty_name="MMLU-de",
+    source="EuroEval/mmlu-de-mini",
     task=KNOW,
-    languages=[DE],
+    languages=[GERMAN],
 )
 
 HELLASWAG_DE_CONFIG = DatasetConfig(
     name="hellaswag-de",
-    pretty_name="the truncated version of the German common-sense reasoning dataset "
-    "HellaSwag-de, translated from the English HellaSwag dataset",
-    huggingface_id="EuroEval/hellaswag-de-mini",
+    pretty_name="HellaSwag-de",
+    source="EuroEval/hellaswag-de-mini",
     task=COMMON_SENSE,
-    languages=[DE],
+    languages=[GERMAN],
+)
+
+VALEU_DE_CONFIG = DatasetConfig(
+    name="valeu-de",
+    pretty_name="VaLEU-de",
+    source="EuroEval/european-values-de",
+    task=EUROPEAN_VALUES,
+    languages=[GERMAN],
+    splits=["test"],
+    bootstrap_samples=False,
+    _instruction_prompt="{text}",
 )
 
 
 ### Unofficial datasets ###
 
+XQUAD_DE_CONFIG = DatasetConfig(
+    name="xquad-de",
+    pretty_name="XQuAD-de",
+    source="EuroEval/xquad-de",
+    task=RC,
+    languages=[GERMAN],
+    unofficial=True,
+)
+
 ARC_DE_CONFIG = DatasetConfig(
     name="arc-de",
-    pretty_name="the truncated version of the German knowledge dataset ARC-de, "
-    "translated from the English ARC dataset",
-    huggingface_id="EuroEval/arc-de-mini",
+    pretty_name="ARC-de",
+    source="EuroEval/arc-de-mini",
     task=KNOW,
-    languages=[DE],
+    languages=[GERMAN],
     unofficial=True,
 )
 
 BELEBELE_DE_CONFIG = DatasetConfig(
     name="belebele-de",
-    pretty_name="the German multiple choice reading comprehension dataset BeleBele-de, "
-    "translated from the English BeleBele dataset",
-    huggingface_id="EuroEval/belebele-de-mini",
+    pretty_name="Belebele-de",
+    source="EuroEval/belebele-de-mini",
     task=MCRC,
-    languages=[DE],
+    languages=[GERMAN],
+    unofficial=True,
+)
+
+MULTI_WIKI_QA_DE_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-de",
+    pretty_name="MultiWikiQA-de",
+    source="EuroEval/multi-wiki-qa-de-mini",
+    task=RC,
+    languages=[GERMAN],
+    unofficial=True,
+)
+
+GOLDENSWAG_DE_CONFIG = DatasetConfig(
+    name="goldenswag-de",
+    pretty_name="GoldenSwag-de",
+    source="EuroEval/goldenswag-de-mini",
+    task=COMMON_SENSE,
+    languages=[GERMAN],
+    unofficial=True,
+)
+
+WINOGRANDE_DE_CONFIG = DatasetConfig(
+    name="winogrande-de",
+    pretty_name="Winogrande-de",
+    source="EuroEval/winogrande-de",
+    task=COMMON_SENSE,
+    languages=[GERMAN],
+    _labels=["a", "b"],
     unofficial=True,
 )

euroeval/dataset_configs/greek.py

@@ -0,0 +1,64 @@
+"""All Greek dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import GREEK
+from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+GREEK_SA_CONFIG = DatasetConfig(
+    name="greek-sa",
+    pretty_name="Greek Sentiment Analysis",
+    source="EuroEval/greek-sa-mini",
+    task=SENT,
+    languages=[GREEK],
+    _labels=["negative", "positive"],
+)
+
+SCALA_EL_CONFIG = DatasetConfig(
+    name="scala-el",
+    pretty_name="ScaLA-el",
+    source="EuroEval/scala-el",
+    task=LA,
+    languages=[GREEK],
+)
+
+ELNER_CONFIG = DatasetConfig(
+    name="elner",
+    pretty_name="ElNER",
+    source="EuroEval/elner-mini",
+    task=NER,
+    languages=[GREEK],
+)
+
+MULTI_WIKI_QA_EL_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-el",
+    pretty_name="MultiWikiQA-el",
+    source="EuroEval/multi-wiki-qa-el-mini",
+    task=RC,
+    languages=[GREEK],
+)
+
+GREEK_WIKIPEDIA_CONFIG = DatasetConfig(
+    name="greek-wikipedia",
+    pretty_name="Greek Wikipedia",
+    source="EuroEval/greek-wikipedia-mini",
+    task=SUMM,
+    languages=[GREEK],
+)
+
+GLOBAL_MMLU_EL_CONFIG = DatasetConfig(
+    name="global-mmlu-el",
+    pretty_name="GlobalMMLU-el",
+    source="EuroEval/global-mmlu-el-mini",
+    task=KNOW,
+    languages=[GREEK],
+)
+
+WINOGRANDE_EL_CONFIG = DatasetConfig(
+    name="winogrande-el",
+    pretty_name="Winogrande-el",
+    source="EuroEval/winogrande-el",
+    task=COMMON_SENSE,
+    languages=[GREEK],
+)

euroeval/dataset_configs/icelandic.py

@@ -1,71 +1,77 @@
 """All Icelandic dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import IS
-from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+from ..languages import ICELANDIC
+from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 HOTTER_AND_COLDER_SENTIMENT_CONFIG = DatasetConfig(
     name="hotter-and-colder-sentiment",
-    pretty_name="the sentiment classification part of the Icelandic dataset Hotter "
-    "and Colder",
-    huggingface_id="EuroEval/hotter-and-colder-sentiment",
+    pretty_name="Hotter and Colder Sentiment",
+    source="EuroEval/hotter-and-colder-sentiment",
     task=SENT,
-    languages=[IS],
+    languages=[ICELANDIC],
 )
 
 SCALA_IS_CONFIG = DatasetConfig(
     name="scala-is",
-    pretty_name="the Icelandic part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-is",
+    pretty_name="ScaLA-is",
+    source="EuroEval/scala-is",
     task=LA,
-    languages=[IS],
+    languages=[ICELANDIC],
 )
 
 MIM_GOLD_NER_CONFIG = DatasetConfig(
     name="mim-gold-ner",
-    pretty_name="the truncated version of the Icelandic named entity recognition "
-    "dataset MIM-GOLD-NER",
-    huggingface_id="EuroEval/mim-gold-ner-mini",
+    pretty_name="MIM-GOLD-NER",
+    source="EuroEval/mim-gold-ner-mini",
     task=NER,
-    languages=[IS],
+    languages=[ICELANDIC],
 )
 
 NQII_CONFIG = DatasetConfig(
     name="nqii",
-    pretty_name="the truncated version of the Icelandic reading comprehension dataset "
-    "Natural Questions in Icelandic",
-    huggingface_id="EuroEval/nqii-mini",
+    pretty_name="NQiI",
+    source="EuroEval/nqii-mini",
     task=RC,
-    languages=[IS],
+    languages=[ICELANDIC],
 )
 
 RRN_CONFIG = DatasetConfig(
     name="rrn",
-    pretty_name="the truncated version of the Icelandic summarisation dataset "
-    "RÚV Radio News",
-    huggingface_id="EuroEval/rrn-mini",
+    pretty_name="RRN",
+    source="EuroEval/rrn-mini",
     task=SUMM,
-    languages=[IS],
+    languages=[ICELANDIC],
 )
 
 ICELANDIC_KNOWLEDGE_CONFIG = DatasetConfig(
     name="icelandic-knowledge",
-    pretty_name="the Icelandic knowledge dataset IcelandicKnowledge, derived from the "
-    "IcelandicQA dataset",
-    huggingface_id="EuroEval/icelandic-knowledge",
+    pretty_name="Icelandic Knowledge",
+    source="EuroEval/icelandic-knowledge",
     task=KNOW,
-    languages=[IS],
+    languages=[ICELANDIC],
 )
 
 WINOGRANDE_IS_CONFIG = DatasetConfig(
     name="winogrande-is",
-    pretty_name="the Icelandic common-sense reasoning dataset "
-    "Winogrande-is, manually translated from the English Winogrande dataset",
-    huggingface_id="EuroEval/winogrande-is",
+    pretty_name="Winogrande-is",
+    source="EuroEval/winogrande-is",
     task=COMMON_SENSE,
-    languages=[IS],
+    languages=[ICELANDIC],
+    _labels=["a", "b"],
+)
+
+VALEU_IS_CONFIG = DatasetConfig(
+    name="valeu-is",
+    pretty_name="VaLEU-is",
+    source="EuroEval/european-values-is",
+    task=EUROPEAN_VALUES,
+    languages=[ICELANDIC],
+    splits=["test"],
+    bootstrap_samples=False,
+    _instruction_prompt="{text}",
 )
 
 
@@ -73,76 +79,81 @@ WINOGRANDE_IS_CONFIG = DatasetConfig(
 
 ICE_EC_CONFIG = DatasetConfig(
     name="ice-ec",
-    pretty_name="the truncated version of the Icelandic Error Corpus",
-    huggingface_id="EuroEval/ice-ec",
+    pretty_name="ICE-EC",
+    source="EuroEval/ice-ec",
     task=LA,
-    languages=[IS],
+    languages=[ICELANDIC],
     unofficial=True,
 )
 
 ICE_EC_FULL_CONFIG = DatasetConfig(
     name="ice-ec-full",
-    pretty_name="the Icelandic Error Corpus",
-    huggingface_id="EuroEval/ice-ec-full",
+    pretty_name="ICE-EC Full",
+    source="EuroEval/ice-ec-full",
     task=LA,
-    languages=[IS],
+    languages=[ICELANDIC],
     unofficial=True,
 )
 
 ICE_LINGUISTIC_CONFIG = DatasetConfig(
     name="ice-linguistic",
-    pretty_name="the Icelandic linguistic acceptability dataset IceLinguistic",
-    huggingface_id="EuroEval/ice-linguistic",
+    pretty_name="IceLinguistic",
+    source="EuroEval/ice-linguistic",
     task=LA,
-    languages=[IS],
+    languages=[ICELANDIC],
     unofficial=True,
 )
 
 ICELANDIC_QA_CONFIG = DatasetConfig(
     name="icelandic-qa",
-    pretty_name="the Icelandic reading comprehension dataset IcelandicQA",
-    huggingface_id="EuroEval/icelandic-qa",
+    pretty_name="Icelandic QA",
+    source="EuroEval/icelandic-qa",
     task=RC,
-    languages=[IS],
+    languages=[ICELANDIC],
     unofficial=True,
 )
 
 MMLU_IS_CONFIG = DatasetConfig(
     name="mmlu-is",
-    pretty_name="the truncated version of the Icelandic knowledge dataset MMLU-is, "
-    "translated from the English MMLU dataset",
-    huggingface_id="EuroEval/mmlu-is-mini",
+    pretty_name="MMLU-is",
+    source="EuroEval/mmlu-is-mini",
     task=KNOW,
-    languages=[IS],
+    languages=[ICELANDIC],
     unofficial=True,
 )
 
 ARC_IS_CONFIG = DatasetConfig(
     name="arc-is",
-    pretty_name="the truncated version of the Icelandic knowledge dataset ARC-is, "
-    "translated from the English ARC dataset",
-    huggingface_id="EuroEval/arc-is-mini",
+    pretty_name="ARC-is",
+    source="EuroEval/arc-is-mini",
     task=KNOW,
-    languages=[IS],
+    languages=[ICELANDIC],
     unofficial=True,
 )
 
 HELLASWAG_IS_CONFIG = DatasetConfig(
     name="hellaswag-is",
-    pretty_name="the truncated version of the Icelandic common-sense reasoning dataset "
-    "HellaSwag-is, translated from the English HellaSwag dataset",
-    huggingface_id="EuroEval/hellaswag-is-mini",
+    pretty_name="HellaSwag-is",
+    source="EuroEval/hellaswag-is-mini",
     task=COMMON_SENSE,
-    languages=[IS],
+    languages=[ICELANDIC],
     unofficial=True,
 )
 
 BELEBELE_IS_CONFIG = DatasetConfig(
     name="belebele-is",
-    pretty_name="the Icelandic multiple choice reading comprehension dataset "
-    "BeleBele-is, translated from the English BeleBele dataset",
-    huggingface_id="EuroEval/belebele-is-mini",
+    pretty_name="Belebele-is",
+    source="EuroEval/belebele-is-mini",
     task=MCRC,
-    languages=[IS],
+    languages=[ICELANDIC],
+    unofficial=True,
+)
+
+MULTI_WIKI_QA_IS_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-is",
+    pretty_name="MultiWikiQA-is",
+    source="EuroEval/multi-wiki-qa-is-mini",
+    task=RC,
+    languages=[ICELANDIC],
     unofficial=True,
 )

euroeval/dataset_configs/italian.py

@@ -1,70 +1,76 @@
 """All Italian dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import IT
-from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+from ..languages import ITALIAN
+from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 SENTIPOLC_CONFIG = DatasetConfig(
     name="sentipolc16",
-    pretty_name="the truncated version of the Italian sentiment classification "
-    "dataset Sentipolc-16",
-    huggingface_id="EuroEval/sentipolc16-mini",
+    pretty_name="Sentipolc16",
+    source="EuroEval/sentipolc16-mini",
     task=SENT,
-    languages=[IT],
+    languages=[ITALIAN],
 )
 
 SCALA_IT_CONFIG = DatasetConfig(
     name="scala-it",
-    pretty_name="the Italian part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-it",
+    pretty_name="ScaLA-it",
+    source="EuroEval/scala-it",
     task=LA,
-    languages=[IT],
+    languages=[ITALIAN],
 )
 
 MULTINERD_IT_CONFIG = DatasetConfig(
     name="multinerd-it",
-    pretty_name="the truncated version of the Italian part of the named "
-    "entity recognition dataset MultiNERD",
-    huggingface_id="EuroEval/multinerd-mini-it",
+    pretty_name="MultiNERD-it",
+    source="EuroEval/multinerd-mini-it",
     task=NER,
-    languages=[IT],
+    languages=[ITALIAN],
 )
 
 SQUAD_IT_CONFIG = DatasetConfig(
     name="squad-it",
-    pretty_name="the truncated version of the Italian reading comprehension dataset "
-    "SQuAD-it, translated from the English SQuAD dataset",
-    huggingface_id="EuroEval/squad-it-mini",
+    pretty_name="SQuAD-it",
+    source="EuroEval/squad-it-mini",
     task=RC,
-    languages=[IT],
+    languages=[ITALIAN],
 )
 
 ILPOST_SUM_CONFIG = DatasetConfig(
     name="ilpost-sum",
-    pretty_name="the truncated version of the Italian summarisation dataset IlPost-Sum",
-    huggingface_id="EuroEval/ilpost-sum",
+    pretty_name="IlPost-Sum",
+    source="EuroEval/ilpost-sum",
     task=SUMM,
-    languages=[IT],
+    languages=[ITALIAN],
 )
 
 MMLU_IT_CONFIG = DatasetConfig(
     name="mmlu-it",
-    pretty_name="the truncated version of the Italian knowledge dataset MMLU-it, "
-    "translated from the English MMLU dataset",
-    huggingface_id="EuroEval/mmlu-it-mini",
+    pretty_name="MMLU-it",
+    source="EuroEval/mmlu-it-mini",
     task=KNOW,
-    languages=[IT],
+    languages=[ITALIAN],
 )
 
 HELLASWAG_IT_CONFIG = DatasetConfig(
     name="hellaswag-it",
-    pretty_name="the truncated version of the Italian common-sense reasoning dataset "
-    "HellaSwag-it, translated from the English HellaSwag dataset",
-    huggingface_id="EuroEval/hellaswag-it-mini",
+    pretty_name="HellaSwag-it",
+    source="EuroEval/hellaswag-it-mini",
     task=COMMON_SENSE,
-    languages=[IT],
+    languages=[ITALIAN],
+)
+
+VALEU_IT_CONFIG = DatasetConfig(
+    name="valeu-it",
+    pretty_name="VaLEU-it",
+    source="EuroEval/european-values-it",
+    task=EUROPEAN_VALUES,
+    languages=[ITALIAN],
+    splits=["test"],
+    bootstrap_samples=False,
+    _instruction_prompt="{text}",
 )
 
 
@@ -72,20 +78,46 @@ HELLASWAG_IT_CONFIG = DatasetConfig(
 
 WIKINEURAL_IT_CONFIG = DatasetConfig(
     name="wikineural-it",
-    pretty_name="the truncated version of the Italian named "
-    "entity recognition dataset WikiNEuRal IT",
-    huggingface_id="EuroEval/wikineural-mini-it",
+    pretty_name="WikiNeural-it",
+    source="EuroEval/wikineural-mini-it",
     task=NER,
-    languages=[IT],
+    languages=[ITALIAN],
     unofficial=True,
 )
 
 BELEBELE_IT_CONFIG = DatasetConfig(
     name="belebele-it",
-    pretty_name="the Italian multiple choice reading comprehension dataset "
-    "BeleBele-it, translated from the English BeleBele dataset",
-    huggingface_id="EuroEval/belebele-it-mini",
+    pretty_name="Belebele-it",
+    source="EuroEval/belebele-it-mini",
     task=MCRC,
-    languages=[IT],
+    languages=[ITALIAN],
+    unofficial=True,
+)
+
+MULTI_WIKI_QA_IT_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-it",
+    pretty_name="MultiWikiQA-it",
+    source="EuroEval/multi-wiki-qa-it-mini",
+    task=RC,
+    languages=[ITALIAN],
+    unofficial=True,
+)
+
+GOLDENSWAG_IT_CONFIG = DatasetConfig(
+    name="goldenswag-it",
+    pretty_name="GoldenSwag-it",
+    source="EuroEval/goldenswag-it-mini",
+    task=COMMON_SENSE,
+    languages=[ITALIAN],
+    unofficial=True,
+)
+
+WINOGRANDE_IT_CONFIG = DatasetConfig(
+    name="winogrande-it",
+    pretty_name="Winogrande-it",
+    source="EuroEval/winogrande-it",
+    task=COMMON_SENSE,
+    languages=[ITALIAN],
+    _labels=["a", "b"],
     unofficial=True,
 )