EuroEval 16.4.0__py3-none-any.whl → 16.5.0__py3-none-any.whl

euroeval/dataset_configs/french.py

@@ -1,80 +1,75 @@
 """All French dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import FR
+from ..languages import FRENCH
 from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 ALLOCINE_CONFIG = DatasetConfig(
     name="allocine",
-    pretty_name="the truncated version of the French sentiment classification "
-    "dataset AlloCiné",
-    huggingface_id="EuroEval/allocine-mini",
+    pretty_name="AlloCiné",
+    source="EuroEval/allocine-mini",
     task=SENT,
-    languages=[FR],
+    languages=[FRENCH],
     _labels=["negative", "positive"],
     _prompt_label_mapping=dict(positive="positif", negative="négatif"),
 )
 
 SCALA_FR_CONFIG = DatasetConfig(
     name="scala-fr",
-    pretty_name="the French part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-fr",
+    pretty_name="ScaLA-fr",
+    source="EuroEval/scala-fr",
     task=LA,
-    languages=[FR],
+    languages=[FRENCH],
 )
 
 ELTEC_CONFIG = DatasetConfig(
     name="eltec",
-    pretty_name="the truncated version of the French named entity recognition "
-    "dataset ELTeC",
-    huggingface_id="EuroEval/eltec-mini",
+    pretty_name="ELTeC",
+    source="EuroEval/eltec-mini",
     task=NER,
-    languages=[FR],
+    languages=[FRENCH],
 )
 
 FQUAD_CONFIG = DatasetConfig(
     name="fquad",
-    pretty_name="the truncated version of the French reading comprehension dataset "
-    "FQuAD",
-    huggingface_id="EuroEval/fquad-mini",
+    pretty_name="FQuAD",
+    source="EuroEval/fquad-mini",
     task=RC,
-    languages=[FR],
+    languages=[FRENCH],
 )
 
 ORANGE_SUM_CONFIG = DatasetConfig(
     name="orange-sum",
-    pretty_name="the truncated version of the French summarisation dataset OrangeSum",
-    huggingface_id="EuroEval/orange-sum-mini",
+    pretty_name="OrangeSum",
+    source="EuroEval/orange-sum-mini",
     task=SUMM,
-    languages=[FR],
+    languages=[FRENCH],
 )
 
 MMLU_FR_CONFIG = DatasetConfig(
     name="mmlu-fr",
-    pretty_name="the truncated version of the French knowledge dataset MMLU-fr, "
-    "translated from the English MMLU dataset",
-    huggingface_id="EuroEval/mmlu-fr-mini",
+    pretty_name="MMLU-fr",
+    source="EuroEval/mmlu-fr-mini",
     task=KNOW,
-    languages=[FR],
+    languages=[FRENCH],
 )
 
 HELLASWAG_FR_CONFIG = DatasetConfig(
     name="hellaswag-fr",
-    pretty_name="the truncated version of the French common-sense reasoning dataset "
-    "HellaSwag-fr, translated from the English HellaSwag dataset",
-    huggingface_id="EuroEval/hellaswag-fr-mini",
+    pretty_name="HellaSwag-fr",
+    source="EuroEval/hellaswag-fr-mini",
     task=COMMON_SENSE,
-    languages=[FR],
+    languages=[FRENCH],
 )
 
-EUROPEAN_VALUES_FR_CONFIG = DatasetConfig(
-    name="european-values-fr",
-    pretty_name="the French version of the European values evaluation dataset",
-    huggingface_id="EuroEval/european-values-fr",
+VALEU_FR_CONFIG = DatasetConfig(
+    name="valeu-fr",
+    pretty_name="VaLEU-fr",
+    source="EuroEval/european-values-fr",
     task=EUROPEAN_VALUES,
-    languages=[FR],
+    languages=[FRENCH],
     splits=["test"],
     bootstrap_samples=False,
     _instruction_prompt="{text}",
@@ -85,67 +80,37 @@ EUROPEAN_VALUES_FR_CONFIG = DatasetConfig(
 
 BELEBELE_FR_CONFIG = DatasetConfig(
     name="belebele-fr",
-    pretty_name="the French multiple choice reading comprehension dataset BeleBele-fr, "
-    "translated from the English BeleBele dataset",
-    huggingface_id="EuroEval/belebele-fr-mini",
+    pretty_name="Belebele-fr",
+    source="EuroEval/belebele-fr-mini",
     task=MCRC,
-    languages=[FR],
+    languages=[FRENCH],
     unofficial=True,
 )
 
 MULTI_WIKI_QA_FR_CONFIG = DatasetConfig(
     name="multi-wiki-qa-fr",
-    pretty_name="the truncated version of the French part of the reading "
-    "comprehension dataset MultiWikiQA",
-    huggingface_id="EuroEval/multi-wiki-qa-fr-mini",
+    pretty_name="MultiWikiQA-fr",
+    source="EuroEval/multi-wiki-qa-fr-mini",
     task=RC,
-    languages=[FR],
+    languages=[FRENCH],
     unofficial=True,
 )
 
 GOLDENSWAG_FR_CONFIG = DatasetConfig(
     name="goldenswag-fr",
-    pretty_name="the truncated version of the French common-sense reasoning "
-    "dataset GoldenSwag-fr, translated from the English GoldenSwag dataset",
-    huggingface_id="EuroEval/goldenswag-fr-mini",
+    pretty_name="GoldenSwag-fr",
+    source="EuroEval/goldenswag-fr-mini",
     task=COMMON_SENSE,
-    languages=[FR],
+    languages=[FRENCH],
     unofficial=True,
 )
 
 WINOGRANDE_FR_CONFIG = DatasetConfig(
     name="winogrande-fr",
-    pretty_name="the French common-sense reasoning dataset Winogrande-fr, translated "
-    "from the English Winogrande dataset",
-    huggingface_id="EuroEval/winogrande-fr",
+    pretty_name="Winogrande-fr",
+    source="EuroEval/winogrande-fr",
     task=COMMON_SENSE,
-    languages=[FR],
+    languages=[FRENCH],
     _labels=["a", "b"],
     unofficial=True,
 )
-
-EUROPEAN_VALUES_SITUATIONAL_FR_CONFIG = DatasetConfig(
-    name="european-values-situational-fr",
-    pretty_name="the French version of the European values evaluation dataset, where "
-    "the questions are phrased in a situational way",
-    huggingface_id="EuroEval/european-values-situational-fr",
-    task=EUROPEAN_VALUES,
-    languages=[FR],
-    splits=["test"],
-    bootstrap_samples=False,
-    _instruction_prompt="{text}",
-    unofficial=True,
-)
-
-EUROPEAN_VALUES_COMPLETIONS_FR_CONFIG = DatasetConfig(
-    name="european-values-completions-fr",
-    pretty_name="the French version of the European values evaluation dataset, where "
-    "the questions are phrased as sentence completions",
-    huggingface_id="EuroEval/european-values-completions-fr",
-    task=EUROPEAN_VALUES,
-    languages=[FR],
-    splits=["test"],
-    bootstrap_samples=False,
-    _instruction_prompt="{text}",
-    unofficial=True,
-)

euroeval/dataset_configs/german.py

@@ -1,78 +1,73 @@
 """All German dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import DE
+from ..languages import GERMAN
 from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 SB10K_CONFIG = DatasetConfig(
     name="sb10k",
-    pretty_name="the truncated version of the German sentiment classification "
-    "dataset SB10k",
-    huggingface_id="EuroEval/sb10k-mini",
+    pretty_name="SB10K",
+    source="EuroEval/sb10k-mini",
     task=SENT,
-    languages=[DE],
+    languages=[GERMAN],
 )
 
 SCALA_DE_CONFIG = DatasetConfig(
     name="scala-de",
-    pretty_name="the German part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-de",
+    pretty_name="ScaLA-de",
+    source="EuroEval/scala-de",
     task=LA,
-    languages=[DE],
+    languages=[GERMAN],
 )
 
 GERMEVAL_CONFIG = DatasetConfig(
     name="germeval",
-    pretty_name="the truncated version of the German named entity recognition "
-    "dataset GermEval",
-    huggingface_id="EuroEval/germeval-mini",
+    pretty_name="GermEval",
+    source="EuroEval/germeval-mini",
     task=NER,
-    languages=[DE],
+    languages=[GERMAN],
 )
 
 GERMANQUAD_CONFIG = DatasetConfig(
     name="germanquad",
-    pretty_name="the truncated version of the German reading comprehension dataset "
-    "GermanQuAD",
-    huggingface_id="EuroEval/germanquad-mini",
+    pretty_name="GermanQuAD",
+    source="EuroEval/germanquad-mini",
     task=RC,
-    languages=[DE],
+    languages=[GERMAN],
 )
 
 MLSUM_DE_CONFIG = DatasetConfig(
     name="mlsum-de",
-    pretty_name="the truncated version of the German summarisation dataset MLSum-de",
-    huggingface_id="EuroEval/mlsum-mini",
+    pretty_name="MLSUM-de",
+    source="EuroEval/mlsum-mini",
     task=SUMM,
-    languages=[DE],
+    languages=[GERMAN],
 )
 
 MMLU_DE_CONFIG = DatasetConfig(
     name="mmlu-de",
-    pretty_name="the truncated version of the German knowledge dataset MMLU-de, "
-    "translated from the English MMLU dataset",
-    huggingface_id="EuroEval/mmlu-de-mini",
+    pretty_name="MMLU-de",
+    source="EuroEval/mmlu-de-mini",
     task=KNOW,
-    languages=[DE],
+    languages=[GERMAN],
 )
 
 HELLASWAG_DE_CONFIG = DatasetConfig(
     name="hellaswag-de",
-    pretty_name="the truncated version of the German common-sense reasoning dataset "
-    "HellaSwag-de, translated from the English HellaSwag dataset",
-    huggingface_id="EuroEval/hellaswag-de-mini",
+    pretty_name="HellaSwag-de",
+    source="EuroEval/hellaswag-de-mini",
     task=COMMON_SENSE,
-    languages=[DE],
+    languages=[GERMAN],
 )
 
-EUROPEAN_VALUES_DE_CONFIG = DatasetConfig(
-    name="european-values-de",
-    pretty_name="the German version of the European values evaluation dataset",
-    huggingface_id="EuroEval/european-values-de",
+VALEU_DE_CONFIG = DatasetConfig(
+    name="valeu-de",
+    pretty_name="VaLEU-de",
+    source="EuroEval/european-values-de",
     task=EUROPEAN_VALUES,
-    languages=[DE],
+    languages=[GERMAN],
     splits=["test"],
     bootstrap_samples=False,
     _instruction_prompt="{text}",
@@ -83,86 +78,55 @@ EUROPEAN_VALUES_DE_CONFIG = DatasetConfig(
 
 XQUAD_DE_CONFIG = DatasetConfig(
     name="xquad-de",
-    pretty_name="the German version of the reading comprehension dataset XQuAD",
-    huggingface_id="EuroEval/xquad-de",
+    pretty_name="XQuAD-de",
+    source="EuroEval/xquad-de",
     task=RC,
-    languages=[DE],
+    languages=[GERMAN],
     unofficial=True,
 )
 
 ARC_DE_CONFIG = DatasetConfig(
     name="arc-de",
-    pretty_name="the truncated version of the German knowledge dataset ARC-de, "
-    "translated from the English ARC dataset",
-    huggingface_id="EuroEval/arc-de-mini",
+    pretty_name="ARC-de",
+    source="EuroEval/arc-de-mini",
     task=KNOW,
-    languages=[DE],
+    languages=[GERMAN],
     unofficial=True,
 )
 
 BELEBELE_DE_CONFIG = DatasetConfig(
     name="belebele-de",
-    pretty_name="the German multiple choice reading comprehension dataset BeleBele-de, "
-    "translated from the English BeleBele dataset",
-    huggingface_id="EuroEval/belebele-de-mini",
+    pretty_name="Belebele-de",
+    source="EuroEval/belebele-de-mini",
     task=MCRC,
-    languages=[DE],
+    languages=[GERMAN],
     unofficial=True,
 )
 
 MULTI_WIKI_QA_DE_CONFIG = DatasetConfig(
     name="multi-wiki-qa-de",
-    pretty_name="the truncated version of the German part of the reading "
-    "comprehension dataset MultiWikiQA",
-    huggingface_id="EuroEval/multi-wiki-qa-de-mini",
+    pretty_name="MultiWikiQA-de",
+    source="EuroEval/multi-wiki-qa-de-mini",
     task=RC,
-    languages=[DE],
+    languages=[GERMAN],
     unofficial=True,
 )
 
 GOLDENSWAG_DE_CONFIG = DatasetConfig(
     name="goldenswag-de",
-    pretty_name="the truncated version of the German common-sense reasoning "
-    "dataset GoldenSwag-de, translated from the English GoldenSwag dataset",
-    huggingface_id="EuroEval/goldenswag-de-mini",
+    pretty_name="GoldenSwag-de",
+    source="EuroEval/goldenswag-de-mini",
     task=COMMON_SENSE,
-    languages=[DE],
+    languages=[GERMAN],
     unofficial=True,
 )
 
 WINOGRANDE_DE_CONFIG = DatasetConfig(
     name="winogrande-de",
-    pretty_name="the German common-sense reasoning dataset Winogrande-de, translated "
-    "from the English Winogrande dataset",
-    huggingface_id="EuroEval/winogrande-de",
+    pretty_name="Winogrande-de",
+    source="EuroEval/winogrande-de",
     task=COMMON_SENSE,
-    languages=[DE],
+    languages=[GERMAN],
     _labels=["a", "b"],
     unofficial=True,
 )
-
-EUROPEAN_VALUES_SITUATIONAL_DE_CONFIG = DatasetConfig(
-    name="european-values-situational-de",
-    pretty_name="the German version of the European values evaluation dataset, where "
-    "the questions are phrased in a situational way",
-    huggingface_id="EuroEval/european-values-situational-de",
-    task=EUROPEAN_VALUES,
-    languages=[DE],
-    splits=["test"],
-    bootstrap_samples=False,
-    _instruction_prompt="{text}",
-    unofficial=True,
-)
-
-EUROPEAN_VALUES_COMPLETIONS_DE_CONFIG = DatasetConfig(
-    name="european-values-completions-de",
-    pretty_name="the German version of the European values evaluation dataset, where "
-    "the questions are phrased as sentence completions",
-    huggingface_id="EuroEval/european-values-completions-de",
-    task=EUROPEAN_VALUES,
-    languages=[DE],
-    splits=["test"],
-    bootstrap_samples=False,
-    _instruction_prompt="{text}",
-    unofficial=True,
-)

euroeval/dataset_configs/greek.py

@@ -0,0 +1,64 @@
+"""All Greek dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import GREEK
+from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+GREEK_SA_CONFIG = DatasetConfig(
+    name="greek-sa",
+    pretty_name="Greek Sentiment Analysis",
+    source="EuroEval/greek-sa-mini",
+    task=SENT,
+    languages=[GREEK],
+    _labels=["negative", "positive"],
+)
+
+SCALA_EL_CONFIG = DatasetConfig(
+    name="scala-el",
+    pretty_name="ScaLA-el",
+    source="EuroEval/scala-el",
+    task=LA,
+    languages=[GREEK],
+)
+
+ELNER_CONFIG = DatasetConfig(
+    name="elner",
+    pretty_name="ElNER",
+    source="EuroEval/elner-mini",
+    task=NER,
+    languages=[GREEK],
+)
+
+MULTI_WIKI_QA_EL_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-el",
+    pretty_name="MultiWikiQA-el",
+    source="EuroEval/multi-wiki-qa-el-mini",
+    task=RC,
+    languages=[GREEK],
+)
+
+GREEK_WIKIPEDIA_CONFIG = DatasetConfig(
+    name="greek-wikipedia",
+    pretty_name="Greek Wikipedia",
+    source="EuroEval/greek-wikipedia-mini",
+    task=SUMM,
+    languages=[GREEK],
+)
+
+GLOBAL_MMLU_EL_CONFIG = DatasetConfig(
+    name="global-mmlu-el",
+    pretty_name="GlobalMMLU-el",
+    source="EuroEval/global-mmlu-el-mini",
+    task=KNOW,
+    languages=[GREEK],
+)
+
+WINOGRANDE_EL_CONFIG = DatasetConfig(
+    name="winogrande-el",
+    pretty_name="Winogrande-el",
+    source="EuroEval/winogrande-el",
+    task=COMMON_SENSE,
+    languages=[GREEK],
+)