EuroEval 15.5.0__py3-none-any.whl → 15.6.0__py3-none-any.whl

euroeval/dataset_configs/dutch.py

@@ -0,0 +1,123 @@
+"""All Dutch dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import NL
+from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+DUTCH_SOCIAL_CONFIG = DatasetConfig(
+    name="dutch-social",
+    pretty_name="the truncated version of the Dutch sentiment classification "
+    "dataset Dutch Social",
+    huggingface_id="EuroEval/dutch-social-mini",
+    task=SENT,
+    languages=[NL],
+)
+
+SCALA_NL_CONFIG = DatasetConfig(
+    name="scala-nl",
+    pretty_name="the Dutch part of the linguistic acceptability dataset ScaLA",
+    huggingface_id="EuroEval/scala-nl",
+    task=LA,
+    languages=[NL],
+)
+
+CONLL_NL_CONFIG = DatasetConfig(
+    name="conll-nl",
+    pretty_name="the Dutch part of the truncated version of the named entity "
+    "recognition dataset CoNLL 2002",
+    huggingface_id="EuroEval/conll-nl-mini",
+    task=NER,
+    languages=[NL],
+)
+
+SQUAD_NL_CONFIG = DatasetConfig(
+    name="squad-nl",
+    pretty_name="the truncated version of the Dutch reading comprehension dataset "
+    "SQuAD-nl, translated from the English SQuAD dataset",
+    huggingface_id="EuroEval/squad-nl-v2-mini",
+    task=RC,
+    languages=[NL],
+)
+
+WIKI_LINGUA_NL_CONFIG = DatasetConfig(
+    name="wiki-lingua-nl",
+    pretty_name="the Dutch part of the truncated version of the summarisation dataset "
+    "WikiLingua",
+    huggingface_id="EuroEval/wiki-lingua-nl-mini",
+    task=SUMM,
+    languages=[NL],
+)
+
+MMLU_NL_CONFIG = DatasetConfig(
+    name="mmlu-nl",
+    pretty_name="the truncated version of the Dutch knowledge dataset MMLU-nl, "
+    "translated from the English MMLU dataset",
+    huggingface_id="EuroEval/mmlu-nl-mini",
+    task=KNOW,
+    languages=[NL],
+)
+
+HELLASWAG_NL_CONFIG = DatasetConfig(
+    name="hellaswag-nl",
+    pretty_name="the truncated version of the Dutch common-sense reasoning dataset "
+    "HellaSwag-nl, translated from the English HellaSwag dataset",
+    huggingface_id="EuroEval/hellaswag-nl-mini",
+    task=COMMON_SENSE,
+    languages=[NL],
+)
+
+
+### Unofficial datasets ###
+
+DBRD_CONFIG = DatasetConfig(
+    name="dbrd",
+    pretty_name="the truncated version of the Dutch sentiment classification "
+    "dataset DBRD",
+    huggingface_id="EuroEval/dbrd-mini",
+    task=SENT,
+    languages=[NL],
+    _labels=["negative", "positive"],
+    _prompt_label_mapping=dict(positive="positief", negative="negatief"),
+    unofficial=True,
+)
+
+DUTCH_COLA_CONFIG = DatasetConfig(
+    name="dutch-cola",
+    pretty_name="the truncated version of the Dutch linguistic acceptability dataset "
+    "Dutch CoLA",
+    huggingface_id="EuroEval/dutch-cola",
+    task=LA,
+    languages=[NL],
+    unofficial=True,
+)
+
+DUTCH_COLA_FULL_CONFIG = DatasetConfig(
+    name="dutch-cola-full",
+    pretty_name="the Dutch linguistic acceptability dataset Dutch CoLA",
+    huggingface_id="EuroEval/dutch-cola-full",
+    task=LA,
+    languages=[NL],
+    unofficial=True,
+)
+
+ARC_NL_CONFIG = DatasetConfig(
+    name="arc-nl",
+    pretty_name="the truncated version of the Dutch knowledge dataset ARC-nl, "
+    "translated from the English ARC dataset",
+    huggingface_id="EuroEval/arc-nl-mini",
+    task=KNOW,
+    languages=[NL],
+    unofficial=True,
+)
+
+BELEBELE_NL_CONFIG = DatasetConfig(
+    name="belebele-nl",
+    pretty_name="the Dutch multiple choice reading comprehension dataset BeleBele-nl, "
+    "translated from the English BeleBele dataset",
+    huggingface_id="EuroEval/belebele-nl-mini",
+    task=MCRC,
+    languages=[NL],
+    unofficial=True,
+)

euroeval/dataset_configs/english.py

@@ -0,0 +1,88 @@
+"""All English dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import EN
+from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+SST5_CONFIG = DatasetConfig(
+    name="sst5",
+    pretty_name="the truncated version of the English sentiment classification "
+    "dataset SST5",
+    huggingface_id="EuroEval/sst5-mini",
+    task=SENT,
+    languages=[EN],
+)
+
+SCALA_EN_CONFIG = DatasetConfig(
+    name="scala-en",
+    pretty_name="the English part of the linguistic acceptability dataset ScaLA",
+    huggingface_id="EuroEval/scala-en",
+    task=LA,
+    languages=[EN],
+)
+
+CONLL_EN_CONFIG = DatasetConfig(
+    name="conll-en",
+    pretty_name="the truncated version of the English named entity recognition "
+    "dataset CoNLL 2003",
+    huggingface_id="EuroEval/conll-en-mini",
+    task=NER,
+    languages=[EN],
+)
+
+SQUAD_CONFIG = DatasetConfig(
+    name="squad",
+    pretty_name="the truncated version of the English question answering dataset SQuAD",
+    huggingface_id="EuroEval/squad-mini",
+    task=RC,
+    languages=[EN],
+)
+
+CNN_DAILYMAIL_CONFIG = DatasetConfig(
+    name="cnn-dailymail",
+    pretty_name="the truncated version of the English summarisation dataset "
+    "CNN-DailyMail",
+    huggingface_id="EuroEval/cnn-dailymail-mini",
+    task=SUMM,
+    languages=[EN],
+)
+
+MMLU_CONFIG = DatasetConfig(
+    name="mmlu",
+    pretty_name="the truncated version of the English knowledge dataset MMLU",
+    huggingface_id="EuroEval/mmlu-mini",
+    task=KNOW,
+    languages=[EN],
+)
+
+HELLASWAG_CONFIG = DatasetConfig(
+    name="hellaswag",
+    pretty_name="the truncated version of the English common-sense reasoning "
+    "dataset HellaSwag",
+    huggingface_id="EuroEval/hellaswag-mini",
+    task=COMMON_SENSE,
+    languages=[EN],
+)
+
+
+### Unofficial datasets ###
+
+ARC_CONFIG = DatasetConfig(
+    name="arc",
+    pretty_name="the truncated version of the English knowledge dataset ARC",
+    huggingface_id="EuroEval/arc-mini",
+    task=KNOW,
+    languages=[EN],
+    unofficial=True,
+)
+
+BELEBELE_CONFIG = DatasetConfig(
+    name="belebele",
+    pretty_name="the English multiple choice reading comprehension dataset BeleBele",
+    huggingface_id="EuroEval/belebele-mini",
+    task=MCRC,
+    languages=[EN],
+    unofficial=True,
+)

euroeval/dataset_configs/faroese.py

@@ -0,0 +1,53 @@
+"""All Faroese dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import FO
+from ..tasks import LA, NER, RC, SENT
+
+### Official datasets ###
+
+FOSENT_CONFIG = DatasetConfig(
+    name="fosent",
+    pretty_name="the Faroese sentiment classification dataset FoSent",
+    huggingface_id="EuroEval/fosent",
+    task=SENT,
+    languages=[FO],
+)
+
+SCALA_FO_CONFIG = DatasetConfig(
+    name="scala-fo",
+    pretty_name="the Faroese part of the linguistic acceptability dataset ScaLA",
+    huggingface_id="EuroEval/scala-fo",
+    task=LA,
+    languages=[FO],
+)
+
+FONE_CONFIG = DatasetConfig(
+    name="fone",
+    pretty_name="the truncated version of the Faroese named entity recognition "
+    "dataset FoNE",
+    huggingface_id="EuroEval/fone-mini",
+    task=NER,
+    languages=[FO],
+)
+
+FOQA_CONFIG = DatasetConfig(
+    name="foqa",
+    pretty_name="the Faroese reading comprehension dataset FoQA",
+    huggingface_id="EuroEval/foqa",
+    task=RC,
+    languages=[FO],
+)
+
+
+### Unofficial datasets ###
+
+WIKIANN_FO_CONFIG = DatasetConfig(
+    name="wikiann-fo",
+    pretty_name="the truncated version of the Faroese part of the named entity "
+    "recognition dataset WikiANN",
+    huggingface_id="EuroEval/wikiann-fo-mini",
+    task=NER,
+    languages=[FO],
+    unofficial=True,
+)

euroeval/dataset_configs/french.py

@@ -0,0 +1,83 @@
+"""All French dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import FR
+from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+ALLOCINE_CONFIG = DatasetConfig(
+    name="allocine",
+    pretty_name="the truncated version of the French sentiment classification "
+    "dataset AlloCiné",
+    huggingface_id="EuroEval/allocine-mini",
+    task=SENT,
+    languages=[FR],
+    _labels=["negative", "positive"],
+    _prompt_label_mapping=dict(positive="positif", negative="négatif"),
+)
+
+SCALA_FR_CONFIG = DatasetConfig(
+    name="scala-fr",
+    pretty_name="the French part of the linguistic acceptability dataset ScaLA",
+    huggingface_id="EuroEval/scala-fr",
+    task=LA,
+    languages=[FR],
+)
+
+ELTEC_CONFIG = DatasetConfig(
+    name="eltec",
+    pretty_name="the truncated version of the French named entity recognition "
+    "dataset ELTeC",
+    huggingface_id="EuroEval/eltec-mini",
+    task=NER,
+    languages=[FR],
+)
+
+FQUAD_CONFIG = DatasetConfig(
+    name="fquad",
+    pretty_name="the truncated version of the French reading comprehension dataset "
+    "FQuAD",
+    huggingface_id="EuroEval/fquad-mini",
+    task=RC,
+    languages=[FR],
+)
+
+ORANGE_SUM_CONFIG = DatasetConfig(
+    name="orange-sum",
+    pretty_name="the truncated version of the French summarisation dataset OrangeSum",
+    huggingface_id="EuroEval/orange-sum-mini",
+    task=SUMM,
+    languages=[FR],
+)
+
+MMLU_FR_CONFIG = DatasetConfig(
+    name="mmlu-fr",
+    pretty_name="the truncated version of the French knowledge dataset MMLU-fr, "
+    "translated from the English MMLU dataset",
+    huggingface_id="EuroEval/mmlu-fr-mini",
+    task=KNOW,
+    languages=[FR],
+)
+
+HELLASWAG_FR_CONFIG = DatasetConfig(
+    name="hellaswag-fr",
+    pretty_name="the truncated version of the French common-sense reasoning dataset "
+    "HellaSwag-fr, translated from the English HellaSwag dataset",
+    huggingface_id="EuroEval/hellaswag-fr-mini",
+    task=COMMON_SENSE,
+    languages=[FR],
+)
+
+
+### Unofficial datasets ###
+
+BELEBELE_FR_CONFIG = DatasetConfig(
+    name="belebele-fr",
+    pretty_name="the French multiple choice reading comprehension dataset BeleBele-fr, "
+    "translated from the English BeleBele dataset",
+    huggingface_id="EuroEval/belebele-fr-mini",
+    task=MCRC,
+    languages=[FR],
+    unofficial=True,
+)

euroeval/dataset_configs/german.py

@@ -0,0 +1,91 @@
+"""All German dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import DE
+from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+SB10K_CONFIG = DatasetConfig(
+    name="sb10k",
+    pretty_name="the truncated version of the German sentiment classification "
+    "dataset SB10k",
+    huggingface_id="EuroEval/sb10k-mini",
+    task=SENT,
+    languages=[DE],
+)
+
+SCALA_DE_CONFIG = DatasetConfig(
+    name="scala-de",
+    pretty_name="the German part of the linguistic acceptability dataset ScaLA",
+    huggingface_id="EuroEval/scala-de",
+    task=LA,
+    languages=[DE],
+)
+
+GERMEVAL_CONFIG = DatasetConfig(
+    name="germeval",
+    pretty_name="the truncated version of the German named entity recognition "
+    "dataset GermEval",
+    huggingface_id="EuroEval/germeval-mini",
+    task=NER,
+    languages=[DE],
+)
+
+GERMANQUAD_CONFIG = DatasetConfig(
+    name="germanquad",
+    pretty_name="the truncated version of the German reading comprehension dataset "
+    "GermanQuAD",
+    huggingface_id="EuroEval/germanquad-mini",
+    task=RC,
+    languages=[DE],
+)
+
+MLSUM_DE_CONFIG = DatasetConfig(
+    name="mlsum-de",
+    pretty_name="the truncated version of the German summarisation dataset MLSum-de",
+    huggingface_id="EuroEval/mlsum-mini",
+    task=SUMM,
+    languages=[DE],
+)
+
+MMLU_DE_CONFIG = DatasetConfig(
+    name="mmlu-de",
+    pretty_name="the truncated version of the German knowledge dataset MMLU-de, "
+    "translated from the English MMLU dataset",
+    huggingface_id="EuroEval/mmlu-de-mini",
+    task=KNOW,
+    languages=[DE],
+)
+
+HELLASWAG_DE_CONFIG = DatasetConfig(
+    name="hellaswag-de",
+    pretty_name="the truncated version of the German common-sense reasoning dataset "
+    "HellaSwag-de, translated from the English HellaSwag dataset",
+    huggingface_id="EuroEval/hellaswag-de-mini",
+    task=COMMON_SENSE,
+    languages=[DE],
+)
+
+
+### Unofficial datasets ###
+
+ARC_DE_CONFIG = DatasetConfig(
+    name="arc-de",
+    pretty_name="the truncated version of the German knowledge dataset ARC-de, "
+    "translated from the English ARC dataset",
+    huggingface_id="EuroEval/arc-de-mini",
+    task=KNOW,
+    languages=[DE],
+    unofficial=True,
+)
+
+BELEBELE_DE_CONFIG = DatasetConfig(
+    name="belebele-de",
+    pretty_name="the German multiple choice reading comprehension dataset BeleBele-de, "
+    "translated from the English BeleBele dataset",
+    huggingface_id="EuroEval/belebele-de-mini",
+    task=MCRC,
+    languages=[DE],
+    unofficial=True,
+)

euroeval/dataset_configs/icelandic.py

@@ -0,0 +1,148 @@
+"""All Icelandic dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import IS
+from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+HOTTER_AND_COLDER_SENTIMENT_CONFIG = DatasetConfig(
+    name="hotter-and-colder-sentiment",
+    pretty_name="the sentiment classification part of the Icelandic dataset Hotter "
+    "and Colder",
+    huggingface_id="EuroEval/hotter-and-colder-sentiment",
+    task=SENT,
+    languages=[IS],
+)
+
+SCALA_IS_CONFIG = DatasetConfig(
+    name="scala-is",
+    pretty_name="the Icelandic part of the linguistic acceptability dataset ScaLA",
+    huggingface_id="EuroEval/scala-is",
+    task=LA,
+    languages=[IS],
+)
+
+MIM_GOLD_NER_CONFIG = DatasetConfig(
+    name="mim-gold-ner",
+    pretty_name="the truncated version of the Icelandic named entity recognition "
+    "dataset MIM-GOLD-NER",
+    huggingface_id="EuroEval/mim-gold-ner-mini",
+    task=NER,
+    languages=[IS],
+)
+
+NQII_CONFIG = DatasetConfig(
+    name="nqii",
+    pretty_name="the truncated version of the Icelandic reading comprehension dataset "
+    "Natural Questions in Icelandic",
+    huggingface_id="EuroEval/nqii-mini",
+    task=RC,
+    languages=[IS],
+)
+
+RRN_CONFIG = DatasetConfig(
+    name="rrn",
+    pretty_name="the truncated version of the Icelandic summarisation dataset "
+    "RÚV Radio News",
+    huggingface_id="EuroEval/rrn-mini",
+    task=SUMM,
+    languages=[IS],
+)
+
+ICELANDIC_KNOWLEDGE_CONFIG = DatasetConfig(
+    name="icelandic-knowledge",
+    pretty_name="the Icelandic knowledge dataset IcelandicKnowledge, derived from the "
+    "IcelandicQA dataset",
+    huggingface_id="EuroEval/icelandic-knowledge",
+    task=KNOW,
+    languages=[IS],
+)
+
+WINOGRANDE_IS_CONFIG = DatasetConfig(
+    name="winogrande-is",
+    pretty_name="the Icelandic common-sense reasoning dataset "
+    "Winogrande-is, manually translated from the English Winogrande dataset",
+    huggingface_id="EuroEval/winogrande-is",
+    task=COMMON_SENSE,
+    languages=[IS],
+)
+
+
+### Unofficial datasets ###
+
+ICE_EC_CONFIG = DatasetConfig(
+    name="ice-ec",
+    pretty_name="the truncated version of the Icelandic Error Corpus",
+    huggingface_id="EuroEval/ice-ec",
+    task=LA,
+    languages=[IS],
+    unofficial=True,
+)
+
+ICE_EC_FULL_CONFIG = DatasetConfig(
+    name="ice-ec-full",
+    pretty_name="the Icelandic Error Corpus",
+    huggingface_id="EuroEval/ice-ec-full",
+    task=LA,
+    languages=[IS],
+    unofficial=True,
+)
+
+ICE_LINGUISTIC_CONFIG = DatasetConfig(
+    name="ice-linguistic",
+    pretty_name="the Icelandic linguistic acceptability dataset IceLinguistic",
+    huggingface_id="EuroEval/ice-linguistic",
+    task=LA,
+    languages=[IS],
+    unofficial=True,
+)
+
+ICELANDIC_QA_CONFIG = DatasetConfig(
+    name="icelandic-qa",
+    pretty_name="the Icelandic reading comprehension dataset IcelandicQA",
+    huggingface_id="EuroEval/icelandic-qa",
+    task=RC,
+    languages=[IS],
+    unofficial=True,
+)
+
+MMLU_IS_CONFIG = DatasetConfig(
+    name="mmlu-is",
+    pretty_name="the truncated version of the Icelandic knowledge dataset MMLU-is, "
+    "translated from the English MMLU dataset",
+    huggingface_id="EuroEval/mmlu-is-mini",
+    task=KNOW,
+    languages=[IS],
+    unofficial=True,
+)
+
+ARC_IS_CONFIG = DatasetConfig(
+    name="arc-is",
+    pretty_name="the truncated version of the Icelandic knowledge dataset ARC-is, "
+    "translated from the English ARC dataset",
+    huggingface_id="EuroEval/arc-is-mini",
+    task=KNOW,
+    languages=[IS],
+    unofficial=True,
+)
+
+HELLASWAG_IS_CONFIG = DatasetConfig(
+    name="hellaswag-is",
+    pretty_name="the truncated version of the Icelandic common-sense reasoning dataset "
+    "HellaSwag-is, translated from the English HellaSwag dataset",
+    huggingface_id="EuroEval/hellaswag-is-mini",
+    task=COMMON_SENSE,
+    languages=[IS],
+    unofficial=True,
+)
+
+BELEBELE_IS_CONFIG = DatasetConfig(
+    name="belebele-is",
+    pretty_name="the Icelandic multiple choice reading comprehension dataset "
+    "BeleBele-is, translated from the English BeleBele dataset",
+    huggingface_id="EuroEval/belebele-is-mini",
+    task=MCRC,
+    languages=[IS],
+    unofficial=True,
+)

euroeval/dataset_configs/italian.py

@@ -0,0 +1,81 @@
+"""All Italian dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import IT
+from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+SENTIPOLC_CONFIG = DatasetConfig(
+    name="sentipolc16",
+    pretty_name="the truncated version of the Italian sentiment classification "
+    "dataset Sentipolc-16",
+    huggingface_id="EuroEval/sentipolc16-mini",
+    task=SENT,
+    languages=[IT],
+)
+
+SCALA_IT_CONFIG = DatasetConfig(
+    name="scala-it",
+    pretty_name="the Italian part of the linguistic acceptability dataset ScaLA",
+    huggingface_id="EuroEval/scala-it",
+    task=LA,
+    languages=[IT],
+)
+
+MULTINERD_IT_CONFIG = DatasetConfig(
+    name="multinerd-it",
+    pretty_name="the truncated version of the Italian part of the named "
+    "entity recognition dataset MultiNERD",
+    huggingface_id="EuroEval/multinerd-mini-it",
+    task=NER,
+    languages=[IT],
+)
+
+SQUAD_IT_CONFIG = DatasetConfig(
+    name="squad-it",
+    pretty_name="the truncated version of the Italian reading comprehension dataset "
+    "SQuAD-it, translated from the English SQuAD dataset",
+    huggingface_id="EuroEval/squad-it-mini",
+    task=RC,
+    languages=[IT],
+)
+
+ILPOST_SUM_CONFIG = DatasetConfig(
+    name="ilpost-sum",
+    pretty_name="the truncated version of the Italian summarisation dataset IlPost-Sum",
+    huggingface_id="EuroEval/ilpost-sum",
+    task=SUMM,
+    languages=[IT],
+)
+
+MMLU_IT_CONFIG = DatasetConfig(
+    name="mmlu-it",
+    pretty_name="the truncated version of the Italian knowledge dataset MMLU-it, "
+    "translated from the English MMLU dataset",
+    huggingface_id="EuroEval/mmlu-it-mini",
+    task=KNOW,
+    languages=[IT],
+)
+
+HELLASWAG_IT_CONFIG = DatasetConfig(
+    name="hellaswag-it",
+    pretty_name="the truncated version of the Italian common-sense reasoning dataset "
+    "HellaSwag-it, translated from the English HellaSwag dataset",
+    huggingface_id="EuroEval/hellaswag-it-mini",
+    task=COMMON_SENSE,
+    languages=[IT],
+)
+
+
+### Unofficial datasets ###
+
+WIKINEURAL_IT_CONFIG = DatasetConfig(
+    name="wikineural-it",
+    pretty_name="the truncated version of the Italian named "
+    "entity recognition dataset WikiNEuRal IT",
+    huggingface_id="EuroEval/wikineural-mini-it",
+    task=NER,
+    languages=[IT],
+    unofficial=True,
+)