EuroEval 16.3.0__py3-none-any.whl → 16.5.0__py3-none-any.whl

euroeval/dataset_configs/serbian.py

@@ -0,0 +1,64 @@
+"""All Serbian dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import SERBIAN
+from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+MMS_SR_CONFIG = DatasetConfig(
+    name="mms-sr",
+    pretty_name="MMS-sr",
+    source="EuroEval/mms-sr-mini",
+    task=SENT,
+    languages=[SERBIAN],
+)
+
+SCALA_SR_CONFIG = DatasetConfig(
+    name="scala-sr",
+    pretty_name="ScaLA-sr",
+    source="EuroEval/scala-sr",
+    task=LA,
+    languages=[SERBIAN],
+)
+
+UNER_SR_CONFIG = DatasetConfig(
+    name="uner-sr",
+    pretty_name="UNER-sr",
+    source="EuroEval/uner-sr-mini",
+    task=NER,
+    languages=[SERBIAN],
+)
+
+MULTI_WIKI_QA_SR_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-sr",
+    pretty_name="MultiWikiQA-sr",
+    source="EuroEval/multi-wiki-qa-sr-mini",
+    task=RC,
+    languages=[SERBIAN],
+)
+
+LR_SUM_SR_CONFIG = DatasetConfig(
+    name="lr-sum-sr",
+    pretty_name="LRSum-sr",
+    source="EuroEval/lr-sum-sr-mini",
+    task=SUMM,
+    languages=[SERBIAN],
+)
+
+MMLU_SR_CONFIG = DatasetConfig(
+    name="mmlu-sr",
+    pretty_name="MMLU-sr",
+    source="EuroEval/mmlu-sr-mini",
+    task=KNOW,
+    languages=[SERBIAN],
+)
+
+WINOGRANDE_SR_CONFIG = DatasetConfig(
+    name="winogrande-sr",
+    pretty_name="Winogrande-sr",
+    source="EuroEval/winogrande-sr",
+    task=COMMON_SENSE,
+    languages=[SERBIAN],
+    _labels=["a", "b"],
+)

euroeval/dataset_configs/slovak.py

@@ -0,0 +1,55 @@
+"""All Slovak dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import SLOVAK
+from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT
+
+### Official datasets ###
+
+CSFD_SENTIMENT_SK_CONFIG = DatasetConfig(
+    name="csfd-sentiment-sk",
+    pretty_name="CSFD Sentiment SK",
+    source="EuroEval/csfd-sentiment-sk-mini",
+    task=SENT,
+    languages=[SLOVAK],
+)
+
+SCALA_SK_CONFIG = DatasetConfig(
+    name="scala-sk",
+    pretty_name="ScaLA-sk",
+    source="EuroEval/scala-sk",
+    task=LA,
+    languages=[SLOVAK],
+)
+
+UNER_SK_CONFIG = DatasetConfig(
+    name="uner-sk",
+    pretty_name="UNER-sk",
+    source="EuroEval/uner-sk-mini",
+    task=NER,
+    languages=[SLOVAK],
+)
+
+MULTI_WIKI_QA_SK_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-sk",
+    pretty_name="MultiWikiQA-sk",
+    source="EuroEval/multi-wiki-qa-sk-mini",
+    task=RC,
+    languages=[SLOVAK],
+)
+
+MMLU_SK_CONFIG = DatasetConfig(
+    name="mmlu-sk",
+    pretty_name="MMLU-sk",
+    source="EuroEval/mmlu-sk-mini",
+    task=KNOW,
+    languages=[SLOVAK],
+)
+
+WINOGRANDE_SK_CONFIG = DatasetConfig(
+    name="winogrande-sk",
+    pretty_name="Winogrande-sk",
+    source="EuroEval/winogrande-sk",
+    task=COMMON_SENSE,
+    languages=[SLOVAK],
+)

euroeval/dataset_configs/spanish.py

@@ -1,77 +1,73 @@
 """All Spanish dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import ES
+from ..languages import SPANISH
 from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 SENTIMENT_HEADLINES_CONFIG = DatasetConfig(
     name="sentiment-headlines-es",
-    pretty_name="the truncated version of the Spanish sentiment classification dataset "
-    "SentimentHeadlines",
-    huggingface_id="EuroEval/sentiment-headlines-es",
+    pretty_name="Sentiment Headlines ES",
+    source="EuroEval/sentiment-headlines-es",
     task=SENT,
-    languages=[ES],
+    languages=[SPANISH],
 )
 
 SCALA_ES_CONFIG = DatasetConfig(
     name="scala-es",
-    pretty_name="the Spanish part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-es",
+    pretty_name="ScaLA-es",
+    source="EuroEval/scala-es",
     task=LA,
-    languages=[ES],
+    languages=[SPANISH],
 )
 
 CONLL_ES_CONFIG = DatasetConfig(
     name="conll-es",
-    pretty_name="the Spanish part of the truncated version of the named entity "
-    "recognition dataset CoNLL 2002",
-    huggingface_id="EuroEval/conll-es-mini",
+    pretty_name="CoNLL-es",
+    source="EuroEval/conll-es-mini",
     task=NER,
-    languages=[ES],
+    languages=[SPANISH],
 )
 
 MLQA_ES_CONFIG = DatasetConfig(
     name="mlqa-es",
-    pretty_name="the Spanish version of the reading comprehension dataset MLQA",
-    huggingface_id="EuroEval/mlqa-es",
+    pretty_name="MLQA-es",
+    source="EuroEval/mlqa-es",
     task=RC,
-    languages=[ES],
+    languages=[SPANISH],
 )
 
 MLSUM_ES_CONFIG = DatasetConfig(
     name="mlsum-es",
-    pretty_name="the truncated version of the Spanish summarisation dataset MLSum-es",
-    huggingface_id="EuroEval/mlsum-es-mini",
+    pretty_name="MLSUM-es",
+    source="EuroEval/mlsum-es-mini",
     task=SUMM,
-    languages=[ES],
+    languages=[SPANISH],
 )
 
 MMLU_ES_CONFIG = DatasetConfig(
     name="mmlu-es",
-    pretty_name="the truncated version of the Spanish knowledge dataset MMLU-es, "
-    "translated from the English MMLU dataset",
-    huggingface_id="EuroEval/mmlu-es-mini",
+    pretty_name="MMLU-es",
+    source="EuroEval/mmlu-es-mini",
     task=KNOW,
-    languages=[ES],
+    languages=[SPANISH],
 )
 
 HELLASWAG_ES_CONFIG = DatasetConfig(
     name="hellaswag-es",
-    pretty_name="the truncated version of the Spanish common-sense reasoning dataset "
-    "HellaSwag-es, translated from the English HellaSwag dataset",
-    huggingface_id="EuroEval/hellaswag-es-mini",
+    pretty_name="HellaSwag-es",
+    source="EuroEval/hellaswag-es-mini",
     task=COMMON_SENSE,
-    languages=[ES],
+    languages=[SPANISH],
 )
 
-EUROPEAN_VALUES_ES_CONFIG = DatasetConfig(
-    name="european-values-es",
-    pretty_name="the Spanish version of the European values evaluation dataset",
-    huggingface_id="EuroEval/european-values-es",
+VALEU_ES_CONFIG = DatasetConfig(
+    name="valeu-es",
+    pretty_name="VaLEU-es",
+    source="EuroEval/european-values-es",
     task=EUROPEAN_VALUES,
-    languages=[ES],
+    languages=[SPANISH],
     splits=["test"],
     bootstrap_samples=False,
     _instruction_prompt="{text}",
@@ -82,77 +78,46 @@ EUROPEAN_VALUES_ES_CONFIG = DatasetConfig(
 
 XQUAD_ES_CONFIG = DatasetConfig(
     name="xquad-es",
-    pretty_name="the Spanish version of the reading comprehension dataset XQuAD",
-    huggingface_id="EuroEval/xquad-es",
+    pretty_name="XQuAD-es",
+    source="EuroEval/xquad-es",
     task=RC,
-    languages=[ES],
+    languages=[SPANISH],
     unofficial=True,
 )
 
 BELEBELE_ES_CONFIG = DatasetConfig(
     name="belebele-es",
-    pretty_name="the Spanish multiple choice reading comprehension dataset "
-    "BeleBele-es, translated from the English BeleBele dataset",
-    huggingface_id="EuroEval/belebele-es-mini",
+    pretty_name="Belebele-es",
+    source="EuroEval/belebele-es-mini",
     task=MCRC,
-    languages=[ES],
+    languages=[SPANISH],
     unofficial=True,
 )
 
 MULTI_WIKI_QA_ES_CONFIG = DatasetConfig(
     name="multi-wiki-qa-es",
-    pretty_name="the truncated version of the Spanish part of the reading "
-    "comprehension dataset MultiWikiQA",
-    huggingface_id="EuroEval/multi-wiki-qa-es-mini",
+    pretty_name="MultiWikiQA-es",
+    source="EuroEval/multi-wiki-qa-es-mini",
     task=RC,
-    languages=[ES],
+    languages=[SPANISH],
     unofficial=True,
 )
 
 GOLDENSWAG_ES_CONFIG = DatasetConfig(
     name="goldenswag-es",
-    pretty_name="the truncated version of the Spanish common-sense reasoning "
-    "dataset GoldenSwag-es, translated from the English GoldenSwag dataset",
-    huggingface_id="EuroEval/goldenswag-es-mini",
+    pretty_name="GoldenSwag-es",
+    source="EuroEval/goldenswag-es-mini",
     task=COMMON_SENSE,
-    languages=[ES],
+    languages=[SPANISH],
     unofficial=True,
 )
 
 WINOGRANDE_ES_CONFIG = DatasetConfig(
     name="winogrande-es",
-    pretty_name="the Spanish common-sense reasoning dataset Winogrande-es, translated "
-    "from the English Winogrande dataset",
-    huggingface_id="EuroEval/winogrande-es",
+    pretty_name="Winogrande-es",
+    source="EuroEval/winogrande-es",
     task=COMMON_SENSE,
-    languages=[ES],
-    splits=["train", "test"],
+    languages=[SPANISH],
     _labels=["a", "b"],
     unofficial=True,
 )
-
-EUROPEAN_VALUES_SITUATIONAL_ES_CONFIG = DatasetConfig(
-    name="european-values-situational-es",
-    pretty_name="the Spanish version of the European values evaluation dataset, where "
-    "the questions are phrased in a situational way",
-    huggingface_id="EuroEval/european-values-situational-es",
-    task=EUROPEAN_VALUES,
-    languages=[ES],
-    splits=["test"],
-    bootstrap_samples=False,
-    _instruction_prompt="{text}",
-    unofficial=True,
-)
-
-EUROPEAN_VALUES_COMPLETIONS_ES_CONFIG = DatasetConfig(
-    name="european-values-completions-es",
-    pretty_name="the Spanish version of the European values evaluation dataset, where "
-    "the questions are phrased as sentence completions",
-    huggingface_id="EuroEval/european-values-completions-es",
-    task=EUROPEAN_VALUES,
-    languages=[ES],
-    splits=["test"],
-    bootstrap_samples=False,
-    _instruction_prompt="{text}",
-    unofficial=True,
-)

euroeval/dataset_configs/swedish.py

@@ -1,78 +1,73 @@
 """All Swedish dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import SV
+from ..languages import SWEDISH
 from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 SWEREC_CONFIG = DatasetConfig(
     name="swerec",
-    pretty_name="the truncated version of the Swedish sentiment classification "
-    "dataset SweReC",
-    huggingface_id="EuroEval/swerec-mini",
+    pretty_name="SweReC",
+    source="EuroEval/swerec-mini",
     task=SENT,
-    languages=[SV],
+    languages=[SWEDISH],
 )
 
 SCALA_SV_CONFIG = DatasetConfig(
     name="scala-sv",
-    pretty_name="The Swedish part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-sv",
+    pretty_name="ScaLA-sv",
+    source="EuroEval/scala-sv",
     task=LA,
-    languages=[SV],
+    languages=[SWEDISH],
 )
 
 SUC3_CONFIG = DatasetConfig(
     name="suc3",
-    pretty_name="the truncated version of the Swedish named entity recognition "
-    "dataset SUC 3.0",
-    huggingface_id="EuroEval/suc3-mini",
+    pretty_name="SUC3",
+    source="EuroEval/suc3-mini",
     task=NER,
-    languages=[SV],
+    languages=[SWEDISH],
 )
 
-SCANDIQA_SV_CONFIG = DatasetConfig(
-    name="scandiqa-sv",
-    pretty_name="the Swedish part of the truncated version of the question answering "
-    "dataset ScandiQA",
-    huggingface_id="EuroEval/scandiqa-sv-mini",
+MULTI_WIKI_QA_SV_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-sv",
+    pretty_name="MultiWikiQA-sv",
+    source="EuroEval/multi-wiki-qa-sv-mini",
     task=RC,
-    languages=[SV],
+    languages=[SWEDISH],
 )
 
 SWEDN_CONFIG = DatasetConfig(
     name="swedn",
-    pretty_name="the truncated version of the Swedish summarisation dataset SweDN",
-    huggingface_id="EuroEval/swedn-mini",
+    pretty_name="SweDN",
+    source="EuroEval/swedn-mini",
     task=SUMM,
-    languages=[SV],
+    languages=[SWEDISH],
 )
 
 MMLU_SV_CONFIG = DatasetConfig(
     name="mmlu-sv",
-    pretty_name="the truncated version of the Swedish knowledge dataset MMLU-sv, "
-    "translated from the English MMLU dataset",
-    huggingface_id="EuroEval/mmlu-sv-mini",
+    pretty_name="MMLU-sv",
+    source="EuroEval/mmlu-sv-mini",
     task=KNOW,
-    languages=[SV],
+    languages=[SWEDISH],
 )
 
 HELLASWAG_SV_CONFIG = DatasetConfig(
     name="hellaswag-sv",
-    pretty_name="the truncated version of the Swedish common-sense reasoning dataset "
-    "HellaSwag-sv, translated from the English HellaSwag dataset",
-    huggingface_id="EuroEval/hellaswag-sv-mini",
+    pretty_name="HellaSwag-sv",
+    source="EuroEval/hellaswag-sv-mini",
     task=COMMON_SENSE,
-    languages=[SV],
+    languages=[SWEDISH],
 )
 
-EUROPEAN_VALUES_SV_CONFIG = DatasetConfig(
-    name="european-values-sv",
-    pretty_name="the Swedish version of the European values evaluation dataset",
-    huggingface_id="EuroEval/european-values-sv",
+VALEU_SV_CONFIG = DatasetConfig(
+    name="valeu-sv",
+    pretty_name="VaLEU-sv",
+    source="EuroEval/european-values-sv",
     task=EUROPEAN_VALUES,
-    languages=[SV],
+    languages=[SWEDISH],
     splits=["test"],
     bootstrap_samples=False,
     _instruction_prompt="{text}",
@@ -83,97 +78,64 @@ EUROPEAN_VALUES_SV_CONFIG = DatasetConfig(
 
 SCHIBSTED_SV_CONFIG = DatasetConfig(
     name="schibsted-sv",
-    pretty_name="the Swedish summarisation dataset Schibsted-sv",
-    huggingface_id="EuroEval/schibsted-article-summaries-sv",
+    pretty_name="Schibsted-sv",
+    source="EuroEval/schibsted-article-summaries-sv",
     task=SUMM,
-    languages=[SV],
+    languages=[SWEDISH],
     unofficial=True,
 )
 
 ARC_SV_CONFIG = DatasetConfig(
     name="arc-sv",
-    pretty_name="the truncated version of the Swedish knowledge dataset ARC-sv, "
-    "translated from the English ARC dataset",
-    huggingface_id="EuroEval/arc-sv-mini",
+    pretty_name="ARC-sv",
+    source="EuroEval/arc-sv-mini",
     task=KNOW,
-    languages=[SV],
+    languages=[SWEDISH],
     unofficial=True,
 )
 
 BELEBELE_SV_CONFIG = DatasetConfig(
     name="belebele-sv",
-    pretty_name="the Swedish multiple choice reading comprehension dataset "
-    "BeleBele-sv, translated from the English BeleBele dataset",
-    huggingface_id="EuroEval/belebele-sv-mini",
+    pretty_name="Belebele-sv",
+    source="EuroEval/belebele-sv-mini",
     task=MCRC,
-    languages=[SV],
+    languages=[SWEDISH],
     unofficial=True,
 )
 
-MULTI_WIKI_QA_SV_CONFIG = DatasetConfig(
-    name="multi-wiki-qa-sv",
-    pretty_name="the truncated version of the Swedish part of the reading "
-    "comprehension dataset MultiWikiQA",
-    huggingface_id="EuroEval/multi-wiki-qa-sv-mini",
+SCANDIQA_SV_CONFIG = DatasetConfig(
+    name="scandiqa-sv",
+    pretty_name="ScandiQA-sv",
+    source="EuroEval/scandiqa-sv-mini",
     task=RC,
-    languages=[SV],
+    languages=[SWEDISH],
     unofficial=True,
 )
 
 GOLDENSWAG_SV_CONFIG = DatasetConfig(
     name="goldenswag-sv",
-    pretty_name="the truncated version of the Swedish common-sense reasoning "
-    "dataset GoldenSwag-sv, translated from the English GoldenSwag dataset",
-    huggingface_id="EuroEval/goldenswag-sv-mini",
+    pretty_name="GoldenSwag-sv",
+    source="EuroEval/goldenswag-sv-mini",
     task=COMMON_SENSE,
-    languages=[SV],
+    languages=[SWEDISH],
     unofficial=True,
 )
 
 WINOGRANDE_SV_CONFIG = DatasetConfig(
     name="winogrande-sv",
-    pretty_name="the Swedish common-sense reasoning dataset Winogrande-sv, translated "
-    "from the English Winogrande dataset",
-    huggingface_id="EuroEval/winogrande-sv",
+    pretty_name="Winogrande-sv",
+    source="EuroEval/winogrande-sv",
     task=COMMON_SENSE,
-    languages=[SV],
-    splits=["train", "test"],
+    languages=[SWEDISH],
     _labels=["a", "b"],
     unofficial=True,
 )
 
-EUROPEAN_VALUES_SITUATIONAL_SV_CONFIG = DatasetConfig(
-    name="european-values-situational-sv",
-    pretty_name="the Swedish version of the European values evaluation dataset, where "
-    "the questions are phrased in a situational way",
-    huggingface_id="EuroEval/european-values-situational-sv",
-    task=EUROPEAN_VALUES,
-    languages=[SV],
-    splits=["test"],
-    bootstrap_samples=False,
-    _instruction_prompt="{text}",
-    unofficial=True,
-)
-
-EUROPEAN_VALUES_COMPLETIONS_SV_CONFIG = DatasetConfig(
-    name="european-values-completions-sv",
-    pretty_name="the Swedish version of the European values evaluation dataset, where "
-    "the questions are phrased as sentence completions",
-    huggingface_id="EuroEval/european-values-completions-sv",
-    task=EUROPEAN_VALUES,
-    languages=[SV],
-    splits=["test"],
-    bootstrap_samples=False,
-    _instruction_prompt="{text}",
-    unofficial=True,
-)
-
 SKOLPROV_CONFIG = DatasetConfig(
     name="skolprov",
-    pretty_name="the Swedish knowledge dataset Skolprov",
-    huggingface_id="EuroEval/skolprov",
+    pretty_name="Skolprov",
+    source="EuroEval/skolprov",
     task=KNOW,
-    languages=[SV],
-    splits=["train", "test"],
+    languages=[SWEDISH],
     unofficial=True,
 )

euroeval/dataset_configs/ukrainian.py

@@ -0,0 +1,64 @@
+"""All Ukrainian dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import UKRAINIAN
+from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+CROSS_DOMAIN_UK_REVIEWS_CONFIG = DatasetConfig(
+    name="cross-domain-uk-reviews",
+    pretty_name="Cross Domain Ukrainian Reviews",
+    source="EuroEval/cross-domain-uk-reviews-mini",
+    task=SENT,
+    languages=[UKRAINIAN],
+)
+
+SCALA_UK_CONFIG = DatasetConfig(
+    name="scala-uk",
+    pretty_name="ScaLA-uk",
+    source="EuroEval/scala-uk",
+    task=LA,
+    languages=[UKRAINIAN],
+)
+
+NER_UK_CONFIG = DatasetConfig(
+    name="ner-uk",
+    pretty_name="NER-uk",
+    source="EuroEval/ner-uk-mini",
+    task=NER,
+    languages=[UKRAINIAN],
+)
+
+MULTI_WIKI_QA_UK_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-uk",
+    pretty_name="MultiWikiQA-uk",
+    source="EuroEval/multi-wiki-qa-uk-mini",
+    task=RC,
+    languages=[UKRAINIAN],
+)
+
+LR_SUM_UK_CONFIG = DatasetConfig(
+    name="lr-sum-uk",
+    pretty_name="LRSum-uk",
+    source="EuroEval/lr-sum-uk-mini",
+    task=SUMM,
+    languages=[UKRAINIAN],
+)
+
+GLOBAL_MMLU_UK_CONFIG = DatasetConfig(
+    name="global-mmlu-uk",
+    pretty_name="GlobalMMLU-uk",
+    source="EuroEval/global-mmlu-uk-mini",
+    task=KNOW,
+    languages=[UKRAINIAN],
+)
+
+WINOGRANDE_UK_CONFIG = DatasetConfig(
+    name="winogrande-uk",
+    pretty_name="Winogrande-uk",
+    source="EuroEval/winogrande-uk",
+    task=COMMON_SENSE,
+    languages=[UKRAINIAN],
+    _labels=["a", "b"],
+)

euroeval/exceptions.py

@@ -145,7 +145,7 @@ class NeedsAdditionalArgument(InvalidModel):
         else:
             self.message = (
                 f"The model you are trying to load requires the `{script_argument}` "
-                "argument  to be passed to the `Benchmarker` class. Please pass the "
+                "argument to be passed to the `Benchmarker` class. Please pass the "
                 "argument and try again."
             )
         super().__init__(self.message)