EuroEval 15.12.0__py3-none-any.whl → 16.7.1__py3-none-any.whl

euroeval/dataset_configs/serbian.py

@@ -0,0 +1,64 @@
+"""All Serbian dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import SERBIAN
+from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+MMS_SR_CONFIG = DatasetConfig(
+    name="mms-sr",
+    pretty_name="MMS-sr",
+    source="EuroEval/mms-sr-mini",
+    task=SENT,
+    languages=[SERBIAN],
+)
+
+SCALA_SR_CONFIG = DatasetConfig(
+    name="scala-sr",
+    pretty_name="ScaLA-sr",
+    source="EuroEval/scala-sr",
+    task=LA,
+    languages=[SERBIAN],
+)
+
+UNER_SR_CONFIG = DatasetConfig(
+    name="uner-sr",
+    pretty_name="UNER-sr",
+    source="EuroEval/uner-sr-mini",
+    task=NER,
+    languages=[SERBIAN],
+)
+
+MULTI_WIKI_QA_SR_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-sr",
+    pretty_name="MultiWikiQA-sr",
+    source="EuroEval/multi-wiki-qa-sr-mini",
+    task=RC,
+    languages=[SERBIAN],
+)
+
+LR_SUM_SR_CONFIG = DatasetConfig(
+    name="lr-sum-sr",
+    pretty_name="LRSum-sr",
+    source="EuroEval/lr-sum-sr-mini",
+    task=SUMM,
+    languages=[SERBIAN],
+)
+
+MMLU_SR_CONFIG = DatasetConfig(
+    name="mmlu-sr",
+    pretty_name="MMLU-sr",
+    source="EuroEval/mmlu-sr-mini",
+    task=KNOW,
+    languages=[SERBIAN],
+)
+
+WINOGRANDE_SR_CONFIG = DatasetConfig(
+    name="winogrande-sr",
+    pretty_name="Winogrande-sr",
+    source="EuroEval/winogrande-sr",
+    task=COMMON_SENSE,
+    languages=[SERBIAN],
+    _labels=["a", "b"],
+)

euroeval/dataset_configs/slovak.py

@@ -0,0 +1,55 @@
+"""All Slovak dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import SLOVAK
+from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT
+
+### Official datasets ###
+
+CSFD_SENTIMENT_SK_CONFIG = DatasetConfig(
+    name="csfd-sentiment-sk",
+    pretty_name="CSFD Sentiment SK",
+    source="EuroEval/csfd-sentiment-sk-mini",
+    task=SENT,
+    languages=[SLOVAK],
+)
+
+SCALA_SK_CONFIG = DatasetConfig(
+    name="scala-sk",
+    pretty_name="ScaLA-sk",
+    source="EuroEval/scala-sk",
+    task=LA,
+    languages=[SLOVAK],
+)
+
+UNER_SK_CONFIG = DatasetConfig(
+    name="uner-sk",
+    pretty_name="UNER-sk",
+    source="EuroEval/uner-sk-mini",
+    task=NER,
+    languages=[SLOVAK],
+)
+
+MULTI_WIKI_QA_SK_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-sk",
+    pretty_name="MultiWikiQA-sk",
+    source="EuroEval/multi-wiki-qa-sk-mini",
+    task=RC,
+    languages=[SLOVAK],
+)
+
+MMLU_SK_CONFIG = DatasetConfig(
+    name="mmlu-sk",
+    pretty_name="MMLU-sk",
+    source="EuroEval/mmlu-sk-mini",
+    task=KNOW,
+    languages=[SLOVAK],
+)
+
+WINOGRANDE_SK_CONFIG = DatasetConfig(
+    name="winogrande-sk",
+    pretty_name="Winogrande-sk",
+    source="EuroEval/winogrande-sk",
+    task=COMMON_SENSE,
+    languages=[SLOVAK],
+)

euroeval/dataset_configs/slovene.py

@@ -0,0 +1,56 @@
+"""All Slovene dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import SLOVENE
+from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT
+
+### Official datasets ###
+
+SENTINEWS_CONFIG = DatasetConfig(
+    name="sentinews",
+    pretty_name="Sentinews-sl",
+    source="EuroEval/sentinews-mini",
+    task=SENT,
+    languages=[SLOVENE],
+)
+
+SCALA_SL_CONFIG = DatasetConfig(
+    name="scala-sl",
+    pretty_name="ScaLA-sl",
+    source="EuroEval/scala-sl",
+    task=LA,
+    languages=[SLOVENE],
+)
+
+SSJ500K_NER_CONFIG = DatasetConfig(
+    name="ssj500k-ner",
+    pretty_name="ssj500k-NER",
+    source="EuroEval/ssj500k-ner-mini",
+    task=NER,
+    languages=[SLOVENE],
+)
+
+MULTI_WIKI_QA_SL_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-sl",
+    pretty_name="MultiWikiQA-sl",
+    source="EuroEval/multi-wiki-qa-sl-mini",
+    task=RC,
+    languages=[SLOVENE],
+)
+
+MMLU_SL_CONFIG = DatasetConfig(
+    name="mmlu-sl",
+    pretty_name="MMLU-sl",
+    source="EuroEval/mmlu-sl-mini",
+    task=KNOW,
+    languages=[SLOVENE],
+)
+
+WINOGRANDE_SL_CONFIG = DatasetConfig(
+    name="winogrande-sl",
+    pretty_name="Winogrande-sl",
+    source="EuroEval/winogrande-sl",
+    task=COMMON_SENSE,
+    languages=[SLOVENE],
+    _labels=["a", "b"],
+)

euroeval/dataset_configs/spanish.py

@@ -1,69 +1,76 @@
 """All Spanish dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import ES
-from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+from ..languages import SPANISH
+from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 SENTIMENT_HEADLINES_CONFIG = DatasetConfig(
     name="sentiment-headlines-es",
-    pretty_name="the truncated version of the Spanish sentiment classification dataset "
-    "SentimentHeadlines",
-    huggingface_id="EuroEval/sentiment-headlines-es",
+    pretty_name="Sentiment Headlines ES",
+    source="EuroEval/sentiment-headlines-es",
     task=SENT,
-    languages=[ES],
+    languages=[SPANISH],
 )
 
 SCALA_ES_CONFIG = DatasetConfig(
     name="scala-es",
-    pretty_name="the Spanish part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-es",
+    pretty_name="ScaLA-es",
+    source="EuroEval/scala-es",
     task=LA,
-    languages=[ES],
+    languages=[SPANISH],
 )
 
 CONLL_ES_CONFIG = DatasetConfig(
     name="conll-es",
-    pretty_name="the Spanish part of the truncated version of the named entity "
-    "recognition dataset CoNLL 2002",
-    huggingface_id="EuroEval/conll-es-mini",
+    pretty_name="CoNLL-es",
+    source="EuroEval/conll-es-mini",
     task=NER,
-    languages=[ES],
+    languages=[SPANISH],
 )
 
 MLQA_ES_CONFIG = DatasetConfig(
     name="mlqa-es",
-    pretty_name="the Spanish version of the reading comprehension dataset MLQA",
-    huggingface_id="EuroEval/mlqa-es",
+    pretty_name="MLQA-es",
+    source="EuroEval/mlqa-es",
     task=RC,
-    languages=[ES],
+    languages=[SPANISH],
 )
 
 MLSUM_ES_CONFIG = DatasetConfig(
     name="mlsum-es",
-    pretty_name="the truncated version of the Spanish summarisation dataset MLSum-es",
-    huggingface_id="EuroEval/mlsum-es-mini",
+    pretty_name="MLSUM-es",
+    source="EuroEval/mlsum-es-mini",
     task=SUMM,
-    languages=[ES],
+    languages=[SPANISH],
 )
 
 MMLU_ES_CONFIG = DatasetConfig(
     name="mmlu-es",
-    pretty_name="the truncated version of the Spanish knowledge dataset MMLU-es, "
-    "translated from the English MMLU dataset",
-    huggingface_id="EuroEval/mmlu-es-mini",
+    pretty_name="MMLU-es",
+    source="EuroEval/mmlu-es-mini",
     task=KNOW,
-    languages=[ES],
+    languages=[SPANISH],
 )
 
 HELLASWAG_ES_CONFIG = DatasetConfig(
     name="hellaswag-es",
-    pretty_name="the truncated version of the Spanish common-sense reasoning dataset "
-    "HellaSwag-es, translated from the English HellaSwag dataset",
-    huggingface_id="EuroEval/hellaswag-es-mini",
+    pretty_name="HellaSwag-es",
+    source="EuroEval/hellaswag-es-mini",
     task=COMMON_SENSE,
-    languages=[ES],
+    languages=[SPANISH],
+)
+
+VALEU_ES_CONFIG = DatasetConfig(
+    name="valeu-es",
+    pretty_name="VaLEU-es",
+    source="EuroEval/european-values-es",
+    task=EUROPEAN_VALUES,
+    languages=[SPANISH],
+    splits=["test"],
+    bootstrap_samples=False,
+    _instruction_prompt="{text}",
 )
 
 
@@ -71,19 +78,46 @@ HELLASWAG_ES_CONFIG = DatasetConfig(
 
 XQUAD_ES_CONFIG = DatasetConfig(
     name="xquad-es",
-    pretty_name="the Spanish version of the reading comprehension dataset XQuAD",
-    huggingface_id="EuroEval/xquad-es",
+    pretty_name="XQuAD-es",
+    source="EuroEval/xquad-es",
     task=RC,
-    languages=[ES],
+    languages=[SPANISH],
     unofficial=True,
 )
 
 BELEBELE_ES_CONFIG = DatasetConfig(
     name="belebele-es",
-    pretty_name="the Spanish multiple choice reading comprehension dataset "
-    "BeleBele-es, translated from the English BeleBele dataset",
-    huggingface_id="EuroEval/belebele-es-mini",
+    pretty_name="Belebele-es",
+    source="EuroEval/belebele-es-mini",
     task=MCRC,
-    languages=[ES],
+    languages=[SPANISH],
+    unofficial=True,
+)
+
+MULTI_WIKI_QA_ES_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-es",
+    pretty_name="MultiWikiQA-es",
+    source="EuroEval/multi-wiki-qa-es-mini",
+    task=RC,
+    languages=[SPANISH],
+    unofficial=True,
+)
+
+GOLDENSWAG_ES_CONFIG = DatasetConfig(
+    name="goldenswag-es",
+    pretty_name="GoldenSwag-es",
+    source="EuroEval/goldenswag-es-mini",
+    task=COMMON_SENSE,
+    languages=[SPANISH],
+    unofficial=True,
+)
+
+WINOGRANDE_ES_CONFIG = DatasetConfig(
+    name="winogrande-es",
+    pretty_name="Winogrande-es",
+    source="EuroEval/winogrande-es",
+    task=COMMON_SENSE,
+    languages=[SPANISH],
+    _labels=["a", "b"],
     unofficial=True,
 )

euroeval/dataset_configs/swedish.py

@@ -1,70 +1,76 @@
 """All Swedish dataset configurations used in EuroEval."""
 
 from ..data_models import DatasetConfig
-from ..languages import SV
-from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
+from ..languages import SWEDISH
+from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
 SWEREC_CONFIG = DatasetConfig(
     name="swerec",
-    pretty_name="the truncated version of the Swedish sentiment classification "
-    "dataset SweReC",
-    huggingface_id="EuroEval/swerec-mini",
+    pretty_name="SweReC",
+    source="EuroEval/swerec-mini",
     task=SENT,
-    languages=[SV],
+    languages=[SWEDISH],
 )
 
 SCALA_SV_CONFIG = DatasetConfig(
     name="scala-sv",
-    pretty_name="The Swedish part of the linguistic acceptability dataset ScaLA",
-    huggingface_id="EuroEval/scala-sv",
+    pretty_name="ScaLA-sv",
+    source="EuroEval/scala-sv",
     task=LA,
-    languages=[SV],
+    languages=[SWEDISH],
 )
 
 SUC3_CONFIG = DatasetConfig(
     name="suc3",
-    pretty_name="the truncated version of the Swedish named entity recognition "
-    "dataset SUC 3.0",
-    huggingface_id="EuroEval/suc3-mini",
+    pretty_name="SUC3",
+    source="EuroEval/suc3-mini",
     task=NER,
-    languages=[SV],
+    languages=[SWEDISH],
 )
 
-SCANDIQA_SV_CONFIG = DatasetConfig(
-    name="scandiqa-sv",
-    pretty_name="the Swedish part of the truncated version of the question answering "
-    "dataset ScandiQA",
-    huggingface_id="EuroEval/scandiqa-sv-mini",
+MULTI_WIKI_QA_SV_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-sv",
+    pretty_name="MultiWikiQA-sv",
+    source="EuroEval/multi-wiki-qa-sv-mini",
     task=RC,
-    languages=[SV],
+    languages=[SWEDISH],
 )
 
 SWEDN_CONFIG = DatasetConfig(
     name="swedn",
-    pretty_name="the truncated version of the Swedish summarisation dataset SweDN",
-    huggingface_id="EuroEval/swedn-mini",
+    pretty_name="SweDN",
+    source="EuroEval/swedn-mini",
     task=SUMM,
-    languages=[SV],
+    languages=[SWEDISH],
 )
 
 MMLU_SV_CONFIG = DatasetConfig(
     name="mmlu-sv",
-    pretty_name="the truncated version of the Swedish knowledge dataset MMLU-sv, "
-    "translated from the English MMLU dataset",
-    huggingface_id="EuroEval/mmlu-sv-mini",
+    pretty_name="MMLU-sv",
+    source="EuroEval/mmlu-sv-mini",
     task=KNOW,
-    languages=[SV],
+    languages=[SWEDISH],
 )
 
 HELLASWAG_SV_CONFIG = DatasetConfig(
     name="hellaswag-sv",
-    pretty_name="the truncated version of the Swedish common-sense reasoning dataset "
-    "HellaSwag-sv, translated from the English HellaSwag dataset",
-    huggingface_id="EuroEval/hellaswag-sv-mini",
+    pretty_name="HellaSwag-sv",
+    source="EuroEval/hellaswag-sv-mini",
     task=COMMON_SENSE,
-    languages=[SV],
+    languages=[SWEDISH],
+)
+
+VALEU_SV_CONFIG = DatasetConfig(
+    name="valeu-sv",
+    pretty_name="VaLEU-sv",
+    source="EuroEval/european-values-sv",
+    task=EUROPEAN_VALUES,
+    languages=[SWEDISH],
+    splits=["test"],
+    bootstrap_samples=False,
+    _instruction_prompt="{text}",
 )
 
 
@@ -72,29 +78,64 @@ HELLASWAG_SV_CONFIG = DatasetConfig(
 
 SCHIBSTED_SV_CONFIG = DatasetConfig(
     name="schibsted-sv",
-    pretty_name="the Swedish summarisation dataset Schibsted-sv",
-    huggingface_id="EuroEval/schibsted-article-summaries-sv",
+    pretty_name="Schibsted-sv",
+    source="EuroEval/schibsted-article-summaries-sv",
     task=SUMM,
-    languages=[SV],
+    languages=[SWEDISH],
     unofficial=True,
 )
 
 ARC_SV_CONFIG = DatasetConfig(
     name="arc-sv",
-    pretty_name="the truncated version of the Swedish knowledge dataset ARC-sv, "
-    "translated from the English ARC dataset",
-    huggingface_id="EuroEval/arc-sv-mini",
+    pretty_name="ARC-sv",
+    source="EuroEval/arc-sv-mini",
     task=KNOW,
-    languages=[SV],
+    languages=[SWEDISH],
     unofficial=True,
 )
 
 BELEBELE_SV_CONFIG = DatasetConfig(
     name="belebele-sv",
-    pretty_name="the Swedish multiple choice reading comprehension dataset "
-    "BeleBele-sv, translated from the English BeleBele dataset",
-    huggingface_id="EuroEval/belebele-sv-mini",
+    pretty_name="Belebele-sv",
+    source="EuroEval/belebele-sv-mini",
     task=MCRC,
-    languages=[SV],
+    languages=[SWEDISH],
+    unofficial=True,
+)
+
+SCANDIQA_SV_CONFIG = DatasetConfig(
+    name="scandiqa-sv",
+    pretty_name="ScandiQA-sv",
+    source="EuroEval/scandiqa-sv-mini",
+    task=RC,
+    languages=[SWEDISH],
+    unofficial=True,
+)
+
+GOLDENSWAG_SV_CONFIG = DatasetConfig(
+    name="goldenswag-sv",
+    pretty_name="GoldenSwag-sv",
+    source="EuroEval/goldenswag-sv-mini",
+    task=COMMON_SENSE,
+    languages=[SWEDISH],
+    unofficial=True,
+)
+
+WINOGRANDE_SV_CONFIG = DatasetConfig(
+    name="winogrande-sv",
+    pretty_name="Winogrande-sv",
+    source="EuroEval/winogrande-sv",
+    task=COMMON_SENSE,
+    languages=[SWEDISH],
+    _labels=["a", "b"],
+    unofficial=True,
+)
+
+SKOLPROV_CONFIG = DatasetConfig(
+    name="skolprov",
+    pretty_name="Skolprov",
+    source="EuroEval/skolprov",
+    task=KNOW,
+    languages=[SWEDISH],
     unofficial=True,
 )

euroeval/dataset_configs/ukrainian.py

@@ -0,0 +1,64 @@
+"""All Ukrainian dataset configurations used in EuroEval."""
+
+from ..data_models import DatasetConfig
+from ..languages import UKRAINIAN
+from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT, SUMM
+
+### Official datasets ###
+
+CROSS_DOMAIN_UK_REVIEWS_CONFIG = DatasetConfig(
+    name="cross-domain-uk-reviews",
+    pretty_name="Cross Domain Ukrainian Reviews",
+    source="EuroEval/cross-domain-uk-reviews-mini",
+    task=SENT,
+    languages=[UKRAINIAN],
+)
+
+SCALA_UK_CONFIG = DatasetConfig(
+    name="scala-uk",
+    pretty_name="ScaLA-uk",
+    source="EuroEval/scala-uk",
+    task=LA,
+    languages=[UKRAINIAN],
+)
+
+NER_UK_CONFIG = DatasetConfig(
+    name="ner-uk",
+    pretty_name="NER-uk",
+    source="EuroEval/ner-uk-mini",
+    task=NER,
+    languages=[UKRAINIAN],
+)
+
+MULTI_WIKI_QA_UK_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-uk",
+    pretty_name="MultiWikiQA-uk",
+    source="EuroEval/multi-wiki-qa-uk-mini",
+    task=RC,
+    languages=[UKRAINIAN],
+)
+
+LR_SUM_UK_CONFIG = DatasetConfig(
+    name="lr-sum-uk",
+    pretty_name="LRSum-uk",
+    source="EuroEval/lr-sum-uk-mini",
+    task=SUMM,
+    languages=[UKRAINIAN],
+)
+
+GLOBAL_MMLU_UK_CONFIG = DatasetConfig(
+    name="global-mmlu-uk",
+    pretty_name="GlobalMMLU-uk",
+    source="EuroEval/global-mmlu-uk-mini",
+    task=KNOW,
+    languages=[UKRAINIAN],
+)
+
+WINOGRANDE_UK_CONFIG = DatasetConfig(
+    name="winogrande-uk",
+    pretty_name="Winogrande-uk",
+    source="EuroEval/winogrande-uk",
+    task=COMMON_SENSE,
+    languages=[UKRAINIAN],
+    _labels=["a", "b"],
+)

euroeval/enums.py

@@ -12,6 +12,14 @@ class AutoStrEnum(str, Enum):
     ) -> str:
         return name.lower()
 
+    def __str__(self) -> str:
+        """Return the value in upper case for better readability."""
+        return self.value.upper()
+
+    def __repr__(self) -> str:
+        """Return the value in upper case for better readability."""
+        return self.value.upper()
+
 
 class Device(AutoStrEnum):
     """The compute device to use for the evaluation.
@@ -40,14 +48,11 @@ class InferenceBackend(AutoStrEnum):
             VLLM library.
         LITELLM:
             LiteLLM library.
-        NONE:
-            No inference backend used (e.g., for human evaluation).
     """
 
     TRANSFORMERS = auto()
     VLLM = auto()
     LITELLM = auto()
-    NONE = auto()
 
 
 class ModelType(AutoStrEnum):
@@ -58,13 +63,14 @@ class ModelType(AutoStrEnum):
             An encoder (i.e., BERT-style) model.
         GENERATIVE:
             A generative model. Can be either decoder or encoder-decoder (aka seq2seq).
-        HUMAN:
-            Human evaluator.
     """
 
     ENCODER = auto()
     GENERATIVE = auto()
-    HUMAN = auto()
+
+    def __repr__(self) -> str:
+        """Return the value in upper case for better readability."""
+        return self.value.upper()
 
 
 class GenerativeType(AutoStrEnum):

euroeval/exceptions.py

@@ -118,6 +118,26 @@ class NeedsManualDependency(InvalidModel):
         super().__init__(self.message)
 
 
+class NeedsSystemDependency(InvalidModel):
+    """The evaluation requires a system-level dependency to be installed."""
+
+    def __init__(self, dependency: str, instructions: str) -> None:
+        """Initialise the exception.
+
+        Args:
+            dependency:
+                The system dependency that needs to be installed.
+            instructions:
+                Instructions on how to install the dependency.
+        """
+        self.dependency = dependency
+        self.message = (
+            f"The model you are trying to load requires `{dependency}` to be "
+            f"installed. {instructions}"
+        )
+        super().__init__(self.message)
+
+
 class NeedsAdditionalArgument(InvalidModel):
     """The evaluation requires additional arguments to the `euroeval` command."""
 
@@ -145,7 +165,7 @@ class NeedsAdditionalArgument(InvalidModel):
         else:
             self.message = (
                 f"The model you are trying to load requires the `{script_argument}` "
-                "argument  to be passed to the `Benchmarker` class. Please pass the "
+                "argument to be passed to the `Benchmarker` class. Please pass the "
                 "argument and try again."
             )
         super().__init__(self.message)