EuroEval 15.10.1__py3-none-any.whl → 15.11.0__py3-none-any.whl

euroeval/task_group_utils/sequence_classification.py

@@ -4,19 +4,16 @@ import logging
 import re
 import typing as t
 
-import evaluate
 import Levenshtein
 import numpy as np
-from evaluate import EvaluationModule
 
-from ..data_models import BenchmarkConfig, GenerativeModelOutput
 from ..exceptions import InvalidBenchmark
 from ..utils import log_once, raise_if_model_output_contains_nan_values
 
 if t.TYPE_CHECKING:
     from transformers.trainer_utils import EvalPrediction
 
-    from ..data_models import DatasetConfig
+    from ..data_models import DatasetConfig, GenerativeModelOutput
     from ..types import Labels, Predictions
 
 
@@ -26,7 +23,6 @@ logger = logging.getLogger("euroeval")
 def compute_metrics(
     model_outputs_and_labels: "tuple[Predictions, Labels] | EvalPrediction",
     dataset_config: "DatasetConfig",
-    benchmark_config: "BenchmarkConfig",
 ) -> dict[str, float]:
     """Compute the metrics needed for evaluation.
 
@@ -36,8 +32,6 @@ def compute_metrics(
             contains the true labels.
         dataset_config:
             The configuration of the dataset.
-        benchmark_config:
-            The configuration of the benchmark.
 
     Returns:
         A dictionary with the names of the metrics as keys and the metric values as
@@ -51,17 +45,6 @@ def compute_metrics(
     if isinstance(model_outputs, tuple) and len(model_outputs) == 2:
         model_outputs = model_outputs[0]
 
-    metrics = {
-        metric_cfg.name: (
-            evaluate.load(
-                path=metric_cfg.huggingface_id, cache_dir=benchmark_config.cache_dir
-            )
-            if metric_cfg.huggingface_id != ""
-            else None
-        )
-        for metric_cfg in dataset_config.task.metrics
-    }
-
     model_output_dtype = np.asarray(model_outputs).dtype
     if model_output_dtype in [np.float16, np.float32, np.float64]:
         predictions = np.asarray(model_outputs).argmax(axis=-1)
@@ -89,27 +72,20 @@ def compute_metrics(
     ]
 
     results: dict[str, float] = dict()
-    for cfg in dataset_config.task.metrics:
-        metric = metrics[cfg.name]
-        assert isinstance(metric, EvaluationModule)
-        score_dict: dict[str, float] | None = metric.compute(
-            predictions=predictions, references=label_ids, **cfg.compute_kwargs
-        )
+    for metric in dataset_config.task.metrics:
+        score: float | None = metric(predictions=predictions, references=label_ids)
 
         # The metric returns None if we are running on multi-GPU and the current
         # process is not the main process
-        if score_dict is not None:
-            scores = score_dict[cfg.results_key]
-            if isinstance(scores, list):
-                scores = sum(scores) / len(scores)
-            results[cfg.name] = scores
+        if score is not None:
+            results[metric.name] = score
 
     return results
 
 
 def extract_labels_from_generation(
     input_batch: dict[str, list],
-    model_output: GenerativeModelOutput,
+    model_output: "GenerativeModelOutput",
     dataset_config: "DatasetConfig",
     first_label_token_mapping: dict[str, str] | bool,
 ) -> list[str]:

euroeval/task_group_utils/text_to_text.py

@@ -3,18 +3,17 @@
 import logging
 import typing as t
 
-import evaluate
 import numpy as np
-from evaluate import EvaluationModule
 
 from ..constants import METRIC_ATTRIBUTES_TAKING_UP_MEMORY
-from ..data_models import BenchmarkConfig, DatasetConfig, GenerativeModelOutput
 from ..exceptions import InvalidBenchmark
-from ..utils import HiddenPrints, raise_if_model_output_contains_nan_values
+from ..metrics import HuggingFaceMetric
+from ..utils import raise_if_model_output_contains_nan_values
 
 if t.TYPE_CHECKING:
     from transformers.trainer_utils import EvalPrediction
 
+    from ..data_models import BenchmarkConfig, DatasetConfig, GenerativeModelOutput
     from ..types import Labels, Predictions
 
 
@@ -51,17 +50,6 @@ def compute_metrics(
     assert not isinstance(model_outputs, tuple)
     raise_if_model_output_contains_nan_values(model_output=model_outputs)
 
-    metrics = {
-        metric_cfg.name: (
-            evaluate.load(
-                path=metric_cfg.huggingface_id, cache_dir=benchmark_config.cache_dir
-            )
-            if metric_cfg.huggingface_id != ""
-            else None
-        )
-        for metric_cfg in dataset_config.task.metrics
-    }
-
     model_output_dtype = np.asarray(model_outputs).dtype
     output_is_prob = model_output_dtype in [np.float16, np.float32, np.float64]
     if output_is_prob:
@@ -70,21 +58,18 @@ def compute_metrics(
         predictions = model_outputs
 
     results: dict[str, float] = dict()
-    for cfg in dataset_config.task.metrics:
-        metric = metrics[cfg.name]
-        assert isinstance(metric, EvaluationModule)
-
+    for metric in dataset_config.task.metrics:
         # Some metrics can be computed on hardware accelerators. In this case we
         # start by setting the device to the same device as the model
-        if cfg.compute_kwargs.get("device", None) == "auto":
-            cfg.compute_kwargs["device"] = benchmark_config.device.type
+        if (
+            isinstance(metric, HuggingFaceMetric)
+            and metric.compute_kwargs.get("device", None) == "auto"
+        ):
+            metric.compute_kwargs["device"] = benchmark_config.device.type
 
         while True:
             try:
-                with HiddenPrints():
-                    score_dict: dict[str, float] | None = metric.compute(
-                        predictions=predictions, references=labels, **cfg.compute_kwargs
-                    )
+                score: float | None = metric(predictions=predictions, references=labels)
                 break
             except Exception as e:
                 oom_error = [
@@ -95,11 +80,14 @@ def compute_metrics(
                 if not any(error in str(e) for error in oom_error):
                     raise InvalidBenchmark(str(e))
 
-                if cfg.compute_kwargs.get("device", "cpu") != "cpu":
-                    cfg.compute_kwargs["device"] = "cpu"
+                if (
+                    isinstance(metric, HuggingFaceMetric)
+                    and metric.compute_kwargs.get("device", "cpu") != "cpu"
+                ):
+                    metric.compute_kwargs["device"] = "cpu"
                     logger.debug(
                         "Out of memory error occurred during the computation of "
-                        f"the metric {cfg.pretty_name}. Moving the computation to "
+                        f"the metric {metric.pretty_name}. Moving the computation to "
                         "the CPU."
                     )
                 else:
@@ -109,17 +97,14 @@ def compute_metrics(
                     if hasattr(metric, attribute):
                         logger.debug(
                             f"Deleting the {attribute!r} attribute of the metric "
-                            f"{cfg.pretty_name} to free up memory."
+                            f"{metric.pretty_name} to free up memory."
                         )
                         delattr(metric, attribute)
 
         # The metric returns None if we are running on multi-GPU and the current
         # process is not the main process
-        if score_dict is not None:
-            scores = score_dict[cfg.results_key]
-            if isinstance(scores, list):
-                scores = sum(scores) / len(scores)
-            results[cfg.name] = scores
+        if score is not None:
+            results[metric.name] = score
 
     return results
 

euroeval/task_group_utils/token_classification.py

@@ -6,19 +6,17 @@ import typing as t
 from copy import deepcopy
 
 import demjson3
-import evaluate
 import numpy as np
-from evaluate import EvaluationModule
-from transformers.tokenization_utils import PreTrainedTokenizer
 
-from ..data_models import BenchmarkConfig, DatasetConfig, GenerativeModelOutput
 from ..exceptions import InvalidBenchmark
 from ..utils import raise_if_model_output_contains_nan_values
 
 if t.TYPE_CHECKING:
+    from transformers.tokenization_utils import PreTrainedTokenizer
     from transformers.tokenization_utils_base import BatchEncoding
     from transformers.trainer_utils import EvalPrediction
 
+    from ..data_models import DatasetConfig, GenerativeModelOutput
     from ..types import Labels, Predictions
 
 
@@ -29,7 +27,6 @@ def compute_metrics(
     model_outputs_and_labels: "tuple[Predictions, Labels] | EvalPrediction",
     has_misc_tags: bool,
     dataset_config: "DatasetConfig",
-    benchmark_config: "BenchmarkConfig",
 ) -> dict[str, float]:
     """Compute the metrics needed for evaluation.
 
@@ -41,8 +38,6 @@ def compute_metrics(
             Whether the dataset has MISC tags.
         dataset_config:
             The configuration of the dataset.
-        benchmark_config:
-            The configuration of the benchmark.
 
     Returns:
         A dictionary with the names of the metrics as keys and the metric values as
@@ -55,17 +50,6 @@ def compute_metrics(
     if isinstance(model_outputs, tuple) and len(model_outputs) == 2:
         model_outputs = model_outputs[0]
 
-    metrics = {
-        metric_cfg.name: (
-            evaluate.load(
-                path=metric_cfg.huggingface_id, cache_dir=benchmark_config.cache_dir
-            )
-            if metric_cfg.huggingface_id != ""
-            else None
-        )
-        for metric_cfg in dataset_config.task.metrics
-    }
-
     predictions: list[list[str]]
     if not isinstance(model_outputs[0][0], str):
         raw_predictions: list[list[int]] = np.argmax(model_outputs, axis=-1).tolist()
@@ -145,11 +129,14 @@ def compute_metrics(
         all(ner_tag == "o" for ner_tag in label_list) for label_list in labels
     )
     if predictions_all_zero and labels_all_zero:
-        results = dict(overall_f1=1.0)
+        micro_f1_score: float | None = 1.0
     else:
-        metric = metrics["micro_f1"]
-        assert isinstance(metric, EvaluationModule)
-        results = metric.compute(predictions=predictions, references=labels)
+        metric = next(
+            metric
+            for metric in dataset_config.task.metrics
+            if metric.name == "micro_f1"
+        )
+        micro_f1_score = metric(predictions=predictions, references=list(labels))
 
     # Compute the metrics without MISC tags
     # We manually set the F1 metric to be 100% if both the labels and the models
@@ -163,21 +150,22 @@ def compute_metrics(
         all(ner_tag == "o" for ner_tag in label_list) for label_list in labels_no_misc
     )
     if predictions_no_misc_all_zero and labels_no_misc_all_zero:
-        results_no_misc = dict(overall_f1=1.0)
+        micro_f1_no_misc_score: float | None = 1.0
     else:
-        metric = metrics["micro_f1_no_misc"]
-        assert isinstance(metric, EvaluationModule)
-        results_no_misc = metric.compute(
+        metric = next(
+            metric
+            for metric in dataset_config.task.metrics
+            if metric.name == "micro_f1_no_misc"
+        )
+        micro_f1_no_misc_score = metric(
             predictions=predictions_no_misc, references=labels_no_misc
         )
 
     # Raise error if the metrics are invalid
-    if results is None or results_no_misc is None:
+    if micro_f1_score is None or micro_f1_no_misc_score is None:
         raise InvalidBenchmark("The predictions and labels are not of the same length.")
 
-    return dict(
-        micro_f1_no_misc=results_no_misc["overall_f1"], micro_f1=results["overall_f1"]
-    )
+    return dict(micro_f1_no_misc=micro_f1_no_misc_score, micro_f1=micro_f1_score)
 
 
 def extract_labels_from_generation(

euroeval/tasks.py

@@ -1,6 +1,7 @@
 """All benchmarks tasks used in EuroEval."""
 
-from .data_models import MetricConfig, Task
+from . import metrics as m
+from .data_models import Task
 from .enums import TaskGroup
 from .prompt_templates import (
     LA_TEMPLATES,
@@ -25,21 +26,7 @@ LA = Task(
     name="linguistic-acceptability",
     task_group=TaskGroup.SEQUENCE_CLASSIFICATION,
     template_dict=LA_TEMPLATES,
-    metrics=[
-        MetricConfig(
-            name="mcc",
-            pretty_name="Matthew's Correlation Coefficient",
-            huggingface_id="matthews_correlation",
-            results_key="matthews_correlation",
-        ),
-        MetricConfig(
-            name="macro_f1",
-            pretty_name="Macro-average F1-score",
-            huggingface_id="f1",
-            results_key="f1",
-            compute_kwargs=dict(average="macro"),
-        ),
-    ],
+    metrics=[m.mcc_metric, m.macro_f1_metric],
     default_num_few_shot_examples=12,
     default_max_generated_tokens=5,
     default_labels=["correct", "incorrect"],
@@ -50,20 +37,7 @@ NER = Task(
     name="named-entity-recognition",
     task_group=TaskGroup.TOKEN_CLASSIFICATION,
     template_dict=NER_TEMPLATES,
-    metrics=[
-        MetricConfig(
-            name="micro_f1_no_misc",
-            pretty_name="Micro-average F1-score without MISC tags",
-            huggingface_id="seqeval",
-            results_key="overall_f1",
-        ),
-        MetricConfig(
-            name="micro_f1",
-            pretty_name="Micro-average F1-score with MISC tags",
-            huggingface_id="seqeval",
-            results_key="overall_f1",
-        ),
-    ],
+    metrics=[m.micro_f1_no_misc_metric, m.micro_f1_metric],
     default_num_few_shot_examples=8,
     default_max_generated_tokens=128,
     default_labels=[
@@ -84,22 +58,7 @@ RC = Task(
     name="reading-comprehension",
     task_group=TaskGroup.QUESTION_ANSWERING,
     template_dict=RC_TEMPLATES,
-    metrics=[
-        MetricConfig(
-            name="f1",
-            pretty_name="F1-score",
-            huggingface_id="squad_v2",
-            results_key="f1",
-            postprocessing_fn=lambda raw_score: (raw_score, f"{raw_score:.2f}%"),
-        ),
-        MetricConfig(
-            name="em",
-            pretty_name="Exact Match",
-            huggingface_id="squad_v2",
-            results_key="exact",
-            postprocessing_fn=lambda raw_score: (raw_score, f"{raw_score:.2f}%"),
-        ),
-    ],
+    metrics=[m.f1_metric, m.em_metric],
     default_num_few_shot_examples=4,
     default_max_generated_tokens=32,
     default_labels=["start_positions", "end_positions"],
@@ -110,21 +69,7 @@ SENT = Task(
     name="sentiment-classification",
     task_group=TaskGroup.SEQUENCE_CLASSIFICATION,
     template_dict=SENT_TEMPLATES,
-    metrics=[
-        MetricConfig(
-            name="mcc",
-            pretty_name="Matthew's Correlation Coefficient",
-            huggingface_id="matthews_correlation",
-            results_key="matthews_correlation",
-        ),
-        MetricConfig(
-            name="macro_f1",
-            pretty_name="Macro-average F1-score",
-            huggingface_id="f1",
-            results_key="f1",
-            compute_kwargs=dict(average="macro"),
-        ),
-    ],
+    metrics=[m.mcc_metric, m.macro_f1_metric],
     default_num_few_shot_examples=12,
     default_max_generated_tokens=5,
     default_labels=["positive", "neutral", "negative"],
@@ -135,23 +80,7 @@ SUMM = Task(
     name="summarization",
     task_group=TaskGroup.TEXT_TO_TEXT,
     template_dict=SUMM_TEMPLATES,
-    metrics=[
-        MetricConfig(
-            name="bertscore",
-            pretty_name="BERTScore",
-            huggingface_id="bertscore",
-            results_key="f1",
-            compute_kwargs=dict(
-                model_type="microsoft/mdeberta-v3-base", device="auto", batch_size=1
-            ),
-        ),
-        MetricConfig(
-            name="rouge_l",
-            pretty_name="ROUGE-L",
-            huggingface_id="rouge",
-            results_key="rougeL",
-        ),
-    ],
+    metrics=[m.bert_score_metric, m.rouge_l_metric],
     default_num_few_shot_examples=1,
     default_max_generated_tokens=256,
     default_labels=[],
@@ -162,20 +91,7 @@ KNOW = Task(
     name="knowledge",
     task_group=TaskGroup.MULTIPLE_CHOICE_CLASSIFICATION,
     template_dict=MULTIPLE_CHOICE_TEMPLATES,
-    metrics=[
-        MetricConfig(
-            name="mcc",
-            pretty_name="Matthew's Correlation Coefficient",
-            huggingface_id="matthews_correlation",
-            results_key="matthews_correlation",
-        ),
-        MetricConfig(
-            name="accuracy",
-            pretty_name="Accuracy",
-            huggingface_id="accuracy",
-            results_key="accuracy",
-        ),
-    ],
+    metrics=[m.mcc_metric, m.accuracy_metric],
     default_num_few_shot_examples=5,
     default_max_generated_tokens=5,
     default_labels=["a", "b", "c", "d"],
@@ -186,20 +102,7 @@ MCRC = Task(
     name="multiple-choice-reading-comprehension",
     task_group=TaskGroup.MULTIPLE_CHOICE_CLASSIFICATION,
     template_dict=MULTIPLE_CHOICE_TEMPLATES,
-    metrics=[
-        MetricConfig(
-            name="mcc",
-            pretty_name="Matthew's Correlation Coefficient",
-            huggingface_id="matthews_correlation",
-            results_key="matthews_correlation",
-        ),
-        MetricConfig(
-            name="accuracy",
-            pretty_name="Accuracy",
-            huggingface_id="accuracy",
-            results_key="accuracy",
-        ),
-    ],
+    metrics=[m.mcc_metric, m.accuracy_metric],
     default_num_few_shot_examples=5,
     default_max_generated_tokens=5,
     default_labels=["a", "b", "c", "d"],
@@ -210,20 +113,7 @@ COMMON_SENSE = Task(
     name="common-sense-reasoning",
     task_group=TaskGroup.MULTIPLE_CHOICE_CLASSIFICATION,
     template_dict=MULTIPLE_CHOICE_TEMPLATES,
-    metrics=[
-        MetricConfig(
-            name="mcc",
-            pretty_name="Matthew's Correlation Coefficient",
-            huggingface_id="matthews_correlation",
-            results_key="matthews_correlation",
-        ),
-        MetricConfig(
-            name="accuracy",
-            pretty_name="Accuracy",
-            huggingface_id="accuracy",
-            results_key="accuracy",
-        ),
-    ],
+    metrics=[m.mcc_metric, m.accuracy_metric],
     default_num_few_shot_examples=5,
     default_max_generated_tokens=5,
     default_labels=["a", "b", "c", "d"],
@@ -234,22 +124,7 @@ SPEED = Task(
     name="speed",
     task_group=TaskGroup.SPEED,
     template_dict={},
-    metrics=[
-        MetricConfig(
-            name="speed",
-            pretty_name="Tokens per second",
-            huggingface_id="",
-            results_key="speed",
-            postprocessing_fn=lambda raw_score: (raw_score, f"{raw_score:,.0f}"),
-        ),
-        MetricConfig(
-            name="speed_short",
-            pretty_name="Tokens per second on short documents",
-            huggingface_id="",
-            results_key="speed",
-            postprocessing_fn=lambda raw_score: (raw_score, f"{raw_score:,.0f}"),
-        ),
-    ],
+    metrics=[m.speed_metric, m.speed_short_metric],
     default_num_few_shot_examples=0,
     default_max_generated_tokens=5,
     default_labels=[],

euroeval/types.py

@@ -2,16 +2,17 @@
 
 import typing as t
 
-from numpy.typing import NDArray
 from transformers.trainer_utils import EvalPrediction
 
 if t.TYPE_CHECKING:
+    from numpy.typing import NDArray
+
     from .data_models import GenerativeModelOutput
 
 
 ScoreDict: t.TypeAlias = dict[str, dict[str, float] | list[dict[str, float]]]
-Predictions: t.TypeAlias = NDArray | list[str] | list[list[str]]
-Labels: t.TypeAlias = NDArray | list[str] | list[list[str]]
+Predictions: t.TypeAlias = "NDArray | list[str] | list[list[str]]"
+Labels: t.TypeAlias = "NDArray | list[str] | list[list[str]]"
 
 
 class ComputeMetricsFunction(t.Protocol):
@@ -21,7 +22,8 @@ class ComputeMetricsFunction(t.Protocol):
         self,
         model_outputs_and_labels: EvalPrediction
         | tuple[
-            NDArray | list[str] | list[list[str]], NDArray | list[str] | list[list[str]]
+            "NDArray | list[str] | list[list[str]]",
+            "NDArray | list[str] | list[list[str]]",
         ],
     ) -> dict[str, float]:
         """Compute the metrics.

{euroeval-15.10.1.dist-info → euroeval-15.11.0.dist-info}/METADATA

@@ -1,14 +1,14 @@
 Metadata-Version: 2.4
 Name: EuroEval
-Version: 15.10.1
+Version: 15.11.0
 Summary: The robust European language model benchmark.
 Project-URL: Repository, https://github.com/EuroEval/EuroEval
 Project-URL: Issues, https://github.com/EuroEval/EuroEval/issues
-Author-email: Dan Saattrup Nielsen <dan.nielsen@alexandra.dk>
-Maintainer-email: Dan Saattrup Nielsen <dan.nielsen@alexandra.dk>
+Author-email: Dan Saattrup Smart <dan.smart@alexandra.dk>
+Maintainer-email: Dan Saattrup Smart <dan.smart@alexandra.dk>
 License: MIT License
         
-        Copyright (c) 2022-2024 Dan Saattrup Nielsen
+        Copyright (c) 2022-2025 Dan Saattrup Smart
         
         Permission is hereby granted, free of charge, to any person obtaining a copy
         of this software and associated documentation files (the "Software"), to deal
@@ -43,6 +43,7 @@ Requires-Dist: numpy<2.0.0,>=1.23.0
 Requires-Dist: ollama>=0.5.1
 Requires-Dist: pandas>=2.2.0
 Requires-Dist: peft>=0.15.0
+Requires-Dist: protobuf>=2.0.0
 Requires-Dist: pydantic>=2.6.0
 Requires-Dist: pyinfer>=0.0.3
 Requires-Dist: python-dotenv>=1.0.1
@@ -94,8 +95,7 @@ ______________________________________________________________________
 
 ## Maintainer
 
-- Dan Saattrup Nielsen ([@saattrupdan](https://github.com/saattrupdan),
-  dan.nielsen@alexandra.dk)
+- Dan Saattrup Smart ([@saattrupdan](https://github.com/saattrupdan), dan.smart@alexandra.dk)
 
 
 ## Installation
@@ -268,14 +268,14 @@ contributing new datasets, your help makes this project better for everyone.
 If you want to cite the framework then feel free to use this:
 
 ```
-@article{nielsen2024encoder,
+@article{smart2024encoder,
   title={Encoder vs Decoder: Comparative Analysis of Encoder and Decoder Language Models on Multilingual NLU Tasks},
-  author={Nielsen, Dan Saattrup and Enevoldsen, Kenneth and Schneider-Kamp, Peter},
+  author={Smart, Dan Saattrup and Enevoldsen, Kenneth and Schneider-Kamp, Peter},
   journal={arXiv preprint arXiv:2406.13469},
   year={2024}
 }
-@inproceedings{nielsen2023scandeval,
-  author = {Nielsen, Dan Saattrup},
+@inproceedings{smart2023scandeval,
+  author = {Smart, Dan Saattrup},
   booktitle = {Proceedings of the 24th Nordic Conference on Computational Linguistics (NoDaLiDa)},
   month = may,
   pages = {185--201},