EuroEval 15.10.1__py3-none-any.whl → 15.11.0__py3-none-any.whl

euroeval/metrics.py

@@ -0,0 +1,452 @@
+"""All the metrics used in EuroEval."""
+
+import abc
+import logging
+import typing as t
+
+import evaluate
+import litellm
+from litellm.types.utils import Choices, ModelResponse
+from pydantic import BaseModel, Field
+from tqdm.auto import tqdm
+
+from .exceptions import InvalidBenchmark
+from .utils import HiddenPrints
+
+if t.TYPE_CHECKING:
+    from evaluate import EvaluationModule
+
+logger = logging.getLogger(__name__)
+
+
+class Metric(abc.ABC):
+    """Abstract base class for all metrics."""
+
+    def __init__(
+        self,
+        name: str,
+        pretty_name: str,
+        postprocessing_fn: t.Callable[[float], tuple[float, str]] | None = None,
+    ) -> None:
+        """Initialise the metric.
+
+        Args:
+            name:
+                The name of the metric in snake_case.
+            pretty_name:
+                The pretty name of the metric, used for display purposes.
+            postprocessing_fn:
+                A function to apply to the metric scores after they are computed,
+                taking the score to the postprocessed score along with its string
+                representation. Defaults to x -> (100 * x, f"{x:.2%}").
+        """
+        self.name = name
+        self.pretty_name = pretty_name
+        self.postprocessing_fn = (
+            postprocessing_fn
+            if postprocessing_fn is not None
+            else lambda x: (100 * x, f"{x:.2%}")
+        )
+
+    @abc.abstractmethod
+    def __call__(self, predictions: t.Sequence, references: t.Sequence) -> float | None:
+        """Calculate the metric score.
+
+        Args:
+            predictions:
+                The model predictions.
+            references:
+                The ground truth references.
+
+        Returns:
+            The calculated metric score, or None if the score should be ignored.
+        """
+        ...
+
+    def __hash__(self) -> int:
+        """Return a hash of the metric configuration."""
+        return hash(self.name)
+
+
+class HuggingFaceMetric(Metric):
+    """A metric which is implemented in the `evaluate` package.
+
+    Attributes:
+        name:
+            The name of the metric in snake_case.
+        pretty_name:
+            The pretty name of the metric, used for display purposes.
+        huggingface_id:
+            The Hugging Face ID of the metric.
+        results_key:
+            The name of the key used to extract the metric scores from the results
+            dictionary.
+        compute_kwargs:
+            Keyword arguments to pass to the metric's compute function. Defaults to
+            an empty dictionary.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        pretty_name: str,
+        huggingface_id: str,
+        results_key: str,
+        compute_kwargs: dict[str, t.Any] | None = None,
+        postprocessing_fn: t.Callable[[float], tuple[float, str]] | None = None,
+    ) -> None:
+        """Initialise the Hugging Face metric.
+
+        Args:
+            name:
+                The name of the metric in snake_case.
+            pretty_name:
+                The pretty name of the metric, used for display purposes.
+            huggingface_id:
+                The Hugging Face ID of the metric.
+            results_key:
+                The name of the key used to extract the metric scores from the results
+                dictionary.
+            compute_kwargs:
+                Keyword arguments to pass to the metric's compute function. Defaults to
+                an empty dictionary.
+            postprocessing_fn:
+                A function to apply to the metric scores after they are computed, taking
+                the score to the postprocessed score along with its string
+                representation. Defaults to x -> (100 * x, f"{x:.2%}").
+        """
+        super().__init__(
+            name=name, pretty_name=pretty_name, postprocessing_fn=postprocessing_fn
+        )
+        self.huggingface_id = huggingface_id
+        self.results_key = results_key
+        self.compute_kwargs: dict[str, t.Any] = (
+            dict() if compute_kwargs is None else compute_kwargs
+        )
+        self.metric: "EvaluationModule | None" = None
+
+    def __call__(self, predictions: t.Sequence, references: t.Sequence) -> float | None:
+        """Calculate the metric score.
+
+        Args:
+            predictions:
+                The model predictions.
+            references:
+                The ground truth references.
+
+        Returns:
+            The calculated metric score, or None if the score should be ignored.
+        """
+        if self.metric is None:
+            self.metric = evaluate.load(path=self.huggingface_id)
+
+        with HiddenPrints():
+            results = self.metric.compute(
+                predictions=predictions, references=references, **self.compute_kwargs
+            )
+
+        # The metric returns None if we are running on multi-GPU and the current
+        # process is not the main process
+        if results is None:
+            return None
+
+        score = results[self.results_key]
+        if isinstance(score, list):
+            score = sum(score) / len(score)
+
+        return score
+
+
+class LLMAsAJudgeMetric(Metric):
+    """Use an LLM to judge the quality of the predictions."""
+
+    def __init__(
+        self,
+        name: str,
+        pretty_name: str,
+        judge_id: str,
+        judge_kwargs: dict[str, t.Any],
+        user_prompt: str,
+        response_format: t.Type[BaseModel],
+        scoring_fn: t.Callable[[BaseModel], float],
+        condition_formatting_fn: t.Callable[[str], str] = lambda x: x,
+        system_prompt: str | None = None,
+    ) -> None:
+        """Initialise the LLM as a judge metric.
+
+        Args:
+            name:
+                The name of the metric in snake_case.
+            pretty_name:
+                The pretty name of the metric, used for display purposes.
+            judge_id:
+                The model ID of the LLM to use as a judge.
+            judge_kwargs:
+                Generation parameters for the judge model, such as temperature.
+            user_prompt:
+                The user prompt to use for the judge model. The prompt should be
+                formatted with the variables `prediction` and `condition`, to
+                include the model predictions and a description of what the prediction
+                should be judged on, respectively. If the condition is not needed,
+                it can be omitted from the prompt, but the `prediction` variable must
+                still be present.
+            response_format:
+                The response format to use for the judge model. This should be a
+                Pydantic model that defines the expected structure of the judge's
+                response.
+            scoring_fn:
+                A function that takes the judge's response and returns a score.
+            condition_formatting_fn (optional):
+                A function to format the condition string before it is included in the
+                user prompt. Defaults to a no-op function that returns the input
+                unchanged.
+            system_prompt (optional):
+                The system prompt to use for the judge model. If not provided, no system
+                prompt will be used.
+        """
+        super().__init__(name=name, pretty_name=pretty_name)
+        self.judge_id = judge_id
+        self.judge_kwargs = judge_kwargs
+        self.user_prompt = user_prompt
+        self.response_format = response_format
+        self.scoring_fn = scoring_fn
+        self.condition_formatting_fn = condition_formatting_fn
+        self.system_prompt = system_prompt
+
+    def __call__(self, predictions: t.Sequence, references: t.Sequence) -> float | None:
+        """Calculate the metric score using the judge model.
+
+        Args:
+            predictions:
+                The model predictions.
+            references:
+                The ground truth references.
+
+        Returns:
+            The calculated metric score, or None if the score should be ignored.
+
+        Raises:
+            InvalidBenchmark:
+                If the number of predictions does not match the number of references,
+                or if the user prompt requires a condition but none is provided.
+        """
+        if not predictions or not references:
+            return None
+        elif len(predictions) != len(references):
+            raise InvalidBenchmark(
+                f"The number of predictions ({len(predictions):,}) does not match the "
+                f"number of references ({len(references):,})."
+            )
+
+        # Prepare the messages for the LLM
+        conversations: list[list[dict[str, str]]] = [
+            [
+                dict(
+                    role="user",
+                    content=self._apply_user_prompt(
+                        prediction=prediction, condition=condition
+                    ),
+                )
+            ]
+            for prediction, condition in zip(predictions, references)
+        ]
+        if self.system_prompt:
+            conversations = [
+                [dict(role="system", content=self.system_prompt), *conversation]
+                for conversation in conversations
+            ]
+
+        # Get the judge generations
+        generations = [
+            litellm.completion(
+                model=self.judge_id,
+                messages=conversation,
+                response_format=self.response_format,
+                **self.judge_kwargs,
+            )
+            for conversation in tqdm(
+                iterable=conversations,
+                desc=f"Computing {self.pretty_name} scores",
+                unit="sample",
+            )
+        ]
+
+        # Extract the outputs from the generations
+        outputs: list[BaseModel] = list()
+        for generation in generations:
+            assert isinstance(generation, ModelResponse), (
+                f"The judge model did not return a valid response: {generation!r}"
+            )
+            choice = generation.choices[0]
+            assert isinstance(choice, Choices), (
+                f"The judge model did not return a valid choice: {choice!r}"
+            )
+            json_content = choice.message.content
+            assert json_content is not None, (
+                "The judge model returned a None content in the response message."
+            )
+            output = self.response_format.model_validate_json(json_data=json_content)
+            outputs.append(output)
+
+        # Calculate the scores using the scoring function
+        scores = [self.scoring_fn(output) for output in outputs]
+        if not scores:
+            logger.warning(f"No scores were calculated for {self.pretty_name}.")
+            return None
+        return sum(scores) / len(scores)
+
+    def _apply_user_prompt(self, prediction: str, condition: str | None = None) -> str:
+        """Apply the user prompt to the prediction and condition.
+
+        Args:
+            prediction:
+                The model prediction.
+            condition (optional):
+                A description of what the prediction should be judged on. If not
+                provided, it will be omitted from the prompt.
+
+        Returns:
+            The formatted user prompt with the prediction and reference.
+
+        Raises:
+            InvalidBenchmark:
+                If the user prompt requires a reference but none is provided.
+        """
+        condition_required = "{condition}" in self.user_prompt
+        if condition_required and condition is None:
+            raise InvalidBenchmark(
+                f"The user prompt for the {self.pretty_name!r} metric requires a "
+                "condition, but none was provided."
+            )
+        if condition is not None:
+            return self.user_prompt.format(
+                prediction=prediction, condition=self.condition_formatting_fn(condition)
+            )
+        return self.user_prompt.format(prediction=prediction)
+
+
+class SpeedMetric(Metric):
+    """Speed metric."""
+
+    def __init__(self, name: str, pretty_name: str) -> None:
+        """Initialise the speed metric.
+
+        Args:
+            name:
+                The name of the metric in snake_case.
+            pretty_name:
+                The pretty name of the metric, used for display purposes.
+        """
+        super().__init__(
+            name=name,
+            pretty_name=pretty_name,
+            postprocessing_fn=lambda raw_score: (raw_score, f"{raw_score:,.0f}"),
+        )
+
+    def __call__(self, _: t.Sequence, __: t.Sequence) -> float | None:
+        """Not used with the speed metric, but required for consistency."""
+        raise NotImplementedError
+
+
+mcc_metric = HuggingFaceMetric(
+    name="mcc",
+    pretty_name="Matthew's Correlation Coefficient",
+    huggingface_id="matthews_correlation",
+    results_key="matthews_correlation",
+)
+
+macro_f1_metric = HuggingFaceMetric(
+    name="macro_f1",
+    pretty_name="Macro-average F1-score",
+    huggingface_id="f1",
+    results_key="f1",
+    compute_kwargs=dict(average="macro"),
+)
+
+micro_f1_metric = HuggingFaceMetric(
+    name="micro_f1",
+    pretty_name="Micro-average F1-score with MISC tags",
+    huggingface_id="seqeval",
+    results_key="overall_f1",
+)
+
+micro_f1_no_misc_metric = HuggingFaceMetric(
+    name="micro_f1_no_misc",
+    pretty_name="Micro-average F1-score without MISC tags",
+    huggingface_id="seqeval",
+    results_key="overall_f1",
+)
+
+f1_metric = HuggingFaceMetric(
+    name="f1",
+    pretty_name="F1-score",
+    huggingface_id="squad_v2",
+    results_key="f1",
+    postprocessing_fn=lambda x: (x, f"{x:.2f}%"),
+)
+
+em_metric = HuggingFaceMetric(
+    name="em",
+    pretty_name="Exact Match",
+    huggingface_id="squad_v2",
+    results_key="exact",
+    postprocessing_fn=lambda x: (x, f"{x:.2f}%"),
+)
+
+bert_score_metric = HuggingFaceMetric(
+    name="bertscore",
+    pretty_name="BERTScore",
+    huggingface_id="bertscore",
+    results_key="f1",
+    compute_kwargs=dict(
+        model_type="microsoft/mdeberta-v3-base", device="auto", batch_size=1
+    ),
+)
+
+rouge_l_metric = HuggingFaceMetric(
+    name="rouge_l", pretty_name="ROUGE-L", huggingface_id="rouge", results_key="rougeL"
+)
+
+accuracy_metric = HuggingFaceMetric(
+    name="accuracy",
+    pretty_name="Accuracy",
+    huggingface_id="accuracy",
+    results_key="accuracy",
+)
+
+
+class Fluency(BaseModel):
+    """Response format for the fluency metric.
+
+    Attributes:
+        fluency:
+            The fluency rating, an integer between 1 and 5.
+    """
+
+    fluency: t.Annotated[int, Field(ge=1, le=5)]
+
+
+# Example LLM-as-a-judge metric, to measure the fluency of the LLM output
+fluency_metric = LLMAsAJudgeMetric(
+    name="fluency",
+    pretty_name="Fluency",
+    judge_id="gpt-4o-mini",
+    judge_kwargs=dict(temperature=0.0),
+    user_prompt="Please rate the fluency of the following text on a scale from 1 to 5, "
+    "with the following definitions:\n"
+    "- 1: Very poor fluency, many grammatical errors\n"
+    "- 2: Poor fluency, several grammatical errors\n"
+    "- 3: Average fluency, a few grammatical errors\n"
+    "- 4: Good fluency, no grammatical errors but sounds a bit off\n"
+    "- 5: Excellent fluency, no grammatical errors and sounds natural\n\n"
+    "Text: {prediction!r}\n\n"
+    "Output your rating as a JSON object with a single key 'fluency'.",
+    response_format=Fluency,
+    scoring_fn=lambda output: (output.fluency - 1) / 4.0,
+)
+
+speed_metric = SpeedMetric(name="speed", pretty_name="Tokens per second")
+
+speed_short_metric = SpeedMetric(
+    name="speed_short", pretty_name="Tokens per second on short documents"
+)

euroeval/scores.py

@@ -7,7 +7,7 @@ import warnings
 import numpy as np
 
 if t.TYPE_CHECKING:
-    from .data_models import MetricConfig
+    from .metrics import Metric
     from .types import ScoreDict
 
 logger = logging.getLogger("euroeval")
@@ -15,7 +15,7 @@ logger = logging.getLogger("euroeval")
 
 def log_scores(
     dataset_name: str,
-    metric_configs: list["MetricConfig"],
+    metrics: list["Metric"],
     scores: list[dict[str, float]],
     model_id: str,
     model_revision: str,
@@ -25,7 +25,7 @@ def log_scores(
     Args:
         dataset_name:
             Name of the dataset.
-        metric_configs:
+        metrics:
             List of metrics to log.
         scores:
             The scores that are to be logged. This is a list of dictionaries full of
@@ -46,19 +46,19 @@ def log_scores(
     logger.info(f"Finished evaluation of {model_id} on {dataset_name}.")
 
     total_dict: dict[str, float] = dict()
-    for metric_cfg in metric_configs:
-        test_score, test_se = aggregate_scores(scores=scores, metric_config=metric_cfg)
-        test_score, test_score_str = metric_cfg.postprocessing_fn(test_score)
-        test_se, test_se_str = metric_cfg.postprocessing_fn(test_se)
-        total_dict[f"test_{metric_cfg.name}"] = test_score
-        total_dict[f"test_{metric_cfg.name}_se"] = test_se
-        logger.info(f"{metric_cfg.pretty_name}: {test_score_str} ± {test_se_str}")
+    for metric in metrics:
+        test_score, test_se = aggregate_scores(scores=scores, metric=metric)
+        test_score, test_score_str = metric.postprocessing_fn(test_score)
+        test_se, test_se_str = metric.postprocessing_fn(test_se)
+        total_dict[f"test_{metric.name}"] = test_score
+        total_dict[f"test_{metric.name}_se"] = test_se
+        logger.info(f"{metric.pretty_name}: {test_score_str} ± {test_se_str}")
 
     return dict(raw=scores, total=total_dict)
 
 
 def aggregate_scores(
-    scores: list[dict[str, float]], metric_config: "MetricConfig"
+    scores: list[dict[str, float]], metric: "Metric"
 ) -> tuple[float, float]:
     """Helper function to compute the mean with confidence intervals.
 
@@ -66,9 +66,8 @@ def aggregate_scores(
         scores:
             Dictionary with the names of the metrics as keys, of the form
             "<split>_<metric_name>", such as "val_f1", and values the metric values.
-        metric_config:
-            The configuration of the metric, which is used to collect the correct
-            metric from `scores`.
+        metric:
+            The metric, which is used to collect the correct metric from `scores`.
 
     Returns:
         A pair of floats, containing the score and the radius of its 95% confidence
@@ -78,11 +77,7 @@ def aggregate_scores(
         warnings.simplefilter("ignore")
 
         test_scores = [
-            (
-                dct[metric_config.name]
-                if metric_config.name in dct
-                else dct[f"test_{metric_config.name}"]
-            )
+            dct[metric.name] if metric.name in dct else dct[f"test_{metric.name}"]
             for dct in scores
         ]
         test_score = np.mean(test_scores).item()

euroeval/speed_benchmark.py

@@ -1,21 +1,20 @@
 """Benchmarking model inference speed."""
 
 import logging
+import typing as t
 
 import pyinfer
 from tqdm.auto import tqdm
 from transformers.models.auto.tokenization_auto import AutoTokenizer
 
-from .benchmark_modules import (
-    BenchmarkModule,
-    HuggingFaceEncoderModel,
-    LiteLLMModel,
-    VLLMModel,
-)
-from .data_models import BenchmarkConfig
+from .benchmark_modules import HuggingFaceEncoderModel, LiteLLMModel, VLLMModel
 from .exceptions import InvalidBenchmark
 from .utils import clear_memory
 
+if t.TYPE_CHECKING:
+    from .benchmark_modules import BenchmarkModule
+    from .data_models import BenchmarkConfig
+
 logger = logging.getLogger("euroeval")
 
 

euroeval/task_group_utils/multiple_choice_classification.py

@@ -7,14 +7,15 @@ import typing as t
 from collections import defaultdict
 
 import numpy as np
-from datasets import Dataset
-from transformers.tokenization_utils import PreTrainedTokenizer
-from transformers.tokenization_utils_base import BatchEncoding
 from transformers.trainer import Trainer
 
 from ..exceptions import InvalidBenchmark
 
 if t.TYPE_CHECKING:
+    from datasets import Dataset
+    from transformers.tokenization_utils import PreTrainedTokenizer
+    from transformers.tokenization_utils_base import BatchEncoding
+
     from ..types import Labels, Predictions
 
 logger = logging.getLogger("euroeval")
@@ -147,7 +148,8 @@ def postprocess_predictions_and_labels(
 
     Args:
         predictions:
-            The model predictions, of shape (num_examples, 2).
+            The model predictions, of shape (num_examples, 2), corresponding to the
+            False/True probabilities for each example.
         dataset:
             The dataset containing the examples.
 

euroeval/task_group_utils/question_answering.py

@@ -5,13 +5,10 @@ import logging
 import typing as t
 from collections import defaultdict
 
-import evaluate
 import numpy as np
-from evaluate import EvaluationModule
 from transformers.tokenization_utils_base import PreTrainedTokenizerBase
 from transformers.trainer import Trainer
 
-from ..data_models import BenchmarkConfig, DatasetConfig, GenerativeModelOutput
 from ..exceptions import InvalidBenchmark
 from ..tokenization_utils import get_special_token_metadata
 from ..utils import raise_if_model_output_contains_nan_values
@@ -26,6 +23,7 @@ if t.TYPE_CHECKING:
     from transformers.trainer_utils import EvalPrediction
     from transformers.training_args import TrainingArguments
 
+    from ..data_models import DatasetConfig, GenerativeModelOutput
     from ..types import Labels, Predictions
 
 logger = logging.getLogger("euroeval")
@@ -151,7 +149,6 @@ class QuestionAnsweringTrainer(Trainer):
 def compute_metrics(
     model_outputs_and_labels: "tuple[Predictions, Labels] | EvalPrediction",
     dataset_config: "DatasetConfig",
-    benchmark_config: "BenchmarkConfig",
 ) -> dict[str, float]:
     """Compute the metrics needed for evaluation.
 
@@ -161,8 +158,6 @@ def compute_metrics(
             contains the true labels.
         dataset_config:
             The configuration of the dataset.
-        benchmark_config:
-            The configuration of the benchmark.
 
     Returns:
         A dictionary with the names of the metrics as keys and the metric values as
@@ -178,17 +173,6 @@ def compute_metrics(
     assert not isinstance(model_outputs, tuple)
     raise_if_model_output_contains_nan_values(model_output=model_outputs)
 
-    metrics = {
-        metric_cfg.name: (
-            evaluate.load(
-                path=metric_cfg.huggingface_id, cache_dir=benchmark_config.cache_dir
-            )
-            if metric_cfg.huggingface_id != ""
-            else None
-        )
-        for metric_cfg in dataset_config.task.metrics
-    }
-
     model_output_dtype = np.asarray(model_outputs).dtype
     if model_output_dtype in [np.float16, np.float32, np.float64]:
         predictions = np.asarray(model_outputs).argmax(axis=-1)
@@ -196,20 +180,13 @@ def compute_metrics(
         predictions = model_outputs
 
     results: dict[str, float] = dict()
-    for cfg in dataset_config.task.metrics:
-        metric = metrics[cfg.name]
-        assert isinstance(metric, EvaluationModule)
-        score_dict: dict[str, float] | None = metric.compute(
-            predictions=predictions, references=labels, **cfg.compute_kwargs
-        )
+    for metric in dataset_config.task.metrics:
+        score: float | None = metric(predictions=predictions, references=labels)
 
         # The metric returns None if we are running on multi-GPU and the current
         # process is not the main process
-        if score_dict is not None:
-            scores = score_dict[cfg.results_key]
-            if isinstance(scores, list):
-                scores = sum(scores) / len(scores)
-            results[cfg.name] = scores
+        if score is not None:
+            results[metric.name] = score
 
     return results