EuroEval 15.15.0__py3-none-any.whl → 16.0.0__py3-none-any.whl

euroeval/metrics/__init__.py

@@ -0,0 +1,6 @@
+"""All the metrics used in EuroEval."""
+
+from .huggingface import *  # noqa: F403
+from .llm_as_a_judge import *  # noqa: F403
+from .pipeline import *  # noqa: F403
+from .speed import *  # noqa: F403

euroeval/metrics/base.py

@@ -0,0 +1,76 @@
+"""The abstract base class for all metrics."""
+
+import abc
+import collections.abc as c
+import logging
+import typing as t
+
+if t.TYPE_CHECKING:
+    from datasets.arrow_dataset import Dataset
+
+    from ..data_models import BenchmarkConfig, DatasetConfig
+
+logger: logging.Logger = logging.getLogger("euroeval")
+
+
+class Metric(abc.ABC):
+    """Abstract base class for all metrics."""
+
+    def __init__(
+        self,
+        name: str,
+        pretty_name: str,
+        postprocessing_fn: t.Callable[[float], tuple[float, str]] | None = None,
+    ) -> None:
+        """Initialise the metric.
+
+        Args:
+            name:
+                The name of the metric in snake_case.
+            pretty_name:
+                The pretty name of the metric, used for display purposes.
+            postprocessing_fn:
+                A function to apply to the metric scores after they are computed,
+                taking the score to the postprocessed score along with its string
+                representation. Defaults to x -> (100 * x, f"{x:.2%}").
+        """
+        self.name = name
+        self.pretty_name = pretty_name
+        self.postprocessing_fn = (
+            postprocessing_fn
+            if postprocessing_fn is not None
+            else lambda x: (100 * x, f"{x:.2%}")
+        )
+
+    @abc.abstractmethod
+    def __call__(
+        self,
+        predictions: c.Sequence,
+        references: c.Sequence,
+        dataset: "Dataset",
+        dataset_config: "DatasetConfig",
+        benchmark_config: "BenchmarkConfig",
+    ) -> float | None:
+        """Calculate the metric score.
+
+        Args:
+            predictions:
+                The model predictions.
+            references:
+                The ground truth references.
+            dataset:
+                The dataset used for evaluation. This is only used in case any
+                additional metadata is used to compute the metrics.
+            dataset_config:
+                The dataset configuration.
+            benchmark_config:
+                The benchmark configuration.
+
+        Returns:
+            The calculated metric score, or None if the score should be ignored.
+        """
+        ...
+
+    def __hash__(self) -> int:
+        """Return a hash of the metric configuration."""
+        return hash(self.name)

euroeval/metrics/huggingface.py

@@ -0,0 +1,192 @@
+"""All the Hugging Face metrics used in EuroEval."""
+
+import collections.abc as c
+import logging
+import typing as t
+
+import evaluate
+import numpy as np
+
+from ..utils import HiddenPrints
+from .base import Metric
+
+if t.TYPE_CHECKING:
+    from datasets.arrow_dataset import Dataset
+    from evaluate import EvaluationModule
+
+    from ..data_models import BenchmarkConfig, DatasetConfig
+
+logger: logging.Logger = logging.getLogger("euroeval")
+
+
+class HuggingFaceMetric(Metric):
+    """A metric which is implemented in the `evaluate` package.
+
+    Attributes:
+        name:
+            The name of the metric in snake_case.
+        pretty_name:
+            The pretty name of the metric, used for display purposes.
+        huggingface_id:
+            The Hugging Face ID of the metric.
+        results_key:
+            The name of the key used to extract the metric scores from the results
+            dictionary.
+        compute_kwargs:
+            Keyword arguments to pass to the metric's compute function. Defaults to
+            an empty dictionary.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        pretty_name: str,
+        huggingface_id: str,
+        results_key: str,
+        compute_kwargs: dict[str, t.Any] | None = None,
+        postprocessing_fn: t.Callable[[float], tuple[float, str]] | None = None,
+    ) -> None:
+        """Initialise the Hugging Face metric.
+
+        Args:
+            name:
+                The name of the metric in snake_case.
+            pretty_name:
+                The pretty name of the metric, used for display purposes.
+            huggingface_id:
+                The Hugging Face ID of the metric.
+            results_key:
+                The name of the key used to extract the metric scores from the results
+                dictionary.
+            compute_kwargs:
+                Keyword arguments to pass to the metric's compute function. Defaults to
+                an empty dictionary.
+            postprocessing_fn:
+                A function to apply to the metric scores after they are computed, taking
+                the score to the postprocessed score along with its string
+                representation. Defaults to x -> (100 * x, f"{x:.2%}").
+        """
+        super().__init__(
+            name=name, pretty_name=pretty_name, postprocessing_fn=postprocessing_fn
+        )
+        self.huggingface_id = huggingface_id
+        self.results_key = results_key
+        self.compute_kwargs: dict[str, t.Any] = (
+            dict() if compute_kwargs is None else compute_kwargs
+        )
+        self.metric: "EvaluationModule | None" = None
+
+    def __call__(
+        self,
+        predictions: c.Sequence,
+        references: c.Sequence,
+        dataset: "Dataset",
+        dataset_config: "DatasetConfig",
+        benchmark_config: "BenchmarkConfig",
+    ) -> float | None:
+        """Calculate the metric score.
+
+        Args:
+            predictions:
+                The model predictions.
+            references:
+                The ground truth references.
+            dataset:
+                The dataset used for evaluation. This is only used in case any
+                additional metadata is used to compute the metrics.
+            dataset_config:
+                The dataset configuration.
+            benchmark_config:
+                The benchmark configuration.
+
+        Returns:
+            The calculated metric score, or None if the score should be ignored.
+        """
+        if self.metric is None:
+            self.metric = evaluate.load(path=self.huggingface_id)
+
+        with HiddenPrints():
+            results = self.metric.compute(
+                predictions=predictions, references=references, **self.compute_kwargs
+            )
+
+        # The metric returns None if we are running on multi-GPU and the current
+        # process is not the main process
+        if results is None:
+            return None
+
+        # Convert the results to a float score
+        score = results[self.results_key]
+        if isinstance(score, list):
+            score = sum(score) / len(score)
+        if isinstance(score, np.floating):
+            score = float(score)
+
+        return score
+
+
+mcc_metric = HuggingFaceMetric(
+    name="mcc",
+    pretty_name="Matthew's Correlation Coefficient",
+    huggingface_id="matthews_correlation",
+    results_key="matthews_correlation",
+)
+
+macro_f1_metric = HuggingFaceMetric(
+    name="macro_f1",
+    pretty_name="Macro-average F1-score",
+    huggingface_id="f1",
+    results_key="f1",
+    compute_kwargs=dict(average="macro"),
+)
+
+micro_f1_metric = HuggingFaceMetric(
+    name="micro_f1",
+    pretty_name="Micro-average F1-score with MISC tags",
+    huggingface_id="seqeval",
+    results_key="overall_f1",
+)
+
+micro_f1_no_misc_metric = HuggingFaceMetric(
+    name="micro_f1_no_misc",
+    pretty_name="Micro-average F1-score without MISC tags",
+    huggingface_id="seqeval",
+    results_key="overall_f1",
+)
+
+f1_metric = HuggingFaceMetric(
+    name="f1",
+    pretty_name="F1-score",
+    huggingface_id="squad_v2",
+    results_key="f1",
+    postprocessing_fn=lambda x: (x, f"{x:.2f}%"),
+)
+
+em_metric = HuggingFaceMetric(
+    name="em",
+    pretty_name="Exact Match",
+    huggingface_id="squad_v2",
+    results_key="exact",
+    postprocessing_fn=lambda x: (x, f"{x:.2f}%"),
+)
+
+bert_score_metric = HuggingFaceMetric(
+    name="bertscore",
+    pretty_name="BERTScore",
+    huggingface_id="bertscore",
+    results_key="f1",
+    compute_kwargs=dict(
+        model_type="microsoft/mdeberta-v3-base", device="auto", batch_size=1
+    ),
+)
+
+rouge_l_metric = HuggingFaceMetric(
+    name="rouge_l", pretty_name="ROUGE-L", huggingface_id="rouge", results_key="rougeL"
+)
+
+accuracy_metric = HuggingFaceMetric(
+    name="accuracy",
+    pretty_name="Accuracy",
+    huggingface_id="accuracy",
+    results_key="accuracy",
+)

euroeval/metrics/llm_as_a_judge.py

@@ -0,0 +1,257 @@
+"""Metrics based on LLM-as-a-judge."""
+
+import collections.abc as c
+import logging
+import typing as t
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from ..exceptions import InvalidBenchmark
+from ..model_cache import ModelCache
+from ..utils import extract_json_dict_from_string
+from .base import Metric
+
+if t.TYPE_CHECKING:
+    from datasets.arrow_dataset import Dataset
+
+    from ..data_models import BenchmarkConfig, DatasetConfig
+
+logger: logging.Logger = logging.getLogger("euroeval")
+
+
+class LLMAsAJudgeMetric(Metric):
+    """Use an LLM to judge the quality of the predictions."""
+
+    def __init__(
+        self,
+        name: str,
+        pretty_name: str,
+        judge_id: str,
+        judge_kwargs: dict[str, t.Any],
+        user_prompt: str,
+        response_format: t.Type[BaseModel],
+        scoring_fn: t.Callable[[BaseModel | None], float],
+        condition_formatting_fn: t.Callable[[str], str] = lambda x: x,
+        system_prompt: str | None = None,
+    ) -> None:
+        """Initialise the LLM as a judge metric.
+
+        Args:
+            name:
+                The name of the metric in snake_case.
+            pretty_name:
+                The pretty name of the metric, used for display purposes.
+            judge_id:
+                The model ID of the LLM to use as a judge.
+            judge_kwargs:
+                Generation parameters for the judge model, such as temperature.
+            user_prompt:
+                The user prompt to use for the judge model. The prompt should be
+                formatted with the variables `prediction` and `condition`, to
+                include the model predictions and a description of what the prediction
+                should be judged on, respectively. If the condition is not needed,
+                it can be omitted from the prompt, but the `prediction` variable must
+                still be present.
+            response_format:
+                The response format to use for the judge model. This should be a
+                Pydantic model that defines the expected structure of the judge's
+                response.
+            scoring_fn:
+                A function that takes the judge's response and returns a score.
+            condition_formatting_fn (optional):
+                A function to format the condition string before it is included in the
+                user prompt. Defaults to a no-op function that returns the input
+                unchanged.
+            system_prompt (optional):
+                The system prompt to use for the judge model. If not provided, no system
+                prompt will be used.
+        """
+        super().__init__(name=name, pretty_name=pretty_name)
+        self.judge_id = judge_id
+        self.judge_kwargs = judge_kwargs
+        self.user_prompt = user_prompt
+        self.response_format = response_format
+        self.scoring_fn = scoring_fn
+        self.condition_formatting_fn = condition_formatting_fn
+        self.system_prompt = system_prompt
+
+        # Add response format to the generation kwargs
+        self.judge_kwargs["response_format"] = self.response_format
+
+    def __call__(
+        self,
+        predictions: c.Sequence,
+        references: c.Sequence,
+        dataset: "Dataset",
+        dataset_config: "DatasetConfig",
+        benchmark_config: "BenchmarkConfig",
+    ) -> float | None:
+        """Calculate the metric score using the judge model.
+
+        Args:
+            predictions:
+                The model predictions.
+            references:
+                The ground truth references.
+            dataset:
+                The dataset used for evaluation. This is only used in case any
+                additional metadata is used to compute the metrics.
+            dataset_config:
+                The dataset configuration.
+            benchmark_config:
+                The benchmark configuration.
+
+        Returns:
+            The calculated metric score, or None if the score should be ignored.
+
+        Raises:
+            InvalidBenchmark:
+                If the number of predictions does not match the number of references,
+                or if the user prompt requires a condition but none is provided.
+        """
+        # Importing here to avoid circular imports
+        from ..benchmark_modules import LiteLLMModel
+
+        if not predictions or not references:
+            return None
+        elif len(predictions) != len(references):
+            raise InvalidBenchmark(
+                f"The number of predictions ({len(predictions):,}) does not match the "
+                f"number of references ({len(references):,})."
+            )
+
+        # Load the judge model
+        judge_model_config = LiteLLMModel.get_model_config(
+            model_id=self.judge_id, benchmark_config=benchmark_config
+        )
+        self.judge = LiteLLMModel(
+            model_config=judge_model_config,
+            dataset_config=dataset_config,
+            benchmark_config=benchmark_config,
+            log_metadata=False,
+            **self.judge_kwargs,
+        )
+
+        # Create a cache for the judge model
+        judge_cache = ModelCache(
+            model_cache_dir=Path(judge_model_config.model_cache_dir),
+            cache_name=f"{dataset_config.name}-model-outputs.json",
+            max_generated_tokens=dataset_config.max_generated_tokens,
+        )
+        judge_cache.load()
+
+        # Prepare the messages for the LLM
+        conversations = [
+            [
+                dict(
+                    role="user",
+                    content=self._apply_user_prompt(
+                        prediction=prediction, condition=condition
+                    ),
+                )
+            ]
+            for prediction, condition in zip(predictions, references)
+        ]
+        if self.system_prompt:
+            conversations = [
+                [dict(role="system", content=self.system_prompt), *conversation]
+                for conversation in conversations
+            ]
+
+        # Get the non-cached conversations and generate the completions for them
+        non_cached_conversations = [
+            (idx, conversation)
+            for idx, conversation in enumerate(conversations)
+            if conversation not in judge_cache
+        ]
+        if non_cached_conversations:
+            model_inputs = dict(messages=[c for _, c in non_cached_conversations])
+            non_cached_outputs = self.judge.generate(inputs=model_inputs)
+
+            # Store the non-cached outputs in the cache
+            judge_cache.add_to_cache(
+                model_inputs=model_inputs, model_output=non_cached_outputs
+            )
+            judge_cache.save()
+
+        # Load all the outputs from the cache, in the original order, and parse them
+        raw_outputs = [judge_cache[conversation] for conversation in conversations]
+        json_dicts = [
+            extract_json_dict_from_string(s=output.sequence) for output in raw_outputs
+        ]
+        outputs = [
+            self.response_format.model_validate(obj=json_dict)
+            if json_dict is not None
+            else None
+            for json_dict in json_dicts
+        ]
+
+        # Calculate the scores using the scoring function
+        scores = [self.scoring_fn(output) for output in outputs]
+        if not scores:
+            logger.warning(f"No scores were calculated for {self.pretty_name}.")
+            return None
+        return sum(scores) / len(scores)
+
+    def _apply_user_prompt(self, prediction: str, condition: str | None = None) -> str:
+        """Apply the user prompt to the prediction and condition.
+
+        Args:
+            prediction:
+                The model prediction.
+            condition (optional):
+                A description of what the prediction should be judged on. If not
+                provided, it will be omitted from the prompt.
+
+        Returns:
+            The formatted user prompt with the prediction and reference.
+
+        Raises:
+            InvalidBenchmark:
+                If the user prompt requires a reference but none is provided.
+        """
+        condition_required = "{condition}" in self.user_prompt
+        if condition_required and condition is None:
+            raise InvalidBenchmark(
+                f"The user prompt for the {self.pretty_name!r} metric requires a "
+                "condition, but none was provided."
+            )
+        if condition is not None:
+            return self.user_prompt.format(
+                prediction=prediction, condition=self.condition_formatting_fn(condition)
+            )
+        return self.user_prompt.format(prediction=prediction)
+
+
+### Fluency metric ###
+
+
+class Fluency(BaseModel):
+    """Response format for the fluency metric.
+
+    Attributes:
+        fluency:
+            The fluency rating, an integer between 1 and 5.
+    """
+
+    fluency: t.Annotated[int, Field(ge=1, le=5)]
+
+
+fluency_metric = LLMAsAJudgeMetric(
+    name="fluency",
+    pretty_name="Fluency",
+    judge_id="gpt-5-2025-08-07",
+    judge_kwargs=dict(temperature=1.0),
+    user_prompt="Please rate the fluency of the following text on a scale from 1 to 5, "
+    "with the following definitions:\n"
+    "- 1: Very poor fluency, many grammatical errors\n"
+    "- 2: Poor fluency, several grammatical errors\n"
+    "- 3: Average fluency, a few grammatical errors\n"
+    "- 4: Good fluency, no grammatical errors but sounds a bit off\n"
+    "- 5: Excellent fluency, no grammatical errors and sounds natural\n\n"
+    "Text: {prediction!r}\n\n"
+    "Output your rating as a JSON object with a single key 'fluency'.",
+    response_format=Fluency,
+    scoring_fn=lambda output: (output.fluency - 1) / 4.0 if output is not None else 0.0,
+)