EuroEval 15.12.0__py3-none-any.whl → 16.7.1__py3-none-any.whl

euroeval/logging_utils.py

@@ -0,0 +1,268 @@
+"""Utility functions related to logging."""
+
+import datetime as dt
+import logging
+import os
+import sys
+import warnings
+from io import TextIOWrapper
+
+import litellm
+from datasets.utils import disable_progress_bars as disable_datasets_progress_bars
+from evaluate import disable_progress_bar as disable_evaluate_progress_bar
+from huggingface_hub.utils.tqdm import (
+    disable_progress_bars as disable_hf_hub_progress_bars,
+)
+from termcolor import colored
+from tqdm.auto import tqdm
+from transformers import logging as tf_logging
+
+from .caching_utils import cache_arguments
+
+logger = logging.getLogger("euroeval")
+
+
+def get_pbar(*tqdm_args, **tqdm_kwargs) -> tqdm:
+    """Get a progress bar for vLLM with custom hard-coded arguments.
+
+    Args:
+        *tqdm_args:
+            Positional arguments to pass to tqdm.
+        **tqdm_kwargs:
+            Additional keyword arguments to pass to tqdm.
+
+    Returns:
+        A tqdm progress bar.
+    """
+    tqdm_kwargs = dict(colour="yellow", ascii="—▰", leave=False) | tqdm_kwargs
+    tqdm_kwargs["desc"] = colored(
+        text=tqdm_kwargs.get("desc", "Processing"), color="light_yellow"
+    )
+    return tqdm(*tqdm_args, **tqdm_kwargs)
+
+
+def log(message: str, level: int, colour: str | None = None) -> None:
+    """Log a message.
+
+    Args:
+        message:
+            The message to log.
+        level:
+            The logging level. Defaults to logging.INFO.
+        colour:
+            The colour to use for the message. If None, a default colour will be used
+            based on the logging level.
+
+    Raises:
+        ValueError:
+            If the logging level is invalid.
+    """
+    match level:
+        case logging.DEBUG:
+            message = colored(
+                text=(
+                    "[DEBUG] "
+                    + dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                    + f" · {message}"
+                ),
+                color=colour or "light_blue",
+            )
+            logger.debug(message)
+        case logging.INFO:
+            if colour is not None:
+                message = colored(text=message, color=colour)
+            logger.info(message)
+        case logging.WARNING:
+            message = colored(text=message, color=colour or "light_red")
+            logger.warning(message)
+        case logging.ERROR:
+            message = colored(text=message, color=colour or "red")
+            logger.error(message)
+        case logging.CRITICAL:
+            message = colored(text=message, color=colour or "red")
+            logger.critical(message)
+        case _:
+            raise ValueError(f"Invalid logging level: {level}")
+
+
+@cache_arguments("message")
+def log_once(message: str, level: int = logging.INFO, prefix: str = "") -> None:
+    """Log a message once.
+
+    This is ensured by caching the "message" argument and only logging it the first time
+    this function is called with that message.
+
+    Args:
+        message:
+            The message to log.
+        level:
+            The logging level. Defaults to logging.INFO.
+        prefix:
+            A prefix to add to the message, which is not considered when determining if
+            the message has been logged before.
+    """
+    log(message=prefix + message, level=level)
+
+
+def block_terminal_output() -> None:
+    """Blocks libraries from writing output to the terminal.
+
+    This filters warnings from some libraries, sets the logging level to ERROR for some
+    libraries, disabled tokeniser progress bars when using Hugging Face tokenisers, and
+    disables most of the logging from the `transformers` library.
+    """
+    if os.getenv("FULL_LOG") == "1":
+        return
+
+    # Ignore miscellaneous warnings
+    warnings.filterwarnings("ignore", category=UserWarning)
+    warnings.filterwarnings("ignore", category=FutureWarning)
+    logging.getLogger("absl").setLevel(logging.CRITICAL)
+
+    # Disable matplotlib logging
+    logging.getLogger("matplotlib.font_manager").setLevel(logging.CRITICAL)
+
+    # Disable PyTorch logging
+    logging.getLogger("torch.utils.cpp_extension").setLevel(logging.CRITICAL)
+    warnings.filterwarnings(action="ignore", module="torch*")
+    os.environ["TORCH_LOGS"] = "-all"
+
+    # Disable huggingface_hub logging
+    logging.getLogger("huggingface_hub").setLevel(logging.CRITICAL)
+    disable_hf_hub_progress_bars()
+
+    # Disable LiteLLM logging
+    logging.getLogger("LiteLLM").setLevel(logging.CRITICAL)
+    logging.getLogger("LiteLLM Router").setLevel(logging.CRITICAL)
+    logging.getLogger("LiteLLM Proxy").setLevel(logging.CRITICAL)
+    logging.getLogger("openai").setLevel(logging.CRITICAL)
+    logging.getLogger("httpx").setLevel(logging.CRITICAL)
+    litellm.suppress_debug_info = True
+
+    # Disable vLLM logging
+    logging.getLogger("vllm").setLevel(logging.CRITICAL)
+    logging.getLogger("vllm.engine.llm_engine").setLevel(logging.CRITICAL)
+    logging.getLogger("vllm.transformers_utils.tokenizer").setLevel(logging.CRITICAL)
+    logging.getLogger("vllm.core.scheduler").setLevel(logging.CRITICAL)
+    logging.getLogger("vllm.model_executor.weight_utils").setLevel(logging.CRITICAL)
+    logging.getLogger("vllm.platforms").setLevel(logging.CRITICAL)
+    logging.getLogger("mistral_common.tokens.tokenizers.tekken").setLevel(
+        logging.CRITICAL
+    )
+    os.environ["LOG_LEVEL"] = "CRITICAL"
+    os.environ["VLLM_CONFIGURE_LOGGING"] = "0"
+
+    # Disable flashinfer logging
+    os.environ["FLASHINFER_LOGGING_LEVEL"] = "CRITICAL"
+
+    # Disable datasets logging
+    logging.getLogger("datasets").setLevel(logging.CRITICAL)
+    logging.getLogger("filelock").setLevel(logging.CRITICAL)
+    disable_datasets_progress_bars()
+
+    # Disable evaluate logging
+    warnings.filterwarnings("ignore", module="seqeval*")
+    disable_evaluate_progress_bar()
+
+    # Disable most of the `transformers` logging
+    tf_logging._default_log_level = logging.CRITICAL
+    tf_logging.set_verbosity(logging.CRITICAL)
+    logging.getLogger("transformers.trainer").setLevel(logging.CRITICAL)
+    logging.getLogger("accelerate").setLevel(logging.CRITICAL)
+
+
+class no_terminal_output:
+    """Context manager that suppresses all terminal output."""
+
+    def __init__(self, disable: bool = False) -> None:
+        """Initialise the context manager.
+
+        Args:
+            disable:
+                If True, this context manager does nothing.
+        """
+        self.disable = disable
+        self.devnull_file: TextIOWrapper | None = None
+        self._original_stdout_fd: int | None = None
+        self._original_stderr_fd: int | None = None
+
+    def _log_windows_warning(self) -> None:
+        """Log a warning about Windows not supporting blocking terminal output."""
+        log_once(
+            "Your operating system (probably Windows) does not support blocking "
+            "terminal output, so expect more messy output - sorry!",
+            level=logging.WARNING,
+        )
+
+    def __enter__(self) -> None:
+        """Suppress all terminal output."""
+        if self.disable:
+            return
+
+        try:
+            # Save original FDs by duplicating them
+            self._original_stdout_fd = os.dup(sys.stdout.fileno())
+            self._original_stderr_fd = os.dup(sys.stderr.fileno())
+
+            # Open /dev/null
+            self.devnull_file = open(os.devnull, "w")
+
+            # Redirect stdout/stderr to /dev/null
+            os.dup2(self.devnull_file.fileno(), sys.stdout.fileno())
+            os.dup2(self.devnull_file.fileno(), sys.stderr.fileno())
+
+        except OSError:
+            self._log_windows_warning()
+            # If setup fails, clean up any resources we might have acquired
+            self.__exit__(None, None, None)
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: type[BaseException] | None,
+    ) -> None:
+        """Re-enable terminal output."""
+        if self.disable:
+            return
+
+        # Restore stdout/stderr from our saved FDs
+        try:
+            if self._original_stdout_fd is not None:
+                os.dup2(self._original_stdout_fd, sys.stdout.fileno())
+            if self._original_stderr_fd is not None:
+                os.dup2(self._original_stderr_fd, sys.stderr.fileno())
+        except OSError:
+            self._log_windows_warning()
+        finally:
+            # Close the duplicated FDs we created
+            if self._original_stdout_fd is not None:
+                os.close(self._original_stdout_fd)
+            if self._original_stderr_fd is not None:
+                os.close(self._original_stderr_fd)
+
+            # Close the /dev/null file
+            if self.devnull_file is not None:
+                self.devnull_file.close()
+
+
+def adjust_logging_level(verbose: bool, ignore_testing: bool = False) -> int:
+    """Adjust the logging level based on verbosity.
+
+    Args:
+        verbose:
+            Whether to output additional output.
+        ignore_testing:
+            Whether to ignore the testing flag.
+
+    Returns:
+        The logging level that was set.
+    """
+    if hasattr(sys, "_called_from_test") and not ignore_testing:
+        logging_level = logging.CRITICAL
+    elif verbose:
+        logging_level = logging.DEBUG
+    else:
+        logging_level = logging.INFO
+    logger.setLevel(logging_level)
+    return logging_level

euroeval/metrics/__init__.py

@@ -0,0 +1,6 @@
+"""All the metrics used in EuroEval."""
+
+from .huggingface import *  # noqa: F403
+from .llm_as_a_judge import *  # noqa: F403
+from .pipeline import *  # noqa: F403
+from .speed import *  # noqa: F403

euroeval/metrics/base.py

@@ -0,0 +1,85 @@
+"""The abstract base class for all metrics."""
+
+import abc
+import collections.abc as c
+import typing as t
+
+if t.TYPE_CHECKING:
+    from datasets.arrow_dataset import Dataset
+
+    from ..data_models import BenchmarkConfig, DatasetConfig
+
+
+class Metric(abc.ABC):
+    """Abstract base class for all metrics."""
+
+    def __init__(
+        self,
+        name: str,
+        pretty_name: str,
+        postprocessing_fn: t.Callable[[float], tuple[float, str]] | None = None,
+    ) -> None:
+        """Initialise the metric.
+
+        Args:
+            name:
+                The name of the metric in snake_case.
+            pretty_name:
+                The pretty name of the metric, used for display purposes.
+            postprocessing_fn:
+                A function to apply to the metric scores after they are computed,
+                taking the score to the postprocessed score along with its string
+                representation. Defaults to x -> (100 * x, f"{x:.2%}").
+        """
+        self.name = name
+        self.pretty_name = pretty_name
+        self.postprocessing_fn = (
+            postprocessing_fn
+            if postprocessing_fn is not None
+            else lambda x: (100 * x, f"{x:.2%}")
+        )
+
+    def download(self, cache_dir: str) -> "Metric":
+        """Initiates the download of the metric if needed.
+
+        Args:
+            cache_dir:
+                The directory where the metric will be downloaded to.
+
+        Returns:
+            The metric object itself.
+        """
+        return self
+
+    @abc.abstractmethod
+    def __call__(
+        self,
+        predictions: c.Sequence,
+        references: c.Sequence,
+        dataset: "Dataset",
+        dataset_config: "DatasetConfig",
+        benchmark_config: "BenchmarkConfig",
+    ) -> float | None:
+        """Calculate the metric score.
+
+        Args:
+            predictions:
+                The model predictions.
+            references:
+                The ground truth references.
+            dataset:
+                The dataset used for evaluation. This is only used in case any
+                additional metadata is used to compute the metrics.
+            dataset_config:
+                The dataset configuration.
+            benchmark_config:
+                The benchmark configuration.
+
+        Returns:
+            The calculated metric score, or None if the score should be ignored.
+        """
+        ...
+
+    def __hash__(self) -> int:
+        """Return a hash of the metric configuration."""
+        return hash(self.name)

euroeval/metrics/huggingface.py

@@ -0,0 +1,216 @@
+"""All the Hugging Face metrics used in EuroEval."""
+
+import collections.abc as c
+import typing as t
+from pathlib import Path
+
+import evaluate
+import numpy as np
+from datasets import DownloadConfig, DownloadMode
+
+from ..logging_utils import no_terminal_output
+from .base import Metric
+
+if t.TYPE_CHECKING:
+    from datasets.arrow_dataset import Dataset
+    from evaluate import EvaluationModule
+
+    from ..data_models import BenchmarkConfig, DatasetConfig
+
+
+class HuggingFaceMetric(Metric):
+    """A metric which is implemented in the `evaluate` package.
+
+    Attributes:
+        name:
+            The name of the metric in snake_case.
+        pretty_name:
+            The pretty name of the metric, used for display purposes.
+        huggingface_id:
+            The Hugging Face ID of the metric.
+        results_key:
+            The name of the key used to extract the metric scores from the results
+            dictionary.
+        compute_kwargs:
+            Keyword arguments to pass to the metric's compute function. Defaults to
+            an empty dictionary.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        pretty_name: str,
+        huggingface_id: str,
+        results_key: str,
+        compute_kwargs: dict[str, t.Any] | None = None,
+        postprocessing_fn: t.Callable[[float], tuple[float, str]] | None = None,
+    ) -> None:
+        """Initialise the Hugging Face metric.
+
+        Args:
+            name:
+                The name of the metric in snake_case.
+            pretty_name:
+                The pretty name of the metric, used for display purposes.
+            huggingface_id:
+                The Hugging Face ID of the metric.
+            results_key:
+                The name of the key used to extract the metric scores from the results
+                dictionary.
+            compute_kwargs:
+                Keyword arguments to pass to the metric's compute function. Defaults to
+                an empty dictionary.
+            postprocessing_fn:
+                A function to apply to the metric scores after they are computed, taking
+                the score to the postprocessed score along with its string
+                representation. Defaults to x -> (100 * x, f"{x:.2%}").
+        """
+        super().__init__(
+            name=name, pretty_name=pretty_name, postprocessing_fn=postprocessing_fn
+        )
+        self.huggingface_id = huggingface_id
+        self.results_key = results_key
+        self.compute_kwargs: dict[str, t.Any] = (
+            dict() if compute_kwargs is None else compute_kwargs
+        )
+        self.metric: "EvaluationModule | None" = None
+
+    def download(self, cache_dir: str) -> "HuggingFaceMetric":
+        """Initiates the download of the metric if needed.
+
+        Args:
+            cache_dir:
+                The directory where the metric will be downloaded to.
+
+        Returns:
+            The metric object itself.
+        """
+        metric_cache_dir = Path(cache_dir) / "metrics"
+        download_config = DownloadConfig(cache_dir=metric_cache_dir)
+        self.metric = evaluate.load(
+            path=self.huggingface_id,
+            download_config=download_config,
+            download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS,
+            cache_dir=metric_cache_dir.as_posix(),
+        )
+        return self
+
+    def __call__(
+        self,
+        predictions: c.Sequence,
+        references: c.Sequence,
+        dataset: "Dataset",
+        dataset_config: "DatasetConfig",
+        benchmark_config: "BenchmarkConfig",
+    ) -> float | None:
+        """Calculate the metric score.
+
+        Args:
+            predictions:
+                The model predictions.
+            references:
+                The ground truth references.
+            dataset:
+                The dataset used for evaluation. This is only used in case any
+                additional metadata is used to compute the metrics.
+            dataset_config:
+                The dataset configuration.
+            benchmark_config:
+                The benchmark configuration.
+
+        Returns:
+            The calculated metric score, or None if the score should be ignored.
+        """
+        if self.metric is None:
+            self.download(cache_dir=benchmark_config.cache_dir)
+
+        assert self.metric is not None, (
+            "Metric has not been downloaded. Please call download() before using the "
+            "__call__ method."
+        )
+
+        with no_terminal_output(disable=benchmark_config.verbose):
+            results = self.metric.compute(
+                predictions=predictions, references=references, **self.compute_kwargs
+            )
+
+        # The metric returns None if we are running on multi-GPU and the current
+        # process is not the main process
+        if results is None:
+            return None
+
+        # Convert the results to a float score
+        score = results[self.results_key]
+        if isinstance(score, list):
+            score = sum(score) / len(score)
+        if isinstance(score, np.floating):
+            score = float(score)
+
+        return score
+
+
+mcc_metric = HuggingFaceMetric(
+    name="mcc",
+    pretty_name="Matthew's Correlation Coefficient",
+    huggingface_id="matthews_correlation",
+    results_key="matthews_correlation",
+)
+
+macro_f1_metric = HuggingFaceMetric(
+    name="macro_f1",
+    pretty_name="Macro-average F1-score",
+    huggingface_id="f1",
+    results_key="f1",
+    compute_kwargs=dict(average="macro"),
+)
+
+micro_f1_metric = HuggingFaceMetric(
+    name="micro_f1",
+    pretty_name="Micro-average F1-score with MISC tags",
+    huggingface_id="seqeval",
+    results_key="overall_f1",
+)
+
+micro_f1_no_misc_metric = HuggingFaceMetric(
+    name="micro_f1_no_misc",
+    pretty_name="Micro-average F1-score without MISC tags",
+    huggingface_id="seqeval",
+    results_key="overall_f1",
+)
+
+f1_metric = HuggingFaceMetric(
+    name="f1",
+    pretty_name="F1-score",
+    huggingface_id="squad_v2",
+    results_key="f1",
+    postprocessing_fn=lambda x: (x, f"{x:.2f}%"),
+)
+
+em_metric = HuggingFaceMetric(
+    name="em",
+    pretty_name="Exact Match",
+    huggingface_id="squad_v2",
+    results_key="exact",
+    postprocessing_fn=lambda x: (x, f"{x:.2f}%"),
+)
+
+bert_score_metric = HuggingFaceMetric(
+    name="bertscore",
+    pretty_name="BERTScore",
+    huggingface_id="bertscore",
+    results_key="f1",
+    compute_kwargs=dict(
+        model_type="microsoft/mdeberta-v3-base", device="auto", batch_size=1
+    ),
+)
+
+rouge_l_metric = HuggingFaceMetric(
+    name="rouge_l", pretty_name="ROUGE-L", huggingface_id="rouge", results_key="rougeL"
+)
+
+accuracy_metric = HuggingFaceMetric(
+    name="accuracy",
+    pretty_name="Accuracy",
+    huggingface_id="accuracy",
+    results_key="accuracy",
+)