EuroEval 16.3.0__py3-none-any.whl → 16.5.0__py3-none-any.whl

euroeval/logging_utils.py

@@ -0,0 +1,250 @@
+"""Utility functions related to logging."""
+
+import datetime as dt
+import logging
+import os
+import sys
+import warnings
+from io import TextIOWrapper
+
+import litellm
+from datasets.utils import disable_progress_bars as disable_datasets_progress_bars
+from evaluate import disable_progress_bar as disable_evaluate_progress_bar
+from huggingface_hub.utils.tqdm import (
+    disable_progress_bars as disable_hf_hub_progress_bars,
+)
+from termcolor import colored
+from tqdm.auto import tqdm
+from transformers import logging as tf_logging
+
+from .caching_utils import cache_arguments
+
+logger = logging.getLogger("euroeval")
+
+
+def get_pbar(*tqdm_args, **tqdm_kwargs) -> tqdm:
+    """Get a progress bar for vLLM with custom hard-coded arguments.
+
+    Args:
+        *tqdm_args:
+            Positional arguments to pass to tqdm.
+        **tqdm_kwargs:
+            Additional keyword arguments to pass to tqdm.
+
+    Returns:
+        A tqdm progress bar.
+    """
+    tqdm_kwargs = dict(colour="yellow", ascii="—▰", leave=False) | tqdm_kwargs
+    tqdm_kwargs["desc"] = colored(
+        text=tqdm_kwargs.get("desc", "Processing"), color="light_yellow"
+    )
+    return tqdm(*tqdm_args, **tqdm_kwargs)
+
+
+def log(message: str, level: int, colour: str | None = None) -> None:
+    """Log a message.
+
+    Args:
+        message:
+            The message to log.
+        level:
+            The logging level. Defaults to logging.INFO.
+        colour:
+            The colour to use for the message. If None, a default colour will be used
+            based on the logging level.
+
+    Raises:
+        ValueError:
+            If the logging level is invalid.
+    """
+    match level:
+        case logging.DEBUG:
+            message = colored(
+                text=(
+                    "[DEBUG] "
+                    + dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                    + f" · {message}"
+                ),
+                color=colour or "light_blue",
+            )
+            logger.debug(message)
+        case logging.INFO:
+            if colour is not None:
+                message = colored(text=message, color=colour)
+            logger.info(message)
+        case logging.WARNING:
+            message = colored(text=message, color=colour or "light_red")
+            logger.warning(message)
+        case logging.ERROR:
+            message = colored(text=message, color=colour or "red")
+            logger.error(message)
+        case logging.CRITICAL:
+            message = colored(text=message, color=colour or "red")
+            logger.critical(message)
+        case _:
+            raise ValueError(f"Invalid logging level: {level}")
+
+
+@cache_arguments("message")
+def log_once(message: str, level: int = logging.INFO, prefix: str = "") -> None:
+    """Log a message once.
+
+    This is ensured by caching the "message" argument and only logging it the first time
+    this function is called with that message.
+
+    Args:
+        message:
+            The message to log.
+        level:
+            The logging level. Defaults to logging.INFO.
+        prefix:
+            A prefix to add to the message, which is not considered when determining if
+            the message has been logged before.
+    """
+    log(message=prefix + message, level=level)
+
+
+def block_terminal_output() -> None:
+    """Blocks libraries from writing output to the terminal.
+
+    This filters warnings from some libraries, sets the logging level to ERROR for some
+    libraries, disabled tokeniser progress bars when using Hugging Face tokenisers, and
+    disables most of the logging from the `transformers` library.
+    """
+    if os.getenv("FULL_LOG") == "1":
+        return
+
+    # Ignore miscellaneous warnings
+    warnings.filterwarnings("ignore", category=UserWarning)
+    warnings.filterwarnings("ignore", category=FutureWarning)
+    logging.getLogger("absl").setLevel(logging.CRITICAL)
+
+    # Disable matplotlib logging
+    logging.getLogger("matplotlib.font_manager").setLevel(logging.CRITICAL)
+
+    # Disable PyTorch logging
+    logging.getLogger("torch.utils.cpp_extension").setLevel(logging.CRITICAL)
+    warnings.filterwarnings(action="ignore", module="torch*")
+    os.environ["TORCH_LOGS"] = "-all"
+
+    # Disable huggingface_hub logging
+    logging.getLogger("huggingface_hub").setLevel(logging.CRITICAL)
+    disable_hf_hub_progress_bars()
+
+    # Disable LiteLLM logging
+    logging.getLogger("LiteLLM").setLevel(logging.CRITICAL)
+    logging.getLogger("LiteLLM Router").setLevel(logging.CRITICAL)
+    logging.getLogger("LiteLLM Proxy").setLevel(logging.CRITICAL)
+    logging.getLogger("openai").setLevel(logging.CRITICAL)
+    logging.getLogger("httpx").setLevel(logging.CRITICAL)
+    litellm.suppress_debug_info = True
+
+    # Disable vLLM logging
+    logging.getLogger("vllm").setLevel(logging.CRITICAL)
+    logging.getLogger("vllm.engine.llm_engine").setLevel(logging.CRITICAL)
+    logging.getLogger("vllm.transformers_utils.tokenizer").setLevel(logging.CRITICAL)
+    logging.getLogger("vllm.core.scheduler").setLevel(logging.CRITICAL)
+    logging.getLogger("vllm.model_executor.weight_utils").setLevel(logging.CRITICAL)
+    logging.getLogger("vllm.platforms").setLevel(logging.CRITICAL)
+    logging.getLogger("mistral_common.tokens.tokenizers.tekken").setLevel(
+        logging.CRITICAL
+    )
+    os.environ["LOG_LEVEL"] = "CRITICAL"
+    os.environ["VLLM_CONFIGURE_LOGGING"] = "0"
+
+    # Disable flashinfer logging
+    os.environ["FLASHINFER_LOGGING_LEVEL"] = "CRITICAL"
+
+    # Disable datasets logging
+    logging.getLogger("datasets").setLevel(logging.CRITICAL)
+    logging.getLogger("filelock").setLevel(logging.CRITICAL)
+    disable_datasets_progress_bars()
+
+    # Disable evaluate logging
+    warnings.filterwarnings("ignore", module="seqeval*")
+    disable_evaluate_progress_bar()
+
+    # Disable most of the `transformers` logging
+    tf_logging._default_log_level = logging.CRITICAL
+    tf_logging.set_verbosity(logging.CRITICAL)
+    logging.getLogger("transformers.trainer").setLevel(logging.CRITICAL)
+    logging.getLogger("accelerate").setLevel(logging.CRITICAL)
+
+
+class no_terminal_output:
+    """Context manager that suppresses all terminal output."""
+
+    def __init__(self, disable: bool = False) -> None:
+        """Initialise the context manager.
+
+        Args:
+            disable:
+                If True, this context manager does nothing.
+        """
+        self.disable = disable
+        self.nothing_file: TextIOWrapper | None = None
+        self._cpp_stdout_file: int | None = None
+        self._cpp_stderr_file: int | None = None
+        try:
+            self._cpp_stdout_file = os.dup(sys.stdout.fileno())
+            self._cpp_stderr_file = os.dup(sys.stderr.fileno())
+        except OSError:
+            self._log_windows_warning()
+
+    def _log_windows_warning(self) -> None:
+        """Log a warning about Windows not supporting blocking terminal output."""
+        log_once(
+            "Your operating system (probably Windows) does not support blocking "
+            "terminal output, so expect more messy output - sorry!",
+            level=logging.WARNING,
+        )
+
+    def __enter__(self) -> None:
+        """Suppress all terminal output."""
+        if not self.disable:
+            self.nothing_file = open(os.devnull, "w")
+            try:
+                os.dup2(fd=self.nothing_file.fileno(), fd2=sys.stdout.fileno())
+                os.dup2(fd=self.nothing_file.fileno(), fd2=sys.stderr.fileno())
+            except OSError:
+                self._log_windows_warning()
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: type[BaseException] | None,
+    ) -> None:
+        """Re-enable terminal output."""
+        if not self.disable:
+            if self.nothing_file is not None:
+                self.nothing_file.close()
+            try:
+                if self._cpp_stdout_file is not None:
+                    os.dup2(fd=self._cpp_stdout_file, fd2=sys.stdout.fileno())
+                if self._cpp_stderr_file is not None:
+                    os.dup2(fd=self._cpp_stderr_file, fd2=sys.stderr.fileno())
+            except OSError:
+                self._log_windows_warning()
+
+
+def adjust_logging_level(verbose: bool, ignore_testing: bool = False) -> int:
+    """Adjust the logging level based on verbosity.
+
+    Args:
+        verbose:
+            Whether to output additional output.
+        ignore_testing:
+            Whether to ignore the testing flag.
+
+    Returns:
+        The logging level that was set.
+    """
+    if hasattr(sys, "_called_from_test") and not ignore_testing:
+        logging_level = logging.CRITICAL
+    elif verbose:
+        logging_level = logging.DEBUG
+    else:
+        logging_level = logging.INFO
+    logger.setLevel(logging_level)
+    return logging_level

euroeval/metrics/base.py

@@ -2,7 +2,6 @@
 
 import abc
 import collections.abc as c
-import logging
 import typing as t
 
 if t.TYPE_CHECKING:
@@ -10,8 +9,6 @@ if t.TYPE_CHECKING:
 
     from ..data_models import BenchmarkConfig, DatasetConfig
 
-logger: logging.Logger = logging.getLogger("euroeval")
-
 
 class Metric(abc.ABC):
     """Abstract base class for all metrics."""

euroeval/metrics/huggingface.py

@@ -1,7 +1,6 @@
 """All the Hugging Face metrics used in EuroEval."""
 
 import collections.abc as c
-import logging
 import typing as t
 from pathlib import Path
 
@@ -9,7 +8,7 @@ import evaluate
 import numpy as np
 from datasets import DownloadConfig
 
-from ..utils import HiddenPrints
+from ..logging_utils import no_terminal_output
 from .base import Metric
 
 if t.TYPE_CHECKING:
@@ -18,8 +17,6 @@ if t.TYPE_CHECKING:
 
     from ..data_models import BenchmarkConfig, DatasetConfig
 
-logger: logging.Logger = logging.getLogger("euroeval")
-
 
 class HuggingFaceMetric(Metric):
     """A metric which is implemented in the `evaluate` package.
@@ -124,9 +121,12 @@ class HuggingFaceMetric(Metric):
         if self.metric is None:
             self.download(cache_dir=benchmark_config.cache_dir)
 
-        assert self.metric is not None
+        assert self.metric is not None, (
+            "Metric has not been downloaded. Please call download() before using the "
+            "__call__ method."
+        )
 
-        with HiddenPrints():
+        with no_terminal_output(disable=benchmark_config.verbose):
             results = self.metric.compute(
                 predictions=predictions, references=references, **self.compute_kwargs
             )
@@ -143,8 +143,23 @@ class HuggingFaceMetric(Metric):
         if isinstance(score, np.floating):
             score = float(score)
 
+        self.close()
         return score
 
+    def close(self) -> None:
+        """Close any resources held by the metric."""
+        if self.metric is not None:
+            if self.metric.filelock is not None:
+                self.metric.filelock.release(force=True)
+            if self.metric.writer is not None:
+                self.metric.writer.finalize(close_stream=True)
+
+    def __del__(self) -> None:
+        """Clean up the metric from memory."""
+        if self.metric is not None:
+            self.close()
+            del self.metric
+
 
 mcc_metric = HuggingFaceMetric(
     name="mcc",

euroeval/metrics/llm_as_a_judge.py

@@ -8,7 +8,7 @@ from pathlib import Path
 from pydantic import BaseModel, Field
 
 from ..exceptions import InvalidBenchmark
-from ..model_cache import ModelCache
+from ..logging_utils import log
 from ..utils import extract_json_dict_from_string
 from .base import Metric
 
@@ -17,8 +17,6 @@ if t.TYPE_CHECKING:
 
     from ..data_models import BenchmarkConfig, DatasetConfig
 
-logger: logging.Logger = logging.getLogger("euroeval")
-
 
 class LLMAsAJudgeMetric(Metric):
     """Use an LLM to judge the quality of the predictions."""
@@ -112,6 +110,7 @@ class LLMAsAJudgeMetric(Metric):
         """
         # Importing here to avoid circular imports
         from ..benchmark_modules import LiteLLMModel
+        from ..model_cache import ModelCache
 
         if not predictions or not references:
             return None
@@ -190,7 +189,10 @@ class LLMAsAJudgeMetric(Metric):
         # Calculate the scores using the scoring function
         scores = [self.scoring_fn(output) for output in outputs]
         if not scores:
-            logger.warning(f"No scores were calculated for {self.pretty_name}.")
+            log(
+                f"No scores were calculated for {self.pretty_name}.",
+                level=logging.WARNING,
+            )
             return None
         return sum(scores) / len(scores)
 

euroeval/metrics/pipeline.py

@@ -11,6 +11,7 @@ import numpy as np
 from scipy.special import expit as sigmoid
 
 from ..exceptions import InvalidBenchmark
+from ..logging_utils import log, no_terminal_output
 from ..utils import unscramble
 from .base import Metric
 
@@ -20,8 +21,6 @@ if t.TYPE_CHECKING:
 
     from ..data_models import BenchmarkConfig, DatasetConfig
 
-logger: logging.Logger = logging.getLogger("euroeval")
-
 
 T = t.TypeVar("T", bound=int | float | str | bool)
 
@@ -121,16 +120,22 @@ class PipelineMetric(Metric):
             The calculated metric score, or None if the score should be ignored.
         """
         if self.pipeline is None:
-            self.pipeline = self._download_pipeline()
+            self.pipeline = self._download_pipeline(
+                cache_dir=benchmark_config.cache_dir
+            )
         if self.preprocessing_fn is not None:
             predictions = self.preprocessing_fn(
                 predictions=predictions, dataset=dataset
             )
         return self.pipeline_scoring_function(self.pipeline, predictions)
 
-    def _download_pipeline(self) -> "Pipeline":
+    def _download_pipeline(self, cache_dir: str) -> "Pipeline":
         """Download the scikit-learn pipeline from the given URL.
 
+        Args:
+            cache_dir:
+                The directory to use for caching the downloaded pipeline.
+
         Returns:
             The downloaded scikit-learn pipeline.
 
@@ -138,10 +143,13 @@ class PipelineMetric(Metric):
             InvalidBenchmark:
                 If the loading of the pipeline fails for any reason.
         """
-        logger.debug(f"Loading pipeline from {self.pipeline_repo}...")
-        folder_path = hf_hub.HfApi(
-            token=unscramble("HjccJFhIozVymqXDVqTUTXKvYhZMTbfIjMxG_")
-        ).snapshot_download(repo_id=self.pipeline_repo, repo_type="model")
+        log(f"Loading pipeline from {self.pipeline_repo}...", level=logging.DEBUG)
+        with no_terminal_output():
+            folder_path = hf_hub.HfApi(
+                token=unscramble("XbjeOLhwebEaSaDUMqqaPaPIhgOcyOfDpGnX_")
+            ).snapshot_download(
+                repo_id=self.pipeline_repo, repo_type="model", cache_dir=cache_dir
+            )
         model_path = Path(folder_path, self.pipeline_file_name)
         try:
             with model_path.open(mode="rb") as f:
@@ -150,7 +158,7 @@ class PipelineMetric(Metric):
             raise InvalidBenchmark(
                 f"Failed to load pipeline from {self.pipeline_repo!r}: {e}"
             ) from e
-        logger.debug(f"Successfully loaded pipeline: {pipeline}")
+        log(f"Successfully loaded pipeline: {pipeline}", level=logging.DEBUG)
         return pipeline
 
 

euroeval/metrics/speed.py

@@ -1,7 +1,6 @@
 """Inference speed metric."""
 
 import collections.abc as c
-import logging
 import typing as t
 
 from .base import Metric
@@ -11,8 +10,6 @@ if t.TYPE_CHECKING:
 
     from ..data_models import BenchmarkConfig, DatasetConfig
 
-logger: logging.Logger = logging.getLogger("euroeval")
-
 
 class SpeedMetric(Metric):
     """Speed metric."""

euroeval/model_cache.py

@@ -1,5 +1,6 @@
 """ModelCache class for caching model outputs."""
 
+import collections.abc as c
 import hashlib
 import json
 import logging
@@ -8,11 +9,9 @@ import typing as t
 from collections import defaultdict
 from dataclasses import asdict
 
-from tqdm.auto import tqdm
-
 from .constants import NUM_GENERATION_TOKENS_FOR_CLASSIFICATION
 from .data_models import GenerativeModelOutput, SingleGenerativeModelOutput
-from .utils import log_once
+from .logging_utils import get_pbar, log, log_once
 
 if t.TYPE_CHECKING:
     from pathlib import Path
@@ -20,9 +19,6 @@ if t.TYPE_CHECKING:
     from datasets import Dataset
 
 
-logger = logging.getLogger("euroeval")
-
-
 class ModelCache:
     """A cache for model outputs.
 
@@ -65,9 +61,10 @@ class ModelCache:
             with self.cache_path.open() as f:
                 json_cache = json.load(f)
         except json.JSONDecodeError:
-            logger.warning(
+            log(
                 f"Failed to load the cache from {self.cache_path}. The cache will be "
-                f"re-initialised."
+                f"re-initialised.",
+                level=logging.WARNING,
             )
             json_cache = dict()
             with self.cache_path.open("w") as f:
@@ -89,15 +86,16 @@ class ModelCache:
             with self.cache_path.open("w") as f:
                 json.dump(dumpable_cache, f)
         except KeyError:
-            logger.warning(
+            log(
                 f"Failed to load the cache from {self.cache_path}. The cache will be "
-                f"re-initialised."
+                f"re-initialised.",
+                level=logging.WARNING,
             )
             self.cache = dict()
             with self.cache_path.open("w") as f:
                 json.dump(dict(), f)
 
-    def _hash_key(self, key: str | list[dict[str, str]]) -> str:
+    def _hash_key(self, key: str | c.Sequence[dict[str, str]]) -> str:
         """Hash the key to use as an index in the cache.
 
         Args:
@@ -110,7 +108,7 @@ class ModelCache:
         return hashlib.md5(string=str(key).encode()).hexdigest()
 
     def __getitem__(
-        self, key: str | list[dict[str, str]]
+        self, key: str | c.Sequence[dict[str, str]]
     ) -> SingleGenerativeModelOutput:
         """Get an item from the cache.
 
@@ -125,7 +123,7 @@ class ModelCache:
         return self.cache[hashed_key]
 
     def __setitem__(
-        self, key: str | list[dict[str, str]], value: SingleGenerativeModelOutput
+        self, key: str | c.Sequence[dict[str, str]], value: SingleGenerativeModelOutput
     ) -> None:
         """Set an item in the cache.
 
@@ -143,7 +141,7 @@ class ModelCache:
         self.cache_path.unlink()
         del self.cache
 
-    def __contains__(self, key: str | list[dict[str, str]]) -> bool:
+    def __contains__(self, key: str | c.Sequence[dict[str, str]]) -> bool:
         """Check if a key is in the cache.
 
         Args:
@@ -172,18 +170,18 @@ class ModelCache:
 
         # Double check that the number of inputs and outputs match
         if not len(model_inputs) == len(model_output.sequences):
-            logger.warning(
+            log(
                 f"Number of model inputs ({len(model_inputs)}) does not match the "
                 f"number of model outputs ({len(model_output.sequences)}). We will not "
-                f"cache the model outputs."
+                f"cache the model outputs.",
+                level=logging.WARNING,
             )
             return
 
         # Store the generated sequences in the cache, one by one
-        with tqdm(
+        with get_pbar(
             iterable=model_inputs,
             desc="Caching model outputs",
-            leave=False,
             disable=hasattr(sys, "_called_from_test"),
         ) as pbar:
             for sample_idx, model_input in enumerate(pbar):
@@ -261,7 +259,7 @@ def load_cached_model_outputs(
         The model output containing the cached sequences.
     """
     input_column = "messages" if "messages" in cached_dataset.column_names else "text"
-    cached_model_outputs: list[SingleGenerativeModelOutput] = [
+    cached_model_outputs: c.Sequence[SingleGenerativeModelOutput] = [
         cache[prompt] for prompt in cached_dataset[input_column]
     ]
 

euroeval/model_config.py

@@ -5,14 +5,12 @@ import typing as t
 
 from . import benchmark_modules
 from .exceptions import InvalidModel, NeedsEnvironmentVariable, NeedsExtraInstalled
+from .logging_utils import log
 
 if t.TYPE_CHECKING:
     from .data_models import BenchmarkConfig, ModelConfig
 
 
-logger = logging.getLogger("euroeval")
-
-
 def get_model_config(
     model_id: str, benchmark_config: "BenchmarkConfig"
 ) -> "ModelConfig":
@@ -51,9 +49,10 @@ def get_model_config(
         elif isinstance(exists_or_err, NeedsEnvironmentVariable):
             needs_env_vars.append(exists_or_err.env_var)
         elif exists_or_err is True:
-            logger.debug(
+            log(
                 f"The model {model_id!r} was identified by the "
-                f"{benchmark_module.__name__} benchmark module."
+                f"{benchmark_module.__name__} benchmark module.",
+                logging.DEBUG,
             )
             model_config = benchmark_module.get_model_config(
                 model_id=model_id, benchmark_config=benchmark_config

euroeval/model_loading.py

@@ -10,6 +10,7 @@ from .benchmark_modules import (
 )
 from .enums import InferenceBackend, ModelType
 from .exceptions import InvalidModel
+from .logging_utils import log_once
 
 if t.TYPE_CHECKING:
     from .benchmark_modules import BenchmarkModule
@@ -34,6 +35,8 @@ def load_model(
     Returns:
         The model.
     """
+    log_once(f"\nLoading the model {model_config.model_id}...")
+
     # The order matters; the first model type that matches will be used. For this
     # reason, they have been ordered in terms of the most common model types.
     model_class: t.Type[BenchmarkModule]

euroeval/prompt_templates/__init__.py

@@ -1,8 +1,10 @@
 """The different prompt templates used in EuroEval."""
 
+from .classification import CLASSIFICATION_TEMPLATES
 from .linguistic_acceptability import LA_TEMPLATES
 from .multiple_choice import MULTIPLE_CHOICE_TEMPLATES
 from .named_entity_recognition import NER_TEMPLATES
 from .reading_comprehension import RC_TEMPLATES
 from .sentiment_classification import SENT_TEMPLATES
 from .summarization import SUMM_TEMPLATES
+from .token_classification import TOKEN_CLASSIFICATION_TEMPLATES