EuroEval 16.3.0__py3-none-any.whl → 16.4.0__py3-none-any.whl

euroeval/logging_utils.py

@@ -0,0 +1,250 @@
+"""Utility functions related to logging."""
+
+import datetime as dt
+import logging
+import os
+import sys
+import warnings
+from io import TextIOWrapper
+
+import litellm
+from datasets.utils import disable_progress_bars as disable_datasets_progress_bars
+from evaluate import disable_progress_bar as disable_evaluate_progress_bar
+from huggingface_hub.utils.tqdm import (
+    disable_progress_bars as disable_hf_hub_progress_bars,
+)
+from termcolor import colored
+from tqdm.auto import tqdm
+from transformers import logging as tf_logging
+
+from .caching_utils import cache_arguments
+
+logger = logging.getLogger("euroeval")
+
+
+def get_pbar(*tqdm_args, **tqdm_kwargs) -> tqdm:
+    """Get a progress bar for vLLM with custom hard-coded arguments.
+
+    Args:
+        *tqdm_args:
+            Positional arguments to pass to tqdm.
+        **tqdm_kwargs:
+            Additional keyword arguments to pass to tqdm.
+
+    Returns:
+        A tqdm progress bar.
+    """
+    tqdm_kwargs = dict(colour="yellow", ascii="—▰", leave=False) | tqdm_kwargs
+    tqdm_kwargs["desc"] = colored(
+        text=tqdm_kwargs.get("desc", "Processing"), color="light_yellow"
+    )
+    return tqdm(*tqdm_args, **tqdm_kwargs)
+
+
+def log(message: str, level: int, colour: str | None = None) -> None:
+    """Log a message.
+
+    Args:
+        message:
+            The message to log.
+        level:
+            The logging level. Defaults to logging.INFO.
+        colour:
+            The colour to use for the message. If None, a default colour will be used
+            based on the logging level.
+
+    Raises:
+        ValueError:
+            If the logging level is invalid.
+    """
+    match level:
+        case logging.DEBUG:
+            message = colored(
+                text=(
+                    "[DEBUG] "
+                    + dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                    + f" · {message}"
+                ),
+                color=colour or "light_blue",
+            )
+            logger.debug(message)
+        case logging.INFO:
+            if colour is not None:
+                message = colored(text=message, color=colour)
+            logger.info(message)
+        case logging.WARNING:
+            message = colored(text=message, color=colour or "light_red")
+            logger.warning(message)
+        case logging.ERROR:
+            message = colored(text=message, color=colour or "red")
+            logger.error(message)
+        case logging.CRITICAL:
+            message = colored(text=message, color=colour or "red")
+            logger.critical(message)
+        case _:
+            raise ValueError(f"Invalid logging level: {level}")
+
+
+@cache_arguments("message")
+def log_once(message: str, level: int = logging.INFO, prefix: str = "") -> None:
+    """Log a message once.
+
+    This is ensured by caching the "message" argument and only logging it the first time
+    this function is called with that message.
+
+    Args:
+        message:
+            The message to log.
+        level:
+            The logging level. Defaults to logging.INFO.
+        prefix:
+            A prefix to add to the message, which is not considered when determining if
+            the message has been logged before.
+    """
+    log(message=prefix + message, level=level)
+
+
+def block_terminal_output() -> None:
+    """Blocks libraries from writing output to the terminal.
+
+    This filters warnings from some libraries, sets the logging level to ERROR for some
+    libraries, disabled tokeniser progress bars when using Hugging Face tokenisers, and
+    disables most of the logging from the `transformers` library.
+    """
+    if os.getenv("FULL_LOG") == "1":
+        return
+
+    # Ignore miscellaneous warnings
+    warnings.filterwarnings("ignore", category=UserWarning)
+    warnings.filterwarnings("ignore", category=FutureWarning)
+    logging.getLogger("absl").setLevel(logging.CRITICAL)
+
+    # Disable matplotlib logging
+    logging.getLogger("matplotlib.font_manager").setLevel(logging.CRITICAL)
+
+    # Disable PyTorch logging
+    logging.getLogger("torch.utils.cpp_extension").setLevel(logging.CRITICAL)
+    warnings.filterwarnings(action="ignore", module="torch*")
+    os.environ["TORCH_LOGS"] = "-all"
+
+    # Disable huggingface_hub logging
+    logging.getLogger("huggingface_hub").setLevel(logging.CRITICAL)
+    disable_hf_hub_progress_bars()
+
+    # Disable LiteLLM logging
+    logging.getLogger("LiteLLM").setLevel(logging.CRITICAL)
+    logging.getLogger("LiteLLM Router").setLevel(logging.CRITICAL)
+    logging.getLogger("LiteLLM Proxy").setLevel(logging.CRITICAL)
+    logging.getLogger("openai").setLevel(logging.CRITICAL)
+    logging.getLogger("httpx").setLevel(logging.CRITICAL)
+    litellm.suppress_debug_info = True
+
+    # Disable vLLM logging
+    logging.getLogger("vllm").setLevel(logging.CRITICAL)
+    logging.getLogger("vllm.engine.llm_engine").setLevel(logging.CRITICAL)
+    logging.getLogger("vllm.transformers_utils.tokenizer").setLevel(logging.CRITICAL)
+    logging.getLogger("vllm.core.scheduler").setLevel(logging.CRITICAL)
+    logging.getLogger("vllm.model_executor.weight_utils").setLevel(logging.CRITICAL)
+    logging.getLogger("vllm.platforms").setLevel(logging.CRITICAL)
+    logging.getLogger("mistral_common.tokens.tokenizers.tekken").setLevel(
+        logging.CRITICAL
+    )
+    os.environ["LOG_LEVEL"] = "CRITICAL"
+    os.environ["VLLM_CONFIGURE_LOGGING"] = "0"
+
+    # Disable flashinfer logging
+    os.environ["FLASHINFER_LOGGING_LEVEL"] = "CRITICAL"
+
+    # Disable datasets logging
+    logging.getLogger("datasets").setLevel(logging.CRITICAL)
+    logging.getLogger("filelock").setLevel(logging.CRITICAL)
+    disable_datasets_progress_bars()
+
+    # Disable evaluate logging
+    warnings.filterwarnings("ignore", module="seqeval*")
+    disable_evaluate_progress_bar()
+
+    # Disable most of the `transformers` logging
+    tf_logging._default_log_level = logging.CRITICAL
+    tf_logging.set_verbosity(logging.CRITICAL)
+    logging.getLogger("transformers.trainer").setLevel(logging.CRITICAL)
+    logging.getLogger("accelerate").setLevel(logging.CRITICAL)
+
+
+class no_terminal_output:
+    """Context manager that suppresses all terminal output."""
+
+    def __init__(self, disable: bool = False) -> None:
+        """Initialise the context manager.
+
+        Args:
+            disable:
+                If True, this context manager does nothing.
+        """
+        self.disable = disable
+        self.nothing_file: TextIOWrapper | None = None
+        self._cpp_stdout_file: int | None = None
+        self._cpp_stderr_file: int | None = None
+        try:
+            self._cpp_stdout_file = os.dup(sys.stdout.fileno())
+            self._cpp_stderr_file = os.dup(sys.stderr.fileno())
+        except OSError:
+            self._log_windows_warning()
+
+    def _log_windows_warning(self) -> None:
+        """Log a warning about Windows not supporting blocking terminal output."""
+        log_once(
+            "Your operating system (probably Windows) does not support blocking "
+            "terminal output, so expect more messy output - sorry!",
+            level=logging.WARNING,
+        )
+
+    def __enter__(self) -> None:
+        """Suppress all terminal output."""
+        if not self.disable:
+            self.nothing_file = open(os.devnull, "w")
+            try:
+                os.dup2(fd=self.nothing_file.fileno(), fd2=sys.stdout.fileno())
+                os.dup2(fd=self.nothing_file.fileno(), fd2=sys.stderr.fileno())
+            except OSError:
+                self._log_windows_warning()
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: type[BaseException] | None,
+    ) -> None:
+        """Re-enable terminal output."""
+        if not self.disable:
+            if self.nothing_file is not None:
+                self.nothing_file.close()
+            try:
+                if self._cpp_stdout_file is not None:
+                    os.dup2(fd=self._cpp_stdout_file, fd2=sys.stdout.fileno())
+                if self._cpp_stderr_file is not None:
+                    os.dup2(fd=self._cpp_stderr_file, fd2=sys.stderr.fileno())
+            except OSError:
+                self._log_windows_warning()
+
+
+def adjust_logging_level(verbose: bool, ignore_testing: bool = False) -> int:
+    """Adjust the logging level based on verbosity.
+
+    Args:
+        verbose:
+            Whether to output additional output.
+        ignore_testing:
+            Whether to ignore the testing flag.
+
+    Returns:
+        The logging level that was set.
+    """
+    if hasattr(sys, "_called_from_test") and not ignore_testing:
+        logging_level = logging.CRITICAL
+    elif verbose:
+        logging_level = logging.DEBUG
+    else:
+        logging_level = logging.INFO
+    logger.setLevel(logging_level)
+    return logging_level

euroeval/metrics/base.py

@@ -2,7 +2,6 @@
 
 import abc
 import collections.abc as c
-import logging
 import typing as t
 
 if t.TYPE_CHECKING:
@@ -10,8 +9,6 @@ if t.TYPE_CHECKING:
 
     from ..data_models import BenchmarkConfig, DatasetConfig
 
-logger: logging.Logger = logging.getLogger("euroeval")
-
 
 class Metric(abc.ABC):
     """Abstract base class for all metrics."""

euroeval/metrics/huggingface.py

@@ -1,7 +1,6 @@
 """All the Hugging Face metrics used in EuroEval."""
 
 import collections.abc as c
-import logging
 import typing as t
 from pathlib import Path
 
@@ -9,7 +8,7 @@ import evaluate
 import numpy as np
 from datasets import DownloadConfig
 
-from ..utils import HiddenPrints
+from ..logging_utils import no_terminal_output
 from .base import Metric
 
 if t.TYPE_CHECKING:
@@ -18,8 +17,6 @@ if t.TYPE_CHECKING:
 
     from ..data_models import BenchmarkConfig, DatasetConfig
 
-logger: logging.Logger = logging.getLogger("euroeval")
-
 
 class HuggingFaceMetric(Metric):
     """A metric which is implemented in the `evaluate` package.
@@ -126,7 +123,7 @@ class HuggingFaceMetric(Metric):
 
         assert self.metric is not None
 
-        with HiddenPrints():
+        with no_terminal_output(disable=benchmark_config.verbose):
             results = self.metric.compute(
                 predictions=predictions, references=references, **self.compute_kwargs
             )
@@ -145,6 +142,13 @@ class HuggingFaceMetric(Metric):
 
         return score
 
+    def __del__(self) -> None:
+        """Clean up the metric from memory."""
+        if self.metric is not None:
+            if self.metric.writer is not None:
+                self.metric.writer.close()
+            del self.metric
+
 
 mcc_metric = HuggingFaceMetric(
     name="mcc",

euroeval/metrics/llm_as_a_judge.py

@@ -8,6 +8,7 @@ from pathlib import Path
 from pydantic import BaseModel, Field
 
 from ..exceptions import InvalidBenchmark
+from ..logging_utils import log
 from ..model_cache import ModelCache
 from ..utils import extract_json_dict_from_string
 from .base import Metric
@@ -17,8 +18,6 @@ if t.TYPE_CHECKING:
 
     from ..data_models import BenchmarkConfig, DatasetConfig
 
-logger: logging.Logger = logging.getLogger("euroeval")
-
 
 class LLMAsAJudgeMetric(Metric):
     """Use an LLM to judge the quality of the predictions."""
@@ -190,7 +189,10 @@ class LLMAsAJudgeMetric(Metric):
         # Calculate the scores using the scoring function
         scores = [self.scoring_fn(output) for output in outputs]
         if not scores:
-            logger.warning(f"No scores were calculated for {self.pretty_name}.")
+            log(
+                f"No scores were calculated for {self.pretty_name}.",
+                level=logging.WARNING,
+            )
             return None
         return sum(scores) / len(scores)
 

euroeval/metrics/pipeline.py

@@ -11,6 +11,7 @@ import numpy as np
 from scipy.special import expit as sigmoid
 
 from ..exceptions import InvalidBenchmark
+from ..logging_utils import log, no_terminal_output
 from ..utils import unscramble
 from .base import Metric
 
@@ -20,8 +21,6 @@ if t.TYPE_CHECKING:
 
     from ..data_models import BenchmarkConfig, DatasetConfig
 
-logger: logging.Logger = logging.getLogger("euroeval")
-
 
 T = t.TypeVar("T", bound=int | float | str | bool)
 
@@ -121,16 +120,22 @@ class PipelineMetric(Metric):
             The calculated metric score, or None if the score should be ignored.
         """
         if self.pipeline is None:
-            self.pipeline = self._download_pipeline()
+            self.pipeline = self._download_pipeline(
+                cache_dir=benchmark_config.cache_dir
+            )
         if self.preprocessing_fn is not None:
             predictions = self.preprocessing_fn(
                 predictions=predictions, dataset=dataset
             )
         return self.pipeline_scoring_function(self.pipeline, predictions)
 
-    def _download_pipeline(self) -> "Pipeline":
+    def _download_pipeline(self, cache_dir: str) -> "Pipeline":
         """Download the scikit-learn pipeline from the given URL.
 
+        Args:
+            cache_dir:
+                The directory to use for caching the downloaded pipeline.
+
         Returns:
             The downloaded scikit-learn pipeline.
 
@@ -138,10 +143,13 @@ class PipelineMetric(Metric):
             InvalidBenchmark:
                 If the loading of the pipeline fails for any reason.
         """
-        logger.debug(f"Loading pipeline from {self.pipeline_repo}...")
-        folder_path = hf_hub.HfApi(
-            token=unscramble("HjccJFhIozVymqXDVqTUTXKvYhZMTbfIjMxG_")
-        ).snapshot_download(repo_id=self.pipeline_repo, repo_type="model")
+        log(f"Loading pipeline from {self.pipeline_repo}...", level=logging.DEBUG)
+        with no_terminal_output():
+            folder_path = hf_hub.HfApi(
+                token=unscramble("XbjeOLhwebEaSaDUMqqaPaPIhgOcyOfDpGnX_")
+            ).snapshot_download(
+                repo_id=self.pipeline_repo, repo_type="model", cache_dir=cache_dir
+            )
         model_path = Path(folder_path, self.pipeline_file_name)
         try:
             with model_path.open(mode="rb") as f:
@@ -150,7 +158,7 @@ class PipelineMetric(Metric):
             raise InvalidBenchmark(
                 f"Failed to load pipeline from {self.pipeline_repo!r}: {e}"
             ) from e
-        logger.debug(f"Successfully loaded pipeline: {pipeline}")
+        log(f"Successfully loaded pipeline: {pipeline}", level=logging.DEBUG)
         return pipeline
 
 

euroeval/metrics/speed.py

@@ -1,7 +1,6 @@
 """Inference speed metric."""
 
 import collections.abc as c
-import logging
 import typing as t
 
 from .base import Metric
@@ -11,8 +10,6 @@ if t.TYPE_CHECKING:
 
     from ..data_models import BenchmarkConfig, DatasetConfig
 
-logger: logging.Logger = logging.getLogger("euroeval")
-
 
 class SpeedMetric(Metric):
     """Speed metric."""

euroeval/model_cache.py

@@ -8,11 +8,9 @@ import typing as t
 from collections import defaultdict
 from dataclasses import asdict
 
-from tqdm.auto import tqdm
-
 from .constants import NUM_GENERATION_TOKENS_FOR_CLASSIFICATION
 from .data_models import GenerativeModelOutput, SingleGenerativeModelOutput
-from .utils import log_once
+from .logging_utils import get_pbar, log, log_once
 
 if t.TYPE_CHECKING:
     from pathlib import Path
@@ -20,9 +18,6 @@ if t.TYPE_CHECKING:
     from datasets import Dataset
 
 
-logger = logging.getLogger("euroeval")
-
-
 class ModelCache:
     """A cache for model outputs.
 
@@ -65,9 +60,10 @@ class ModelCache:
             with self.cache_path.open() as f:
                 json_cache = json.load(f)
         except json.JSONDecodeError:
-            logger.warning(
+            log(
                 f"Failed to load the cache from {self.cache_path}. The cache will be "
-                f"re-initialised."
+                f"re-initialised.",
+                level=logging.WARNING,
             )
             json_cache = dict()
             with self.cache_path.open("w") as f:
@@ -89,9 +85,10 @@ class ModelCache:
             with self.cache_path.open("w") as f:
                 json.dump(dumpable_cache, f)
         except KeyError:
-            logger.warning(
+            log(
                 f"Failed to load the cache from {self.cache_path}. The cache will be "
-                f"re-initialised."
+                f"re-initialised.",
+                level=logging.WARNING,
             )
             self.cache = dict()
             with self.cache_path.open("w") as f:
@@ -172,18 +169,18 @@ class ModelCache:
 
         # Double check that the number of inputs and outputs match
         if not len(model_inputs) == len(model_output.sequences):
-            logger.warning(
+            log(
                 f"Number of model inputs ({len(model_inputs)}) does not match the "
                 f"number of model outputs ({len(model_output.sequences)}). We will not "
-                f"cache the model outputs."
+                f"cache the model outputs.",
+                level=logging.WARNING,
             )
             return
 
         # Store the generated sequences in the cache, one by one
-        with tqdm(
+        with get_pbar(
             iterable=model_inputs,
             desc="Caching model outputs",
-            leave=False,
             disable=hasattr(sys, "_called_from_test"),
         ) as pbar:
             for sample_idx, model_input in enumerate(pbar):

euroeval/model_config.py

@@ -5,14 +5,12 @@ import typing as t
 
 from . import benchmark_modules
 from .exceptions import InvalidModel, NeedsEnvironmentVariable, NeedsExtraInstalled
+from .logging_utils import log
 
 if t.TYPE_CHECKING:
     from .data_models import BenchmarkConfig, ModelConfig
 
 
-logger = logging.getLogger("euroeval")
-
-
 def get_model_config(
     model_id: str, benchmark_config: "BenchmarkConfig"
 ) -> "ModelConfig":
@@ -51,9 +49,10 @@ def get_model_config(
         elif isinstance(exists_or_err, NeedsEnvironmentVariable):
             needs_env_vars.append(exists_or_err.env_var)
         elif exists_or_err is True:
-            logger.debug(
+            log(
                 f"The model {model_id!r} was identified by the "
-                f"{benchmark_module.__name__} benchmark module."
+                f"{benchmark_module.__name__} benchmark module.",
+                logging.DEBUG,
             )
             model_config = benchmark_module.get_model_config(
                 model_id=model_id, benchmark_config=benchmark_config

euroeval/model_loading.py

@@ -10,6 +10,7 @@ from .benchmark_modules import (
 )
 from .enums import InferenceBackend, ModelType
 from .exceptions import InvalidModel
+from .logging_utils import log_once
 
 if t.TYPE_CHECKING:
     from .benchmark_modules import BenchmarkModule
@@ -34,6 +35,8 @@ def load_model(
     Returns:
         The model.
     """
+    log_once(f"Loading the model {model_config.model_id}...")
+
     # The order matters; the first model type that matches will be used. For this
     # reason, they have been ordered in terms of the most common model types.
     model_class: t.Type[BenchmarkModule]

euroeval/prompt_templates/linguistic_acceptability.py

@@ -4,6 +4,7 @@ import typing as t
 
 from ..data_models import PromptConfig
 from ..languages import (
+    CS,
     DA,
     DE,
     EN,
@@ -22,6 +23,7 @@ from ..languages import (
     NO,
     PL,
     PT,
+    SK,
     SV,
 )
 
@@ -29,6 +31,13 @@ if t.TYPE_CHECKING:
     from ..data_models import Language
 
 LA_TEMPLATES: dict["Language", PromptConfig] = {
+    CS: PromptConfig(
+        default_prompt_label_mapping=dict(correct="ano", incorrect="ne"),
+        default_prompt_prefix="Následující jsou věty a zda jsou gramaticky správné.",
+        default_prompt_template="Věta: {text}\nGramaticky správná: {label}",
+        default_instruction_prompt="Věta: {text}\n\nUrčete, zda je věta gramaticky "
+        "správná nebo ne. Odpovězte {labels_str}, a nic jiné.",
+    ),
     DA: PromptConfig(
         default_prompt_label_mapping=dict(correct="ja", incorrect="nej"),
         default_prompt_prefix="Følgende er sætninger og om de er grammatisk korrekte.",
@@ -71,11 +80,11 @@ LA_TEMPLATES: dict["Language", PromptConfig] = {
     ),
     PL: PromptConfig(
         default_prompt_label_mapping=dict(correct="tak", incorrect="nie"),
-        default_prompt_prefix="Poniżej znajdują się teksty i czy są "
+        default_prompt_prefix="Poniżej znajdują się teksty i informacja, czy są "
         "gramatycznie poprawne.",
         default_prompt_template="Tekst: {text}\nGramatycznie poprawny: {label}",
-        default_instruction_prompt="Tekst: {text}\n\nOkreśl czy tekst jest "
-        "gramatycznie poprawny czy nie. Odpowiedz {labels_str}, i nic więcej.",
+        default_instruction_prompt="Tekst: {text}\n\nOkreśl, czy tekst jest "
+        "gramatycznie poprawny. Odpowiedz używając wyłącznie {labels_str}.",
     ),
     PT: PromptConfig(
         default_prompt_label_mapping=dict(correct="sim", incorrect="não"),
@@ -174,6 +183,15 @@ LA_TEMPLATES: dict["Language", PromptConfig] = {
         default_instruction_prompt="Setning: {text}\n\nBestem om setningen er "
         "grammatisk korrekt eller ikke. Svar med {labels_str}, og ikke noe annet.",
     ),
+    SK: PromptConfig(
+        default_prompt_label_mapping=dict(correct="áno", incorrect="nie"),
+        default_prompt_prefix="Nasledujú vety a či sú gramaticky správne.",
+        default_prompt_template="Veta: {text}\nGramaticky správna: {label}",
+        default_instruction_prompt=(
+            "Veta: {text}\n\nUrčite, či je veta gramaticky správna alebo nie. "
+            "Odpovedzte so {labels_str}, a nič iné."
+        ),
+    ),
     SV: PromptConfig(
         default_prompt_label_mapping=dict(correct="ja", incorrect="nej"),
         default_prompt_prefix="Följande är meningar och huruvida de är grammatiskt "

euroeval/prompt_templates/multiple_choice.py

@@ -4,6 +4,7 @@ import typing as t
 
 from ..data_models import PromptConfig
 from ..languages import (
+    CS,
     DA,
     DE,
     EN,
@@ -21,6 +22,7 @@ from ..languages import (
     NO,
     PL,
     PT,
+    SK,
     SV,
 )
 
@@ -29,6 +31,17 @@ if t.TYPE_CHECKING:
 
 # TODO: Missing Faroese
 MULTIPLE_CHOICE_TEMPLATES: dict["Language", PromptConfig] = {
+    CS: PromptConfig(
+        default_prompt_prefix=(
+            "Následující jsou otázky s výběrem z více možností (s odpověďmi)."
+        ),
+        default_prompt_template="Otázka: {text}\nOdpověď: {label}",
+        default_instruction_prompt=(
+            "Otázka: {text}\n\nOdpovězte na výše uvedenou otázku "
+            "pomocí {labels_str}, a nic jiného."
+        ),
+        default_prompt_label_mapping="auto",
+    ),
     DA: PromptConfig(
         default_prompt_prefix="Følgende er multiple choice spørgsmål (med svar).",
         default_prompt_template="Spørgsmål: {text}\nSvar: {label}",
@@ -155,7 +168,18 @@ MULTIPLE_CHOICE_TEMPLATES: dict["Language", PromptConfig] = {
         "(z odpowiedziami).",
         default_prompt_template="Pytanie: {text}\nOdpowiedź: {label}",
         default_instruction_prompt="Pytanie: {text}\n\nOdpowiedz na powyższe pytanie, "
-        "odpowiadając {labels_str}, i nic więcej.",
+        "używając {labels_str} i niczego więcej.",
+        default_prompt_label_mapping="auto",
+    ),
+    SK: PromptConfig(
+        default_prompt_prefix=(
+            "Nasledujú otázky s viacerými možnosťami (s odpoveďami)."
+        ),
+        default_prompt_template="Otázka: {text}\nOdpoveď: {label}",
+        default_instruction_prompt=(
+            "Otázka: {text}\n\n"
+            "Odpovedzte na nasledujúcu otázku použitím {labels_str}, a nič iné."
+        ),
         default_prompt_label_mapping="auto",
     ),
     SV: PromptConfig(