EuroEval 16.3.0__py3-none-any.whl → 16.5.0__py3-none-any.whl

euroeval/tokenisation_utils.py

@@ -1,15 +1,16 @@
 """Utility functions related to tokenisation."""
 
+import collections.abc as c
 import logging
 import re
 import typing as t
 
 import torch
-from transformers import MistralCommonTokenizer
+from transformers.tokenization_mistral_common import MistralCommonTokenizer
 
 from .enums import GenerativeType
 from .exceptions import InvalidModel
-from .utils import log_once
+from .logging_utils import log, log_once
 
 if t.TYPE_CHECKING:
     from transformers.tokenization_utils import PreTrainedTokenizer
@@ -18,9 +19,6 @@ if t.TYPE_CHECKING:
     from .data_models import DatasetConfig, ModelConfig
 
 
-logger = logging.getLogger("euroeval")
-
-
 def get_special_token_metadata(tokeniser: "PreTrainedTokenizerBase") -> dict:
     """Get the special token metadata for a tokeniser.
 
@@ -74,7 +72,7 @@ def get_special_token_metadata(tokeniser: "PreTrainedTokenizerBase") -> dict:
 
 
 def should_prompts_be_stripped(
-    labels_to_be_generated: list[str], tokeniser: "PreTrainedTokenizer"
+    labels_to_be_generated: c.Sequence[str], tokeniser: "PreTrainedTokenizer"
 ) -> bool:
     """Determine if we should strip the prompts for few-shot evaluation.
 
@@ -113,7 +111,7 @@ def should_prompts_be_stripped(
 
 
 def should_prefix_space_be_added_to_labels(
-    labels_to_be_generated: list[str], tokeniser: "PreTrainedTokenizer"
+    labels_to_be_generated: c.Sequence[str], tokeniser: "PreTrainedTokenizer"
 ) -> bool:
     """Determine if we should add a prefix space to the labels.
 
@@ -182,7 +180,7 @@ def get_bos_token(
             "The model does not have a beginning-of-sequence token. Please ensure that "
             "this has been set in the tokeniser's configuration. Using no BOS token."
             " This may lead to unexpected behavior in the model.",
-            level=logging.INFO,
+            level=logging.WARNING,
         )
         return None, None
 
@@ -223,14 +221,14 @@ def get_eos_token(
             "The model does not have an end-of-sequence token. Please ensure that this "
             "has been set in the tokeniser's configuration. Using no EOS token. This "
             "may lead to unexpected behavior in the model.",
-            level=logging.INFO,
+            level=logging.WARNING,
         )
         return None, None
 
     log_once(
         f"End-of-sequence token was not set, but detected it as {eos_token!r} with "
         f"ID {eos_token_id}.",
-        level=logging.DEBUG,
+        level=logging.WARNING,
     )
     return eos_token, eos_token_id
 
@@ -306,7 +304,7 @@ def get_pad_token(
                 "Could not identify a padding token for the model. Please ensure that "
                 "this has been set in the tokeniser's configuration. Using no padding "
                 "token. This may lead to unexpected behavior in the model.",
-                level=logging.INFO,
+                level=logging.WARNING,
             )
             return None, None
 
@@ -320,7 +318,7 @@ def get_pad_token(
 
 def get_end_of_chat_token_ids(
     tokeniser: "PreTrainedTokenizer", generative_type: GenerativeType | None
-) -> list[int] | None:
+) -> c.Sequence[int] | None:
     """Get the end token ID for chat models.
 
     This is only relevant for tokenisers with a chat template.
@@ -358,12 +356,16 @@ def get_end_of_chat_token_ids(
             x_token_index = idx
             break
     else:
-        logger.debug("Could not locate the end-of-chat token for the model.")
+        log(
+            "Could not locate the end-of-chat token for the model.", level=logging.DEBUG
+        )
         return None
 
     end_of_chat_tokens = token_ids[x_token_index + 1 :]
     if len(end_of_chat_tokens) == 0:
-        logger.debug("Could not locate the end-of-chat token for the model.")
+        log(
+            "Could not locate the end-of-chat token for the model.", level=logging.DEBUG
+        )
         return None
 
     log_once(
@@ -432,13 +434,19 @@ def get_first_label_token_mapping(
 
     # Tokenise some text containing each label, which we will use to extract the
     # first token of each label
-    all_tokens: list[list[str]]
+    all_tokens: c.Sequence[c.Sequence[str]]
     if not has_chat_template(tokeniser=tokeniser):
         add_prefix_space = should_prefix_space_be_added_to_labels(
             labels_to_be_generated=local_labels, tokeniser=tokeniser
         )
         all_tokens = [
-            tokeniser.tokenize(text=f" {label}" if add_prefix_space else label)
+            [
+                tokeniser.decode(token_id)
+                for token_id in tokeniser.encode(
+                    text=f" {label}" if add_prefix_space else label,
+                    add_special_tokens=False,
+                )
+            ]
             for label in local_labels
         ]
     else:
@@ -465,7 +473,7 @@ def get_first_label_token_mapping(
     all_tokens = [
         [
             re.sub(
-                pattern=r"^[^a-zæøåüöä0-9]+|[^a-zæøåüöä0-9]+$",
+                pattern=r"^[^a-zæøåüöä0-9 ]+|[^a-zæøåüöä0-9 ]+$",
                 repl="",
                 string=token.lower(),
             )
@@ -477,11 +485,13 @@ def get_first_label_token_mapping(
     # Extract the first token of each label
     first_tokens: list[str] = list()
     for token_list, label in zip(all_tokens, local_labels):
-        matching_tokens = [tok for tok in token_list if tok and label.startswith(tok)]
+        matching_tokens = [
+            tok for tok in token_list if tok and label.startswith(tok.strip())
+        ]
         if not matching_tokens:
             if log_metadata:
                 log_once(
-                    f"No matching token found in token_list for label '{label}', so "
+                    f"No matching token found in token_list for label {label!r}, so "
                     "we will not use logprobs with the model.",
                     level=logging.DEBUG,
                 )
@@ -506,7 +516,8 @@ def get_first_label_token_mapping(
             log_once(
                 "We will not use logprobs with the model since the first tokens of the "
                 "labels are not distinct. The first tokens for the labels "
-                f"{local_labels} are {first_tokens}"
+                f"{local_labels} are {first_tokens}",
+                level=logging.DEBUG,
             )
         return False
 
@@ -547,12 +558,12 @@ def has_chat_template(tokeniser: "PreTrainedTokenizer") -> bool:
 
 
 def apply_chat_template(
-    conversation: list[dict[str, str]],
+    conversation: c.Sequence[dict[str, str]],
     tokeniser: "PreTrainedTokenizer",
     tokenise: bool,
     add_generation_prompt: bool,
     **extra_kwargs,
-) -> str | list[int]:
+) -> str | c.Sequence[int]:
     """Apply the chat template to a prompt.
 
     Args:

euroeval/types.py

@@ -1,5 +1,6 @@
 """Types used throughout the project."""
 
+import collections.abc as c
 import typing as t
 
 from transformers.trainer_utils import EvalPrediction
@@ -10,9 +11,9 @@ if t.TYPE_CHECKING:
 
     from .data_models import BenchmarkConfig, GenerativeModelOutput
 
-ScoreDict: t.TypeAlias = dict[str, dict[str, float] | list[dict[str, float]]]
-Predictions: t.TypeAlias = "NDArray | list[str] | list[list[str]]"
-Labels: t.TypeAlias = "NDArray | list[str] | list[list[str]]"
+ScoreDict: t.TypeAlias = dict[str, dict[str, float] | c.Sequence[dict[str, float]]]
+Predictions: t.TypeAlias = "NDArray | c.Sequence[str] | c.Sequence[c.Sequence[str]]"
+Labels: t.TypeAlias = "NDArray | c.Sequence[str] | c.Sequence[c.Sequence[str]]"
 
 
 class ComputeMetricsFunction(t.Protocol):
@@ -22,8 +23,8 @@ class ComputeMetricsFunction(t.Protocol):
         self,
         model_outputs_and_labels: EvalPrediction
         | tuple[
-            "NDArray | list[str] | list[list[str]]",
-            "NDArray | list[str] | list[list[str]]",
+            "NDArray | c.Sequence[str] | c.Sequence[c.Sequence[str]]",
+            "NDArray | c.Sequence[str] | c.Sequence[c.Sequence[str]]",
         ],
         dataset: "Dataset",
         benchmark_config: "BenchmarkConfig",
@@ -48,7 +49,7 @@ class ExtractLabelsFunction(t.Protocol):
 
     def __call__(
         self, input_batch: dict[str, list], model_output: "GenerativeModelOutput"
-    ) -> list[str]:
+    ) -> c.Sequence[str]:
         """Extract the labels from the generated output.
 
         Args:
@@ -63,7 +64,7 @@ class ExtractLabelsFunction(t.Protocol):
         ...
 
 
-def is_list_of_int(x: object) -> t.TypeGuard[list[int]]:
+def is_list_of_int(x: object) -> t.TypeGuard[c.Sequence[int]]:
     """Check if an object is a list of integers.
 
     Args:
@@ -76,7 +77,7 @@ def is_list_of_int(x: object) -> t.TypeGuard[list[int]]:
     return isinstance(x, list) and all(isinstance(i, int) for i in x)
 
 
-def is_list_of_list_of_int(x: object) -> t.TypeGuard[list[list[int]]]:
+def is_list_of_list_of_int(x: object) -> t.TypeGuard[c.Sequence[c.Sequence[int]]]:
     """Check if an object is a list of list of integers.
 
     Args:
@@ -93,7 +94,7 @@ def is_list_of_list_of_int(x: object) -> t.TypeGuard[list[list[int]]]:
     )
 
 
-def is_list_of_str(x: object) -> t.TypeGuard[list[str]]:
+def is_list_of_str(x: object) -> t.TypeGuard[c.Sequence[str]]:
     """Check if an object is a list of integers.
 
     Args:

euroeval/utils.py

@@ -1,6 +1,7 @@
 """Utility functions to be used in other scripts."""
 
 import asyncio
+import collections.abc as c
 import gc
 import importlib
 import importlib.metadata
@@ -11,30 +12,23 @@ import re
 import socket
 import sys
 import typing as t
-import warnings
-from functools import cache
 from pathlib import Path
 
 import demjson3
 import huggingface_hub as hf_hub
-import litellm
 import numpy as np
 import torch
-from datasets.utils import disable_progress_bar
-from transformers import logging as tf_logging
 
+from .caching_utils import cache_arguments
+from .constants import T
 from .exceptions import InvalidBenchmark, InvalidModel, NaNValueInModelOutput
+from .logging_utils import log, log_once
 
 if t.TYPE_CHECKING:
-    from types import TracebackType
-
     from .data_models import ModelIdComponents
     from .types import Predictions
 
 
-logger = logging.getLogger("euroeval")
-
-
 def create_model_cache_dir(cache_dir: str, model_id: str) -> str:
     """Create cache directory for a model.
 
@@ -149,69 +143,9 @@ def enforce_reproducibility(seed: int = 4242) -> np.random.Generator:
     return rng
 
 
-def block_terminal_output() -> None:
-    """Blocks libraries from writing output to the terminal.
-
-    This filters warnings from some libraries, sets the logging level to ERROR for some
-    libraries, disabled tokeniser progress bars when using Hugging Face tokenisers, and
-    disables most of the logging from the `transformers` library.
-    """
-    if os.getenv("FULL_LOG") == "1":
-        return
-
-    # Ignore miscellaneous warnings
-    warnings.filterwarnings("ignore", category=UserWarning)
-    warnings.filterwarnings("ignore", category=FutureWarning)
-    logging.getLogger("absl").setLevel(logging.CRITICAL)
-
-    # Disable matplotlib logging
-    logging.getLogger("matplotlib.font_manager").setLevel(logging.CRITICAL)
-
-    # Disable PyTorch logging
-    logging.getLogger("torch.utils.cpp_extension").setLevel(logging.CRITICAL)
-    warnings.filterwarnings(action="ignore", module="torch*")
-    os.environ["TORCH_LOGS"] = "-all"
-
-    # Disable huggingface_hub logging
-    logging.getLogger("huggingface_hub").setLevel(logging.CRITICAL)
-
-    # Disable LiteLLM logging
-    logging.getLogger("LiteLLM").setLevel(logging.CRITICAL)
-    logging.getLogger("LiteLLM Router").setLevel(logging.CRITICAL)
-    logging.getLogger("LiteLLM Proxy").setLevel(logging.CRITICAL)
-    logging.getLogger("openai").setLevel(logging.CRITICAL)
-    logging.getLogger("httpx").setLevel(logging.CRITICAL)
-    litellm.suppress_debug_info = True
-
-    # Disable vLLM logging
-    logging.getLogger("vllm").setLevel(logging.CRITICAL)
-    logging.getLogger("vllm.engine.llm_engine").setLevel(logging.CRITICAL)
-    logging.getLogger("vllm.transformers_utils.tokenizer").setLevel(logging.CRITICAL)
-    logging.getLogger("vllm.core.scheduler").setLevel(logging.CRITICAL)
-    logging.getLogger("vllm.model_executor.weight_utils").setLevel(logging.CRITICAL)
-    logging.getLogger("vllm.platforms").setLevel(logging.CRITICAL)
-    logging.getLogger("mistral_common.tokens.tokenizers.tekken").setLevel(
-        logging.CRITICAL
-    )
-    os.environ["LOG_LEVEL"] = "CRITICAL"
-    os.environ["VLLM_CONFIGURE_LOGGING"] = "0"
-
-    # Disable datasets logging
-    logging.getLogger("datasets").setLevel(logging.CRITICAL)
-    logging.getLogger("filelock").setLevel(logging.CRITICAL)
-    disable_progress_bar()
-
-    # Disable evaluate logging
-    warnings.filterwarnings("ignore", module="seqeval*")
-
-    # Disable most of the `transformers` logging
-    tf_logging._default_log_level = logging.CRITICAL
-    tf_logging.set_verbosity(logging.CRITICAL)
-    logging.getLogger("transformers.trainer").setLevel(logging.CRITICAL)
-    logging.getLogger("accelerate").setLevel(logging.CRITICAL)
-
-
-def get_class_by_name(class_name: str | list[str], module_name: str) -> t.Type | None:
+def get_class_by_name(
+    class_name: str | c.Sequence[str], module_name: str
+) -> t.Type | None:
     """Get a class by its name.
 
     Args:
@@ -240,9 +174,10 @@ def get_class_by_name(class_name: str | list[str], module_name: str) -> t.Type |
 
     if error_messages:
         errors = "\n- " + "\n- ".join(error_messages)
-        logger.debug(
+        log(
             f"Could not find the class with the name(s) {', '.join(class_name)}. The "
-            f"following error messages were raised: {errors}"
+            f"following error messages were raised: {errors}",
+            level=logging.DEBUG,
         )
 
     # If the class could not be found, return None
@@ -264,49 +199,27 @@ def get_min_cuda_compute_capability() -> float | None:
     return float(f"{major}.{minor}")
 
 
-@cache
+@cache_arguments(disable_condition=lambda: hasattr(sys, "_called_from_test"))
 def internet_connection_available() -> bool:
     """Checks if internet connection is available by pinging google.com.
 
     Returns:
         Whether or not internet connection is available.
     """
+    internet_available: bool = False
+
     try:
         s = socket.create_connection(("1.1.1.1", 80))
         s.close()
-        return True
-
-    # We want to only catch exceptions related to socket connections, but as we cannot
-    # import these here as they're developer dependencies, we check the exception name
-    # instead. If the exception is not related to socket connections, we reraise it.
+        internet_available = True
+    except OSError:
+        pass
     except Exception as e:
         pytest_socket_errors = ["SocketConnectBlockedError", "SocketBlockedError"]
-        if type(e).__name__ in pytest_socket_errors or isinstance(e, OSError):
-            return False
-        raise e
-
-
-class HiddenPrints:
-    """Context manager which removes all terminal output."""
-
-    def __enter__(self) -> None:
-        """Enter the context manager."""
-        self._original_stdout = sys.stdout
-        self._original_stderr = sys.stderr
-        sys.stdout = open(os.devnull, "w")
-        sys.stderr = open(os.devnull, "w")
-
-    def __exit__(
-        self,
-        exc_type: t.Type[BaseException],
-        exc_val: BaseException,
-        exc_tb: "TracebackType",
-    ) -> None:
-        """Exit the context manager."""
-        sys.stdout.close()
-        sys.stderr.close()
-        sys.stdout = self._original_stdout
-        sys.stderr = self._original_stderr
+        if type(e).__name__ not in pytest_socket_errors:
+            raise e
+
+    return internet_available
 
 
 def raise_if_model_output_contains_nan_values(model_output: "Predictions") -> None:
@@ -364,34 +277,6 @@ def unscramble(scrambled_text: str) -> str:
     return unscrambled
 
 
-@cache
-def log_once(message: str, level: int = logging.INFO) -> None:
-    """Log a message once.
-
-    This is ensured by caching the input/output pairs of this function, using the
-    `functools.cache` decorator.
-
-    Args:
-        message:
-            The message to log.
-        level:
-            The logging level. Defaults to logging.INFO.
-    """
-    match level:
-        case logging.DEBUG:
-            logger.debug(message)
-        case logging.INFO:
-            logger.info(message)
-        case logging.WARNING:
-            logger.warning(message)
-        case logging.ERROR:
-            logger.error(message)
-        case logging.CRITICAL:
-            logger.critical(message)
-        case _:
-            raise ValueError(f"Invalid logging level: {level}")
-
-
 def get_package_version(package_name: str) -> str | None:
     """Get the version of a package.
 
@@ -408,9 +293,6 @@ def get_package_version(package_name: str) -> str | None:
         return None
 
 
-T = t.TypeVar("T", bound=object)
-
-
 def safe_run(coroutine: t.Coroutine[t.Any, t.Any, T]) -> T:
     """Run a coroutine, ensuring that the event loop is always closed when we're done.
 
@@ -464,37 +346,41 @@ def extract_json_dict_from_string(s: str) -> dict | None:
     """
     json_regex = r"\{[^{}]*?\}"
     if (json_match := re.search(pattern=json_regex, string=s, flags=re.DOTALL)) is None:
-        logger.debug(
+        log(
             "The model output does not contain any JSON dictionary, so cannot parse "
-            f"it. Skipping. Here is the output: {s!r}"
+            f"it. Skipping. Here is the output: {s!r}",
+            level=logging.DEBUG,
         )
         return None
     json_string = json_match.group()
     try:
         json_output = demjson3.decode(txt=json_string)
     except demjson3.JSONDecodeError:
-        logger.debug(
+        log(
             "The model output is not valid JSON, so cannot parse it. Skipping. "
-            f"Here is the output: {json_string!r}"
+            f"Here is the output: {json_string!r}",
+            level=logging.DEBUG,
         )
         return None
     if not isinstance(json_output, dict):
-        logger.debug(
+        log(
             "The model output is not a JSON dictionary, so cannot parse "
-            f"it. Skipping. Here is the output: {json_string!r}"
+            f"it. Skipping. Here is the output: {json_string!r}",
+            level=logging.DEBUG,
         )
         return None
     elif not all(isinstance(key, str) for key in json_output.keys()):
-        logger.debug(
+        log(
             "The model output is not a JSON dictionary with string keys, "
             "so cannot parse it. Skipping. Here is the output: "
-            f"{json_string!r}"
+            f"{json_string!r}",
+            level=logging.DEBUG,
         )
         return None
     return json_output
 
 
-@cache
+@cache_arguments()
 def get_hf_token(api_key: str | None) -> str | bool:
     """Get the Hugging Face token.
 
@@ -538,8 +424,8 @@ def get_hf_token(api_key: str | None) -> str | bool:
 
 
 def extract_multiple_choice_labels(
-    prompt: str, candidate_labels: list[str]
-) -> list[str]:
+    prompt: str, candidate_labels: c.Sequence[str]
+) -> c.Sequence[str]:
     """Extract multiple choice labels from a prompt.
 
     Args: