EuroEval 15.12.0__py3-none-any.whl → 16.7.1__py3-none-any.whl

euroeval/model_cache.py

@@ -1,5 +1,6 @@
 """ModelCache class for caching model outputs."""
 
+import collections.abc as c
 import hashlib
 import json
 import logging
@@ -8,9 +9,9 @@ import typing as t
 from collections import defaultdict
 from dataclasses import asdict
 
-from tqdm.auto import tqdm
-
+from .constants import NUM_GENERATION_TOKENS_FOR_CLASSIFICATION
 from .data_models import GenerativeModelOutput, SingleGenerativeModelOutput
+from .logging_utils import get_pbar, log, log_once
 
 if t.TYPE_CHECKING:
     from pathlib import Path
@@ -18,9 +19,6 @@ if t.TYPE_CHECKING:
     from datasets import Dataset
 
 
-logger = logging.getLogger("euroeval")
-
-
 class ModelCache:
     """A cache for model outputs.
 
@@ -33,10 +31,16 @@ class ModelCache:
             The model output cache.
         max_generated_tokens:
             The maximum number of tokens to generate for each example.
+        progress_bar:
+            Whether to show a progress bar when caching model outputs.
     """
 
     def __init__(
-        self, model_cache_dir: "Path", cache_name: str, max_generated_tokens: int
+        self,
+        model_cache_dir: "Path",
+        cache_name: str,
+        max_generated_tokens: int,
+        progress_bar: bool,
     ) -> None:
         """Initialise the model output cache.
 
@@ -47,11 +51,14 @@ class ModelCache:
                 The name of the cache file.
             max_generated_tokens:
                 The maximum number of tokens to generate for each example.
+            progress_bar:
+                Whether to show a progress bar when caching model outputs.
         """
         self.model_cache_dir = model_cache_dir
         self.model_cache_dir.mkdir(parents=True, exist_ok=True)
         self.cache_path = self.model_cache_dir / cache_name.replace("/", "--")
         self.max_generated_tokens = max_generated_tokens
+        self.progress_bar = progress_bar
 
     def load(self) -> None:
         """Load the model output cache."""
@@ -63,9 +70,10 @@ class ModelCache:
             with self.cache_path.open() as f:
                 json_cache = json.load(f)
         except json.JSONDecodeError:
-            logger.warning(
+            log(
                 f"Failed to load the cache from {self.cache_path}. The cache will be "
-                f"re-initialised."
+                f"re-initialised.",
+                level=logging.WARNING,
             )
             json_cache = dict()
             with self.cache_path.open("w") as f:
@@ -87,15 +95,16 @@ class ModelCache:
             with self.cache_path.open("w") as f:
                 json.dump(dumpable_cache, f)
         except KeyError:
-            logger.warning(
+            log(
                 f"Failed to load the cache from {self.cache_path}. The cache will be "
-                f"re-initialised."
+                f"re-initialised.",
+                level=logging.WARNING,
             )
             self.cache = dict()
             with self.cache_path.open("w") as f:
                 json.dump(dict(), f)
 
-    def _hash_key(self, key: str | list[dict[str, str]]) -> str:
+    def _hash_key(self, key: str | c.Sequence[dict[str, str]]) -> str:
         """Hash the key to use as an index in the cache.
 
         Args:
@@ -108,7 +117,7 @@ class ModelCache:
         return hashlib.md5(string=str(key).encode()).hexdigest()
 
     def __getitem__(
-        self, key: str | list[dict[str, str]]
+        self, key: str | c.Sequence[dict[str, str]]
     ) -> SingleGenerativeModelOutput:
         """Get an item from the cache.
 
@@ -123,7 +132,7 @@ class ModelCache:
         return self.cache[hashed_key]
 
     def __setitem__(
-        self, key: str | list[dict[str, str]], value: SingleGenerativeModelOutput
+        self, key: str | c.Sequence[dict[str, str]], value: SingleGenerativeModelOutput
     ) -> None:
         """Set an item in the cache.
 
@@ -141,7 +150,7 @@ class ModelCache:
         self.cache_path.unlink()
         del self.cache
 
-    def __contains__(self, key: str | list[dict[str, str]]) -> bool:
+    def __contains__(self, key: str | c.Sequence[dict[str, str]]) -> bool:
         """Check if a key is in the cache.
 
         Args:
@@ -170,29 +179,39 @@ class ModelCache:
 
         # Double check that the number of inputs and outputs match
         if not len(model_inputs) == len(model_output.sequences):
-            logger.warning(
+            log(
                 f"Number of model inputs ({len(model_inputs)}) does not match the "
                 f"number of model outputs ({len(model_output.sequences)}). We will not "
-                f"cache the model outputs."
+                f"cache the model outputs.",
+                level=logging.WARNING,
             )
             return
 
         # Store the generated sequences in the cache, one by one
-        with tqdm(
+        with get_pbar(
             iterable=model_inputs,
             desc="Caching model outputs",
-            leave=False,
-            disable=hasattr(sys, "_called_from_test"),
+            disable=hasattr(sys, "_called_from_test") or not self.progress_bar,
         ) as pbar:
             for sample_idx, model_input in enumerate(pbar):
                 # Extract the scores from the model output, to be cached. We only store
                 # the indices of the top scores, to save space. Further, we only store
                 # the scores if the generated sequence is shorter than the maximum
                 # length
-                if model_output.scores is not None and self.max_generated_tokens < 8:
+                if (
+                    model_output.scores is not None
+                    and self.max_generated_tokens
+                    <= NUM_GENERATION_TOKENS_FOR_CLASSIFICATION
+                ):
                     assert model_output.scores is not None
                     scores = model_output.scores[sample_idx]
                 else:
+                    if model_output.scores is not None:
+                        log_once(
+                            "The generated sequence is longer than the maximum "
+                            "length for classification. Not caching the scores.",
+                            level=logging.DEBUG,
+                        )
                     scores = None
                 self[model_input] = SingleGenerativeModelOutput(
                     sequence=model_output.sequences[sample_idx], scores=scores
@@ -249,7 +268,7 @@ def load_cached_model_outputs(
         The model output containing the cached sequences.
     """
     input_column = "messages" if "messages" in cached_dataset.column_names else "text"
-    cached_model_outputs: list[SingleGenerativeModelOutput] = [
+    cached_model_outputs: c.Sequence[SingleGenerativeModelOutput] = [
         cache[prompt] for prompt in cached_dataset[input_column]
     ]
 

euroeval/model_config.py

@@ -5,14 +5,12 @@ import typing as t
 
 from . import benchmark_modules
 from .exceptions import InvalidModel, NeedsEnvironmentVariable, NeedsExtraInstalled
+from .logging_utils import log
 
 if t.TYPE_CHECKING:
     from .data_models import BenchmarkConfig, ModelConfig
 
 
-logger = logging.getLogger("euroeval")
-
-
 def get_model_config(
     model_id: str, benchmark_config: "BenchmarkConfig"
 ) -> "ModelConfig":
@@ -51,9 +49,10 @@ def get_model_config(
         elif isinstance(exists_or_err, NeedsEnvironmentVariable):
             needs_env_vars.append(exists_or_err.env_var)
         elif exists_or_err is True:
-            logger.debug(
+            log(
                 f"The model {model_id!r} was identified by the "
-                f"{benchmark_module.__name__} benchmark module."
+                f"{benchmark_module.__name__} benchmark module.",
+                logging.DEBUG,
             )
             model_config = benchmark_module.get_model_config(
                 model_id=model_id, benchmark_config=benchmark_config

euroeval/model_loading.py

@@ -10,6 +10,7 @@ from .benchmark_modules import (
 )
 from .enums import InferenceBackend, ModelType
 from .exceptions import InvalidModel
+from .logging_utils import log_once
 
 if t.TYPE_CHECKING:
     from .benchmark_modules import BenchmarkModule
@@ -34,6 +35,8 @@ def load_model(
     Returns:
         The model.
     """
+    log_once(f"\nLoading the model {model_config.model_id}...")
+
     # The order matters; the first model type that matches will be used. For this
     # reason, they have been ordered in terms of the most common model types.
     model_class: t.Type[BenchmarkModule]

euroeval/prompt_templates/__init__.py

@@ -1,8 +1,10 @@
 """The different prompt templates used in EuroEval."""
 
+from .classification import CLASSIFICATION_TEMPLATES
 from .linguistic_acceptability import LA_TEMPLATES
 from .multiple_choice import MULTIPLE_CHOICE_TEMPLATES
 from .named_entity_recognition import NER_TEMPLATES
 from .reading_comprehension import RC_TEMPLATES
 from .sentiment_classification import SENT_TEMPLATES
 from .summarization import SUMM_TEMPLATES
+from .token_classification import TOKEN_CLASSIFICATION_TEMPLATES

euroeval/prompt_templates/classification.py

@@ -0,0 +1,206 @@
+"""Templates for the classification task."""
+
+import typing as t
+
+from ..data_models import PromptConfig
+from ..languages import (
+    BULGARIAN,
+    CZECH,
+    DANISH,
+    DUTCH,
+    ENGLISH,
+    ESTONIAN,
+    FAROESE,
+    FINNISH,
+    FRENCH,
+    GERMAN,
+    GREEK,
+    ICELANDIC,
+    ITALIAN,
+    LATVIAN,
+    LITHUANIAN,
+    NORWEGIAN,
+    NORWEGIAN_BOKMÅL,
+    NORWEGIAN_NYNORSK,
+    POLISH,
+    PORTUGUESE,
+    SLOVAK,
+    SPANISH,
+    SWEDISH,
+    UKRAINIAN,
+)
+
+if t.TYPE_CHECKING:
+    from ..languages import Language
+
+CLASSIFICATION_TEMPLATES: dict["Language", PromptConfig] = {
+    ENGLISH: PromptConfig(
+        default_prompt_prefix="The following are texts and their labels.",
+        default_prompt_template="Text: {text}\nLabel: {label}",
+        default_instruction_prompt="Here is a text:\n'{text}'.\n\nClassify the text "
+        "into the categories {labels_str}, and reply with only the label.",
+        default_prompt_label_mapping="auto",
+    ),
+    BULGARIAN: PromptConfig(
+        default_prompt_prefix="Следват текстове и техните етикети.",
+        default_prompt_template="Текст: {text}\nЕтикет: {label}",
+        default_instruction_prompt="Ето един текст:\n'{text}'.\n\nКласифицирайте "
+        "текста в категориите {labels_str} и отговорете само с етикета.",
+        default_prompt_label_mapping="auto",
+    ),
+    CZECH: PromptConfig(
+        default_prompt_prefix="Následují texty a jejich štítky.",
+        default_prompt_template="Text: {text}\nŠtítek: {label}",
+        default_instruction_prompt="Zde je text:\n'{text}'.\n\nKlasifikujte text do "
+        "kategorií {labels_str} a odpovězte pouze štítkem.",
+        default_prompt_label_mapping="auto",
+    ),
+    DANISH: PromptConfig(
+        default_prompt_prefix="Følgende er tekster og deres etiketter.",
+        default_prompt_template="Tekst: {text}\nEtiket: {label}",
+        default_instruction_prompt="Her er en tekst:\n'{text}'.\n\nKlassificer teksten "
+        "i kategorierne {labels_str}, og svar kun med etiketten.",
+        default_prompt_label_mapping="auto",
+    ),
+    GERMAN: PromptConfig(
+        default_prompt_prefix="Im Folgenden sind Texte und ihre Labels aufgeführt.",
+        default_prompt_template="Text: {text}\nLabel: {label}",
+        default_instruction_prompt="Hier ist ein Text:\n'{text}'.\n\nKlassifiziere den "
+        "Text in die Kategorien {labels_str} und antworte nur mit dem Label.",
+        default_prompt_label_mapping="auto",
+    ),
+    GREEK: PromptConfig(
+        default_prompt_prefix="Ακολουθούν κείμενα και οι ετικέτες τους.",
+        default_prompt_template="Κείμενο: {text}\nΕτικέτα: {label}",
+        default_instruction_prompt="Εδώ είναι ένα κείμενο:\n'{text}'.\n\n"
+        "Κατηγοριοποιήστε το κείμενο στις κατηγορίες {labels_str} και απαντήστε μόνο "
+        "με την ετικέτα.",
+        default_prompt_label_mapping="auto",
+    ),
+    SPANISH: PromptConfig(
+        default_prompt_prefix="A continuación se presentan textos y sus etiquetas.",
+        default_prompt_template="Texto: {text}\nEtiqueta: {label}",
+        default_instruction_prompt="Aquí hay un texto:\n'{text}'.\n\nClasifica el "
+        "texto en las categorías {labels_str} y responde solo con la etiqueta.",
+        default_prompt_label_mapping="auto",
+    ),
+    ESTONIAN: PromptConfig(
+        default_prompt_prefix="Järgnevad on tekstid ja nende sildid.",
+        default_prompt_template="Tekst: {text}\nSilt: {label}",
+        default_instruction_prompt="Siin on tekst:\n'{text}'.\n\nKlassifitseeri tekst "
+        "kategooriatesse {labels_str} ja vasta ainult sildiga.",
+        default_prompt_label_mapping="auto",
+    ),
+    FINNISH: PromptConfig(
+        default_prompt_prefix="Seuraavassa on tekstejä ja niiden tunnisteita.",
+        default_prompt_template="Teksti: {text}\nTunniste: {label}",
+        default_instruction_prompt="Tässä on teksti:\n'{text}'.\n\nLuokittele teksti "
+        "kategorioihin {labels_str} ja vastaa vain tunnisteella.",
+        default_prompt_label_mapping="auto",
+    ),
+    FAROESE: PromptConfig(
+        default_prompt_prefix="Hér eru tekster og teirra etikettir.",
+        default_prompt_template="Tekstur: {text}\nEtikettur: {label}",
+        default_instruction_prompt="Her er ein tekstur:\n'{text}'.\n\nFlokka teksturin "
+        "í bólkar {labels_str} og svara bert við etikettinum.",
+        default_prompt_label_mapping="auto",
+    ),
+    FRENCH: PromptConfig(
+        default_prompt_prefix="Voici des textes et leurs étiquettes.",
+        default_prompt_template="Texte : {text}\nÉtiquette : {label}",
+        default_instruction_prompt="Voici un texte :\n'{text}'.\n\nClassifiez le texte "
+        "dans les catégories {labels_str} et répondez uniquement avec l'étiquette.",
+        default_prompt_label_mapping="auto",
+    ),
+    ICELANDIC: PromptConfig(
+        default_prompt_prefix="Hér fyrir neðan eru textar og merkingar þeirra.",
+        default_prompt_template="Texti: {text}\nMerking: {label}",
+        default_instruction_prompt="Hér er texti:\n'{text}'.\n\nFlokkaðu textann "
+        "í flokkana {labels_str} og svaraðu aðeins með merkingenni.",
+        default_prompt_label_mapping="auto",
+    ),
+    ITALIAN: PromptConfig(
+        default_prompt_prefix="Di seguito sono riportati testi e le loro etichette.",
+        default_prompt_template="Testo: {text}\nEtichetta: {label}",
+        default_instruction_prompt="Ecco un testo:\n'{text}'.\n\nClassifica il testo "
+        "nelle categorie {labels_str} e rispondi solo con l'etichetta.",
+        default_prompt_label_mapping="auto",
+    ),
+    LITHUANIAN: PromptConfig(
+        default_prompt_prefix="Toliau pateikiami tekstai ir jų etiketės.",
+        default_prompt_template="Tekstas: {text}\nEtiketė: {label}",
+        default_instruction_prompt="Štai tekstas:\n'{text}'.\n\nKlasifikuokite tekstą "
+        "į kategorijas {labels_str} ir atsakykite tik etiketę.",
+        default_prompt_label_mapping="auto",
+    ),
+    LATVIAN: PromptConfig(
+        default_prompt_prefix="Turpmāk ir teksti un to etiķetes.",
+        default_prompt_template="Teksts: {text}\nEtiķete: {label}",
+        default_instruction_prompt="Šeit ir teksts:\n'{text}'.\n\nKlasificējiet tekstu "
+        "kategorijās {labels_str} un atbildiet tikai ar etiķeti.",
+        default_prompt_label_mapping="auto",
+    ),
+    NORWEGIAN_BOKMÅL: PromptConfig(
+        default_prompt_prefix="Følgende er tekster og deres etiketter.",
+        default_prompt_template="Tekst: {text}\nEtikett: {label}",
+        default_instruction_prompt="Her er en tekst:\n'{text}'.\n\nKlassifiser teksten "
+        "i kategoriene {labels_str}, og svar kun med etiketten.",
+        default_prompt_label_mapping="auto",
+    ),
+    DUTCH: PromptConfig(
+        default_prompt_prefix="Hieronder volgen teksten en hun labels.",
+        default_prompt_template="Tekst: {text}\nLabel: {label}",
+        default_instruction_prompt="Hier is een tekst:\n'{text}'.\n\nClassificeer de "
+        "tekst in de categorieën {labels_str} en antwoord alleen met het label.",
+        default_prompt_label_mapping="auto",
+    ),
+    NORWEGIAN_NYNORSK: PromptConfig(
+        default_prompt_prefix="Følgjande er tekstar og deira etikettar.",
+        default_prompt_template="Tekst: {text}\nEtikett: {label}",
+        default_instruction_prompt="Her er ein tekst:\n'{text}'.\n\nKlassifiser "
+        "teksten i kategoriane {labels_str}, og svar berre med etiketten.",
+        default_prompt_label_mapping="auto",
+    ),
+    NORWEGIAN: PromptConfig(
+        default_prompt_prefix="Følgende er tekster og deres etiketter.",
+        default_prompt_template="Tekst: {text}\nEtikett: {label}",
+        default_instruction_prompt="Her er en tekst:\n'{text}'.\n\nKlassifiser teksten "
+        "i kategoriene {labels_str}, og svar kun med etiketten.",
+        default_prompt_label_mapping="auto",
+    ),
+    POLISH: PromptConfig(
+        default_prompt_prefix="Poniżej znajdują się teksty i ich etykiety.",
+        default_prompt_template="Tekst: {text}\nEtykieta: {label}",
+        default_instruction_prompt="Oto tekst:\n'{text}'.\n\nSklasyfikuj tekst do "
+        "kategorii {labels_str} i odpowiedz tylko etykietą.",
+        default_prompt_label_mapping="auto",
+    ),
+    PORTUGUESE: PromptConfig(
+        default_prompt_prefix="A seguir estão textos e seus rótulos.",
+        default_prompt_template="Texto: {text}\nRótulo: {label}",
+        default_instruction_prompt="Aqui está um texto:\n'{text}'.\n\nClassifique o "
+        "texto nas categorias {labels_str} e responda apenas com o rótulo.",
+        default_prompt_label_mapping="auto",
+    ),
+    SLOVAK: PromptConfig(
+        default_prompt_prefix="Nasledujú texty a ich štítky.",
+        default_prompt_template="Text: {text}\nŠtítok: {label}",
+        default_instruction_prompt="Tu je text:\n'{text}'.\n\nKlasifikujte text do "
+        "kategorií {labels_str} a odpovedzte iba štítkom.",
+        default_prompt_label_mapping="auto",
+    ),
+    SWEDISH: PromptConfig(
+        default_prompt_prefix="Följande är texter och deras etiketter.",
+        default_prompt_template="Text: {text}\nEtikett: {label}",
+        default_instruction_prompt="Här är en text:\n'{text}'.\n\nKlassificera texten "
+        "i kategorierna {labels_str} och svara endast med etiketten.",
+        default_prompt_label_mapping="auto",
+    ),
+    UKRAINIAN: PromptConfig(
+        default_prompt_prefix="Нижче наведено тексти та їхні позначки.",
+        default_prompt_template="Текст: {text}\nПозначка: {label}",
+        default_instruction_prompt="Ось текст:\n'{text}'.\n\nКласифікуйте текст у "
+        "категорії {labels_str} і відповідайте лише позначкою.",
+        default_prompt_label_mapping="auto",
+    ),
+}