EuroEval 15.5.0__py3-none-any.whl → 15.6.1__py3-none-any.whl

euroeval/utils.py

@@ -7,7 +7,6 @@ import importlib.util
 import logging
 import os
 import random
-import re
 import sys
 import typing as t
 import warnings
@@ -22,7 +21,7 @@ from datasets.utils import disable_progress_bar
 from requests.exceptions import RequestException
 from transformers import logging as tf_logging
 
-from .exceptions import InvalidModel, NaNValueInModelOutput
+from .exceptions import NaNValueInModelOutput
 
 if importlib.util.find_spec("ray") is not None:
     import ray
@@ -30,9 +29,6 @@ if importlib.util.find_spec("ray") is not None:
 if t.TYPE_CHECKING:
     from types import TracebackType
 
-    from transformers import PreTrainedTokenizer, PreTrainedTokenizerBase
-
-    from .data_models import DatasetConfig
     from .types import Predictions
 
 
@@ -120,6 +116,7 @@ def block_terminal_output() -> None:
     logging.getLogger("vllm.platforms").setLevel(logging.CRITICAL)
     logging.getLogger("httpx").setLevel(logging.CRITICAL)
     logging.getLogger("ray._private.worker").setLevel(logging.CRITICAL)
+    logging.getLogger("ray._private.services").setLevel(logging.CRITICAL)
     logging.getLogger("matplotlib.font_manager").setLevel(logging.CRITICAL)
     logging.getLogger("accelerate").setLevel(logging.CRITICAL)
     logging.getLogger("LiteLLM").setLevel(logging.CRITICAL)
@@ -197,19 +194,6 @@ def get_min_cuda_compute_capability() -> float | None:
     return float(f"{major}.{minor}")
 
 
-def kebab_to_pascal(kebab_string: str) -> str:
-    """Converts a kebab-case string to PascalCase.
-
-    Args:
-        kebab_string:
-            The kebab-case string.
-
-    Returns:
-        The PascalCase string.
-    """
-    return "".join(word.title() for word in kebab_string.split("-"))
-
-
 def internet_connection_available() -> bool:
     """Checks if internet connection is available by pinging google.com.
 
@@ -223,58 +207,6 @@ def internet_connection_available() -> bool:
         return False
 
 
-def get_special_token_metadata(tokenizer: "PreTrainedTokenizerBase") -> dict:
-    """Get the special token metadata for a tokenizer.
-
-    Args:
-        tokenizer:
-            The tokenizer.
-
-    Returns:
-        The special token metadata.
-    """
-    # Create some test input IDs, to check if the tokenizer is adding special tokens
-    test_input_ids = tokenizer("Test").input_ids
-
-    # Extract the CLS token IDs from the tokenizer, if it's using them
-    has_cls_token = True
-    if tokenizer.cls_token_id in test_input_ids:
-        cls_token_id = tokenizer.cls_token_id
-        cls_token = tokenizer.cls_token
-    elif tokenizer.bos_token_id in test_input_ids:
-        cls_token_id = tokenizer.bos_token_id
-        cls_token = tokenizer.bos_token
-    elif tokenizer.cls_token is not None:
-        cls_token_id = tokenizer.cls_token_id
-        cls_token = tokenizer.cls_token
-        has_cls_token = False
-    else:
-        cls_token_id = tokenizer.bos_token_id
-        cls_token = tokenizer.bos_token
-        has_cls_token = False
-
-    # Extract the SEP token IDs from the tokenizer, if it's using them
-    has_sep_token = True
-    if tokenizer.sep_token_id in test_input_ids:
-        sep_token = tokenizer.sep_token
-    elif tokenizer.eos_token_id in test_input_ids:
-        sep_token = tokenizer.eos_token
-    elif tokenizer.sep_token is not None:
-        sep_token = tokenizer.sep_token
-        has_sep_token = False
-    else:
-        sep_token = tokenizer.eos_token
-        has_sep_token = False
-
-    return dict(
-        cls_token_id=cls_token_id,
-        cls_token=cls_token,
-        sep_token=sep_token,
-        has_cls_token=has_cls_token,
-        has_sep_token=has_sep_token,
-    )
-
-
 class HiddenPrints:
     """Context manager which removes all terminal output."""
 
@@ -320,190 +252,6 @@ def raise_if_model_output_contains_nan_values(model_output: "Predictions") -> No
                 raise NaNValueInModelOutput()
 
 
-def should_prompts_be_stripped(
-    labels_to_be_generated: list[str], tokenizer: "PreTrainedTokenizer"
-) -> bool:
-    """Determine if we should strip the prompts for few-shot evaluation.
-
-    This is the case if the tokenizer needs to include the space as part of the label
-    token. The strategy is thus to tokenize a label with a preceeding colon (as in the
-    prompts), i.e., ": positive", and check if the tokenization starts with the tokens
-    of ": ". If this is the case, then we should not strip the prompts, since the
-    tokenizer produces the whitespace token separately.
-
-    Args:
-        labels_to_be_generated:
-            The labels that are to be generated.
-        tokenizer:
-            The tokenizer used to tokenize the labels.
-
-    Returns:
-        Whether we should strip the prompts.
-    """
-    strip_prompts = True
-    for label in labels_to_be_generated:
-        colon_tokens = tokenizer(": ", add_special_tokens=False).input_ids
-        label_tokens = tokenizer(": " + label, add_special_tokens=False).input_ids
-
-        if isinstance(colon_tokens, torch.Tensor):
-            colon_tokens = list(colon_tokens.squeeze(0))
-        if isinstance(label_tokens, torch.Tensor):
-            label_tokens = list(label_tokens.squeeze(0))
-
-        label_tokens_start_with_colon_tokens = (
-            label_tokens[: len(colon_tokens)] == colon_tokens
-        )
-        if label_tokens_start_with_colon_tokens:
-            strip_prompts = False
-
-    return strip_prompts
-
-
-def should_prefix_space_be_added_to_labels(
-    labels_to_be_generated: list[str], tokenizer: "PreTrainedTokenizer"
-) -> bool:
-    """Determine if we should add a prefix space to the labels.
-
-    This is the case if the prompts are stripped and the tokenizer doesn't
-    automatically add prefix whitespaces to the labels.
-
-    Args:
-        labels_to_be_generated:
-            The labels that are to be generated.
-        tokenizer:
-            The tokenizer used to tokenize the labels.
-
-    Returns:
-        Whether we should add a prefix space to the labels.
-    """
-    if not should_prompts_be_stripped(
-        labels_to_be_generated=labels_to_be_generated, tokenizer=tokenizer
-    ):
-        return False
-
-    whitespace_token = tokenizer.convert_ids_to_tokens(
-        ids=tokenizer(" ", add_special_tokens=False).input_ids[0]
-    )[0]
-
-    add_prefix_space = True
-    for label in labels_to_be_generated:
-        label_tokens = tokenizer(label, add_special_tokens=False).input_ids
-        if isinstance(label_tokens, torch.Tensor):
-            label_tokens = list(label_tokens.squeeze(0))
-        first_label_token: int = int(label_tokens[0])
-        first_character_of_label = tokenizer.convert_ids_to_tokens(first_label_token)[0]
-        has_prefix_space = first_character_of_label == whitespace_token
-        if has_prefix_space:
-            add_prefix_space = False
-            break
-
-    return add_prefix_space
-
-
-def get_bos_token(tokenizer: "PreTrainedTokenizer") -> tuple[str, int]:
-    """Get the beginning-of-sequence token from a tokenizer.
-
-    Args:
-        tokenizer:
-            The tokenizer.
-
-    Returns:
-        A pair (token, token_id) representing the beginning-of-sequence token and its
-        token ID.
-    """
-    if isinstance(tokenizer.bos_token, str) and isinstance(tokenizer.bos_token_id, int):
-        return tokenizer.bos_token, tokenizer.bos_token_id
-
-    vocab: dict[str, int] = tokenizer.get_vocab()
-
-    candidate_bos_tokens = ["<s>", "<|begin_of_text|>", "[CLS]"]
-    for candidate_bos_token in candidate_bos_tokens:
-        if candidate_bos_token in vocab:
-            bos_token = candidate_bos_token
-            bos_token_id = vocab[bos_token]
-            break
-    else:
-        raise InvalidModel(
-            "The model does not have a beginning-of-sequence token. Please ensure that "
-            "this has been set in the tokenizer's configuration."
-        )
-
-    return bos_token, bos_token_id
-
-
-def get_eos_token(tokenizer: "PreTrainedTokenizer") -> tuple[str, int]:
-    """Get the end-of-sequence token from a tokenizer.
-
-    Args:
-        tokenizer:
-            The tokenizer.
-
-    Returns:
-        A pair (token, token_id) representing the end-of-sequence token and its token
-        ID.
-    """
-    if isinstance(tokenizer.eos_token, str) and isinstance(tokenizer.eos_token_id, int):
-        return tokenizer.eos_token, tokenizer.eos_token_id
-
-    vocab: dict[str, int] = tokenizer.get_vocab()
-
-    candidate_eos_tokens = ["</s>", "<|end_of_text|>", "[SEP]"]
-    for candidate_eos_token in candidate_eos_tokens:
-        if candidate_eos_token in vocab:
-            eos_token = candidate_eos_token
-            eos_token_id = vocab[eos_token]
-            break
-    else:
-        raise InvalidModel(
-            "The model does not have an end-of-sequence token. Please ensure that this "
-            "has been set in the tokenizer's configuration."
-        )
-
-    return eos_token, eos_token_id
-
-
-def get_end_of_chat_token_ids(tokenizer: "PreTrainedTokenizer") -> list[int] | None:
-    """Get the end token ID for chat models.
-
-    This is only relevant for tokenizers with a chat template.
-
-    Args:
-        tokenizer:
-            The tokenizer.
-
-    Returns:
-        The token IDs used to end chats, or None if the tokenizer does not have a chat
-        template.
-
-    Raises:
-        ValueError:
-            If the end-of-chat token could not be located.
-    """
-    if tokenizer.chat_template is None:
-        return None
-
-    user_message: dict[t.Literal["role", "content"], str] = dict()
-    user_message["role"] = "user"
-    user_message["content"] = "X"
-    token_ids = tokenizer.apply_chat_template(conversation=[user_message])
-    assert isinstance(token_ids, list)
-
-    for idx, token in enumerate(tokenizer.convert_ids_to_tokens(token_ids)):
-        token_id = tokenizer.convert_tokens_to_ids(token)
-        assert isinstance(token_id, int)
-        token = tokenizer.decode([token_id])
-        if "X" in token:
-            x_token_index = idx
-            break
-    else:
-        raise ValueError("Could not locate the end-of-chat token for the model.")
-
-    end_of_chat_tokens = token_ids[x_token_index + 1 :]
-    if len(end_of_chat_tokens) == 0:
-        return None
-    return end_of_chat_tokens
-
-
 def scramble(text: str) -> str:
     """Scramble a string in a bijective manner.
 
@@ -579,96 +327,3 @@ def get_package_version(package_name: str) -> str | None:
         return importlib.metadata.version(package_name)
     except importlib.metadata.PackageNotFoundError:
         return None
-
-
-def get_first_label_token_mapping(
-    dataset_config: "DatasetConfig", tokenizer: "PreTrainedTokenizer | None"
-) -> dict[str, str] | bool:
-    """Check if the model should output scores.
-
-    Args:
-        dataset_config:
-            The dataset configuration.
-        tokenizer:
-            The tokenizer, or None if not available.
-
-    Returns:
-        A mapping from labels to the first token in each label, or alternatively a
-        Boolean value indicating whether the model should output scores (if the mapping
-        is outputted then the model will always output scores).
-    """
-    # Importing here to avoid circular imports
-    from .constants import TASK_GROUPS_USING_LOGPROBS
-
-    # If we do not have any tokenizer, then we cannot check if the model should output
-    # scores and we just assume it should if the dataset supports it
-    output_scores = dataset_config.task.task_group in TASK_GROUPS_USING_LOGPROBS
-    if tokenizer is None:
-        if output_scores:
-            log_once(
-                "The model will output scores, since the dataset supports it and no "
-                "tokenizer is available.",
-                level=logging.DEBUG,
-            )
-        else:
-            log_once(
-                "The model will not output scores, since the dataset does not support "
-                "it and no tokenizer is available.",
-                level=logging.DEBUG,
-            )
-        return output_scores
-
-    # If there are labels associated with the dataset, and that the first token of each
-    # label is distinct, then we can safely use the logprobs
-    if output_scores and dataset_config.labels:
-        local_labels = [
-            dataset_config.prompt_label_mapping[label].strip()
-            for label in dataset_config.labels
-        ]
-
-        # Get the first token of each label, where we add a prefix space if needed
-        add_prefix_space = (
-            should_prefix_space_be_added_to_labels(
-                labels_to_be_generated=local_labels, tokenizer=tokenizer
-            )
-            and tokenizer.chat_template is None
-        )
-        first_tokens = [
-            tokenizer.tokenize(text=f" {label}" if add_prefix_space else label)[0]
-            for label in local_labels
-        ]
-        first_tokens = [
-            re.sub(
-                pattern=r"^[^a-zæøåüöä]+|[^a-zæøåüöä]+$", repl="", string=token.lower()
-            )
-            for token in first_tokens
-        ]
-
-        # Build a mapping from labels to the first token in each label if the first
-        # tokens are distinct
-        if len(first_tokens) == len(set(first_tokens)):
-            log_once(
-                "The model will output scores, since the first tokens of the labels "
-                "are distinct.",
-                level=logging.DEBUG,
-            )
-            return {
-                label: first_token
-                for label, first_token in zip(local_labels, first_tokens)
-            }
-        else:
-            log_once(
-                "The model will not output scores, since the first tokens of the "
-                "labels are not distinct. The first tokens for the labels "
-                f"{local_labels} are {first_tokens}"
-            )
-            return False
-
-    # Otherwise, we assume that the model should not output scores, to avoid potential
-    # evaluation errors. This will force the label extraction to rely on word edit
-    # distance instead of logprobs.
-    log_once(
-        "The model will not output scores, since the dataset does not have labels.",
-        level=logging.DEBUG,
-    )
-    return False

{euroeval-15.5.0.dist-info → euroeval-15.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: EuroEval
-Version: 15.5.0
+Version: 15.6.1
 Summary: The robust European language model benchmark.
 Project-URL: Repository, https://github.com/EuroEval/EuroEval
 Project-URL: Issues, https://github.com/EuroEval/EuroEval/issues
@@ -35,7 +35,7 @@ Requires-Dist: click>=8.1.3
 Requires-Dist: datasets>=2.15.0
 Requires-Dist: demjson3>=3.0.6
 Requires-Dist: evaluate>=0.4.1
-Requires-Dist: huggingface-hub>=0.24.0
+Requires-Dist: huggingface-hub>=0.30.1
 Requires-Dist: levenshtein>=0.24.0
 Requires-Dist: litellm>=1.63.0
 Requires-Dist: more-itertools>=10.5.0
@@ -56,18 +56,18 @@ Requires-Dist: setuptools>=75.8.2
 Requires-Dist: tenacity>=9.0.0
 Requires-Dist: termcolor>=2.0.0
 Requires-Dist: torch>=2.6.0
-Requires-Dist: transformers>=4.50.0
+Requires-Dist: transformers>=4.51.0
 Provides-Extra: all
 Requires-Dist: bitsandbytes>=0.43.1; (platform_system == 'Linux') and extra == 'all'
 Requires-Dist: fbgemm-gpu>=1.0.0; (platform_system == 'Linux') and extra == 'all'
 Requires-Dist: gradio>=4.26.0; extra == 'all'
 Requires-Dist: outlines>=0.1.11; extra == 'all'
-Requires-Dist: vllm>=0.8.0; (platform_system == 'Linux') and extra == 'all'
+Requires-Dist: vllm>=0.8.3; (platform_system == 'Linux') and extra == 'all'
 Provides-Extra: generative
 Requires-Dist: bitsandbytes>=0.43.1; (platform_system == 'Linux') and extra == 'generative'
 Requires-Dist: fbgemm-gpu>=1.0.0; (platform_system == 'Linux') and extra == 'generative'
 Requires-Dist: outlines>=0.1.11; extra == 'generative'
-Requires-Dist: vllm>=0.8.0; (platform_system == 'Linux') and extra == 'generative'
+Requires-Dist: vllm>=0.8.3; (platform_system == 'Linux') and extra == 'generative'
 Provides-Extra: human-evaluation
 Requires-Dist: gradio>=4.26.0; extra == 'human-evaluation'
 Provides-Extra: test
@@ -89,7 +89,7 @@ ______________________________________________________________________
 [![Second paper](https://img.shields.io/badge/arXiv-2406.13469-b31b1b.svg)](https://arxiv.org/abs/2406.13469)
 [![License](https://img.shields.io/github/license/EuroEval/EuroEval)](https://github.com/EuroEval/EuroEval/blob/main/LICENSE)
 [![LastCommit](https://img.shields.io/github/last-commit/EuroEval/EuroEval)](https://github.com/EuroEval/EuroEval/commits/main)
-[![Code Coverage](https://img.shields.io/badge/Coverage-65%25-yellow.svg)](https://github.com/EuroEval/EuroEval/tree/main/tests)
+[![Code Coverage](https://img.shields.io/badge/Coverage-67%25-yellow.svg)](https://github.com/EuroEval/EuroEval/tree/main/tests)
 [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.0-4baaaa.svg)](https://github.com/EuroEval/EuroEval/blob/main/CODE_OF_CONDUCT.md)
 
 
@@ -206,7 +206,9 @@ sentiment-classification`.
 
 
 ### Reproducing the datasets
-All datasets used in this project are generated using the scripts located in the [src/scripts](src/scripts) folder. To reproduce a dataset, run the corresponding script with the following command
+All datasets used in this project are generated using the scripts located in the
+[src/scripts](src/scripts) folder. To reproduce a dataset, run the corresponding script
+with the following command
 
 ```shell
 $ uv run src/scripts/<name-of-script>.py
@@ -218,8 +220,28 @@ Replace <name-of-script> with the specific script you wish to execute, e.g.,
 $ uv run src/scripts/create_allocine.py
 ```
 
-
-## Special Thanks :pray:
+## Contributors :pray:
+
+A huge thank you to all the contributors who have helped make this project a success!
+
+<a href="https://github.com/peter-sk"><img src="https://avatars.githubusercontent.com/u/6168908" width=50 alt="Contributor avatar for peter-sk"/></a>
+<a href="https://github.com/AJDERS"><img src="https://avatars.githubusercontent.com/u/38854604" width=50 alt="Contributor avatar for AJDERS"/></a>
+<a href="https://github.com/oliverkinch"><img src="https://avatars.githubusercontent.com/u/71556498" width=50 alt="Contributor avatar for oliverkinch"/></a>
+<a href="https://github.com/versae"><img src="https://avatars.githubusercontent.com/u/173537" width=50 alt="Contributor avatar for versae"/></a>
+<a href="https://github.com/viggo-gascou"><img src="https://avatars.githubusercontent.com/u/94069687" width=50 alt="Contributor avatar for viggo-gascou"/></a>
+<a href="https://github.com/mathiasesn"><img src="https://avatars.githubusercontent.com/u/27091759" width=50 alt="Contributor avatar for mathiasesn"/></a>
+<a href="https://github.com/Alkarex"><img src="https://avatars.githubusercontent.com/u/1008324" width=50 alt="Contributor avatar for Alkarex"/></a>
+<a href="https://github.com/marksverdhei"><img src="https://avatars.githubusercontent.com/u/46672778" width=50 alt="Contributor avatar for marksverdhei"/></a>
+<a href="https://github.com/Mikeriess"><img src="https://avatars.githubusercontent.com/u/19728563" width=50 alt="Contributor avatar for Mikeriess"/></a>
+<a href="https://github.com/pakagronglb"><img src="https://avatars.githubusercontent.com/u/178713124" width=50 alt="Contributor avatar for pakagronglb"/></a>
+<a href="https://github.com/ThomasKluiters"><img src="https://avatars.githubusercontent.com/u/8137941" width=50 alt="Contributor avatar for ThomasKluiters"/></a>
+<a href="https://github.com/BramVanroy"><img src="https://avatars.githubusercontent.com/u/2779410" width=50 alt="Contributor avatar for BramVanroy"/></a>
+<a href="https://github.com/peregilk"><img src="https://avatars.githubusercontent.com/u/9079808" width=50 alt="Contributor avatar for peregilk"/></a>
+<a href="https://github.com/Rijgersberg"><img src="https://avatars.githubusercontent.com/u/8604946" width=50 alt="Contributor avatar for Rijgersberg"/></a>
+
+### Special Thanks
+- Thanks to [Google](https://google.com/) for sponsoring Gemini credits as part of their
+  [Google Cloud for Researchers Program](https://cloud.google.com/edu/researchers).
 - Thanks [@Mikeriess](https://github.com/Mikeriess) for evaluating many of the larger
   models on the leaderboards.
 - Thanks to [OpenAI](https://openai.com/) for sponsoring OpenAI credits as part of their

euroeval-15.6.1.dist-info/RECORD

@@ -0,0 +1,59 @@
+euroeval/__init__.py,sha256=NiT6S4II1YpnNl5KFHDNogE-rvVkOHQy5pR483eq_do,2581
+euroeval/benchmark_config_factory.py,sha256=JCjJS2pjtiuQ6tpwZ_DJFvNzwdbZu5YdJcHhFz-q6eU,12562
+euroeval/benchmarker.py,sha256=7LVFr7zL7OeJPs7WVYwekNnEmiIKPXHydcbAkW99MUk,48080
+euroeval/callbacks.py,sha256=F1AJCLB8FJpxqYprwLi_PsH4Bc0x4lyR8UiTG-GlFLY,2452
+euroeval/cli.py,sha256=EMB6g6kRvxIqlfYLSoMzwLAtEd-fqXipo4A_HTkhjkA,8575
+euroeval/constants.py,sha256=t2mAT8tE3Dn2lXWHTnaFoaOIaUcdiBjJTASCt7nSdkg,1984
+euroeval/data_loading.py,sha256=7xXdoFSvEDzpw1FNR8E8YV4c9Vy86hlU5-qLm9RUejE,3318
+euroeval/data_models.py,sha256=oZLrGg1dhIIwbgtEzq4U_fu_ZbBsz35mrqsyizuZNPw,23138
+euroeval/enums.py,sha256=L9LcNeruuhHvze9vKRogXY9vonRzoBqDzWSP6hxKQ7A,3195
+euroeval/exceptions.py,sha256=LRd7HoudupRp5-AX3L0X4hIAWCa6JVx-LViHPg7u7dg,5821
+euroeval/finetuning.py,sha256=IieAhgvxjeLHAHBief1Ay-STcCosQmrDHFTRTXFZX0Q,10743
+euroeval/generation.py,sha256=LSsskfLjIJ-c3gQxmr7eiAobPOm-5bU9vnR7uHQ7XmU,10745
+euroeval/human_evaluation.py,sha256=VGvw1X6Mkdf22r-THSNWXMIqyJP44yh4rW53vq-0huo,27681
+euroeval/languages.py,sha256=IQUbGMyn7pxAyM70M0FTO80m92Q4KgIU604MJhVia-Q,8513
+euroeval/model_cache.py,sha256=n39yFpZkudBCVwz1EQpZ-g5BQtlQemQ5nP3IiFKJZHg,8275
+euroeval/model_config.py,sha256=64KKHPTrpsFhFAANtBnAKkOs7PWZ50GXkXeDl4jICgs,2748
+euroeval/model_loading.py,sha256=B6dyjYO0Dg7NOcUXls8Sjwe6W0c2UqJ1OGw-RkzoSSQ,2239
+euroeval/scores.py,sha256=OL1MPVSgBySc9gMGeZBnj_j6-EvpDtEOwjO12IgeP6o,2899
+euroeval/speed_benchmark.py,sha256=J7VKWMf7GU_l0lRR8f0QeUr_vAaBQqTbgQ_yToHhp_0,3980
+euroeval/tasks.py,sha256=VVXFDcEM250KTGXd1pxQb8vwdia4ZJxgTUY5Kdsa-ik,7070
+euroeval/tokenization_utils.py,sha256=PNuS-FTdVrL9TWNDGlq42MvUggKwmyYM0BnC5I37IO0,11876
+euroeval/types.py,sha256=E0JhLfg-ek5pdFcYJbnGRUSodHxkuR3o8XGuIrBcuRM,2485
+euroeval/utils.py,sha256=DyWhtdFlAM1TZuiYXWNPN8KxNrZGNa-J3WfS6DGwkvM,10467
+euroeval/benchmark_modules/__init__.py,sha256=TNO-sNDwlXE-LMFXfwwqjQqUy55gywSmwRBcoPUFuaU,236
+euroeval/benchmark_modules/base.py,sha256=LcG46I2O5wcvu_3T_irBY6VkUhWVPKifBhcP-ln93TA,10798
+euroeval/benchmark_modules/fresh.py,sha256=_LWmpqiNGGTA-NoVC0v3-fS1sraDS9n-pgKUzz89jVk,9919
+euroeval/benchmark_modules/hf.py,sha256=yFApLL4_ia5Kw2iat5RSI8h5RhI4OP04HlzYidlhBCs,44012
+euroeval/benchmark_modules/litellm.py,sha256=wohdi1WoeJ-JEdQLgg2q3JbZJA77XO7yGZaTRvbRU4o,47575
+euroeval/benchmark_modules/vllm.py,sha256=FTpwal5WdrVsOpkjm_RXwf6-2PrNrrP1LO6BVGYb6GE,48086
+euroeval/dataset_configs/__init__.py,sha256=fkD1hzW7szJLc1MdK-AY4EBFWBUX5Z8t4f9uBHQnRvU,1858
+euroeval/dataset_configs/danish.py,sha256=MTt9EcriSer0QaFQ7_6evYxh-g9OPjroWegYdFpiKag,3395
+euroeval/dataset_configs/dutch.py,sha256=N3zL0vGe4OyPgVU_AiYNNfk96jSc_JDtKrVIHbaEYCU,3536
+euroeval/dataset_configs/english.py,sha256=yHw7D0zSNVbiSBAjR1mWX4V5FSkhqy4y-o-pnyWCLxE,2323
+euroeval/dataset_configs/faroese.py,sha256=QQgLe5gv0f3AtXe5rV65xZ98gFgyITQPDr3UwO4Bnv4,1350
+euroeval/dataset_configs/french.py,sha256=ATsj8_9_GxFTQgmfrniPQFZ1R9hoQCI1_ieWTnscFHU,2382
+euroeval/dataset_configs/german.py,sha256=QO6PrBQY6kyZeQMU1vg6KrC_sKyj9U2ukS9nbKO19is,2560
+euroeval/dataset_configs/icelandic.py,sha256=mncl7X4yO9gBmYqXMBfm7FKU1jcKryerSgd0dqlIA_4,4198
+euroeval/dataset_configs/italian.py,sha256=5yYMMBbxkfSDpLgJ9IH_pgkpzEp-74vMMvx-dT8x4WY,2345
+euroeval/dataset_configs/norwegian.py,sha256=2SD5681gZFa1Ig-AEpnyStbivan_bq_Pada4qwE7tw0,5181
+euroeval/dataset_configs/spanish.py,sha256=fc0dHWU7-g_p6kaSGA8nD1vLVQF_yqR2PkixrYyWywc,2212
+euroeval/dataset_configs/swedish.py,sha256=SOD2nKQTVwTpTvr362mDPHon42kr9vWs5C0mK02Fh-o,2811
+euroeval/prompt_templates/__init__.py,sha256=HWMZpybxs2xHPnVeJ43893conARahIVLWNXeRhXEGZw,357
+euroeval/prompt_templates/linguistic_acceptability.py,sha256=sx_WqLm7N6Thll6COUCCA0lXe9RMZ7WhoH6X498pixM,6232
+euroeval/prompt_templates/multiple_choice.py,sha256=H0CDQPs_WzgSJ7oI_FBzHs0TOF0Na2qZYJLhDC7S8tk,4710
+euroeval/prompt_templates/named_entity_recognition.py,sha256=T65oFEtVT8JRF9c7bq2nPm233rftPdEAGic0DU-toko,11835
+euroeval/prompt_templates/reading_comprehension.py,sha256=WbQoal_tjoTt7qsmSZXEWwlI77vgiANcZoZC1l1AZjc,6090
+euroeval/prompt_templates/sentiment_classification.py,sha256=LcFD89e5nMOv4u-Unj8_jHpNjKMmgKPEfz0-e39VbsM,6639
+euroeval/prompt_templates/summarization.py,sha256=eX0uUTf_5Xorm6f_TlBBNwLC9zKvR7YJkP0RSaLWgIw,4585
+euroeval/task_group_utils/__init__.py,sha256=CorGVkixkoEDOQuDsrOGlTmF1zmM0wnGHs8psWTfD28,72
+euroeval/task_group_utils/multiple_choice_classification.py,sha256=nB78TzOgd0HBvTclmjOYJid9ZVAgu8IHZsqB_n1SAZU,6178
+euroeval/task_group_utils/question_answering.py,sha256=kZBABJ_WYNTH4Xgo2jIvfx7iYvfoGt0EUObSaXRCGmk,27700
+euroeval/task_group_utils/sequence_classification.py,sha256=gqd0-l5o7vAY5QIpGSkSqwJwez3Y0r5SqOiywfPNW8A,12239
+euroeval/task_group_utils/text_to_text.py,sha256=QECnGdZ0YLjsbMc6LwXqVi4KMuITdiOjmJUNQtAAOW0,5712
+euroeval/task_group_utils/token_classification.py,sha256=3idWB81Fcx9UhTuk-gxMfXENrCBmiWBDUWdULXoIhpw,17863
+euroeval-15.6.1.dist-info/METADATA,sha256=4i98IBxn6yWh4ugBW-SnljmDfKEXBSfRGjZyf_dlOUs,13183
+euroeval-15.6.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+euroeval-15.6.1.dist-info/entry_points.txt,sha256=tKQRxN0HX2mGtbZbZQdCRFUDZIecA_z4mZduueor3Ug,135
+euroeval-15.6.1.dist-info/licenses/LICENSE,sha256=oZp5fpOSQ7w-vFui8QNwrBIosrO7cnpArItdbvn52Ao,1082
+euroeval-15.6.1.dist-info/RECORD,,