EuroEval 15.7.2__py3-none-any.whl → 15.8.1__py3-none-any.whl

euroeval/benchmark_modules/litellm.py

@@ -32,6 +32,7 @@ from litellm.llms.vertex_ai.common_utils import VertexAIError
 from litellm.types.utils import ChoiceLogprobs, ModelResponse
 from pydantic import conlist, create_model
 from requests.exceptions import RequestException
+from tqdm.asyncio import tqdm as tqdm_async
 from tqdm.auto import tqdm
 from transformers.trainer import Trainer
 
@@ -66,7 +67,12 @@ from ..task_group_utils import (
 )
 from ..tokenization_utils import get_first_label_token_mapping
 from ..types import ExtractLabelsFunction
-from ..utils import create_model_cache_dir, log_once
+from ..utils import (
+    catch_coroutine_exception,
+    create_model_cache_dir,
+    log_once,
+    safe_run,
+)
 from .base import BenchmarkModule
 from .hf import HuggingFaceEncoderModel, load_hf_model_config, load_tokenizer
 
@@ -159,9 +165,21 @@ class LiteLLMModel(BenchmarkModule):
     """A generative model from LiteLLM."""
 
     fresh_model = False
-    batching_preference = BatchingPreference.SINGLE_SAMPLE
+    batching_preference = BatchingPreference.ALL_AT_ONCE
     high_priority = False
 
+    _handleable_exceptions = (
+        BadRequestError,
+        RateLimitError,
+        APIError,
+        APIConnectionError,
+        Timeout,
+        ServiceUnavailableError,
+        InternalServerError,
+        SystemError,
+        AuthenticationError,
+    )
+
     def __init__(
         self,
         model_config: ModelConfig,
@@ -233,10 +251,7 @@ class LiteLLMModel(BenchmarkModule):
             The generated model outputs.
         """
         assert "messages" in inputs, "The input must contain a 'messages' key."
-        assert len(inputs["messages"]) == 1, (
-            "API models only support single-sample batching."
-        )
-        messages = inputs["messages"][0]
+        messages = inputs["messages"]
 
         generation_kwargs: dict[str, t.Any] = dict(
             model=self.model_config.model_id,
@@ -267,9 +282,20 @@ class LiteLLMModel(BenchmarkModule):
             generation_kwargs["top_logprobs"] = MAX_LOGPROBS
 
         if self.dataset_config.task in TASKS_USING_JSON:
-            assert "json" in messages[0]["content"].lower(), (
-                "Prompt must contain 'json' for JSON tasks."
-            )
+            for msg_list in messages:
+                # msg_list is a list of {'role':…, 'content':…} dicts
+                if not msg_list:
+                    raise InvalidBenchmark(
+                        "Encountered an empty message list in 'messages'."
+                    )
+                last = msg_list[-1]
+                assert isinstance(last, dict), (
+                    f"Expected dict message, got {type(last)}"
+                )
+                assert "json" in last["content"].lower(), (
+                    "Prompt must contain 'json' for JSON tasks."
+                )
+
             if self.generative_type == GenerativeType.REASONING:
                 log_once(
                     f"The model {self.model_config.model_id!r} is a reasoning model "
@@ -321,6 +347,76 @@ class LiteLLMModel(BenchmarkModule):
         # This drops generation kwargs that are not supported by the model
         litellm.drop_params = True
 
+        # Extract the generated sequences from the model response. Some APIs cannot
+        # handle using newlines as stop sequences, so we try both.
+        num_attempts = 10
+
+        all_responses = {}
+        all_failures = []
+        to_run = list(enumerate(messages))
+
+        for attempt in range(num_attempts):
+            if not to_run:
+                break
+
+            batch_indices, batch_msgs = zip(*to_run)
+            model_response, failures = safe_run(
+                self._generate_async(
+                    messages=list(batch_msgs),
+                    generation_kwargs=generation_kwargs,
+                    max_retries=3,
+                    max_reruns=15,
+                )
+            )
+
+            for orig_idx, response in zip(batch_indices, model_response):
+                all_responses[orig_idx] = response
+
+            if not failures:
+                to_run = []
+                break
+
+            all_failures.extend(failures)
+            to_run = [(orig_idx, messages[orig_idx]) for orig_idx, _ in failures]
+            logger.debug(
+                f"Attempt {attempt + 1}/{num_attempts}: "
+                f"retrying {len(to_run)} failed message(s)"
+            )
+
+            for _, error in failures:
+                self._handle_exception(error=error, generation_kwargs=generation_kwargs)
+        else:
+            raise InvalidBenchmark(
+                message=f"Failed to generate text, after {num_attempts} attempts."
+            )
+
+        if to_run:
+            raise InvalidBenchmark(
+                f"Failed to generate text after {num_attempts} attempts. "
+                f"Errors: {all_failures}"
+            )
+
+        ordered_responses = [all_responses[i] for i in range(len(messages))]
+        model_output = self._create_model_output(
+            model_responses=ordered_responses, model_id=self.model_config.model_id
+        )
+
+        return model_output
+
+    def _handle_exception(
+        self, error: Exception, generation_kwargs: dict[str, t.Any]
+    ) -> None:
+        """Handle an exception from the model.
+
+        Args:
+            error:
+                The exception to handle.
+            generation_kwargs:
+                The generation kwargs to pass to the model.
+        """
+        error_msg = str(error).lower()
+        model_id = self.model_config.model_id
+
         # Error messages that we want to catch and handle
         stop_messages = ["stop_sequences", "'stop' is not supported with this model"]
         logprobs_messages = [
@@ -341,153 +437,238 @@ class LiteLLMModel(BenchmarkModule):
         max_items_messages = ["'maxItems' is not permitted."]
         no_json_schema_messages = ["Property keys should match pattern"]
 
-        # Extract the generated sequences from the model response. Some APIs cannot
-        # handle using newlines as stop sequences, so we try both.
-        num_attempts = 10
-        for _ in range(num_attempts):
-            try:
-                model_response = litellm.completion_with_retries(
-                    messages=messages, **generation_kwargs
-                )
-                break
-            except (BadRequestError, RateLimitError) as e:
-                if any(msg.lower() in str(e).lower() for msg in stop_messages):
-                    log_once(
-                        f"The model {self.model_config.model_id!r} does not support "
-                        "stop sequences, so disabling them.",
-                        level=logging.DEBUG,
-                    )
-                    generation_kwargs["stop"] = None
-                elif (
-                    any(msg.lower() in str(e).lower() for msg in logprobs_messages)
-                    # Special case for Vertex AI models, since they have strict rate
-                    # limits on using logprobs. They also have a cap of 5 logprobs, but
-                    # we ignore this since the rate limiting makes it unusable anyway.
-                    or (isinstance(e, VertexAIError) and "logprobs" in str(e).lower())
-                ):
-                    log_once(
-                        f"The model {self.model_config.model_id!r} does not support "
-                        "logprobs, so disabling it.",
-                        level=logging.DEBUG,
-                    )
-                    generation_kwargs.pop("logprobs")
-                    generation_kwargs.pop("top_logprobs")
-                elif any(msg.lower() in str(e).lower() for msg in temperature_messages):
-                    log_once(
-                        f"The model {self.model_config.model_id!r} does not support "
-                        "temperature, so disabling it.",
-                        level=logging.DEBUG,
-                    )
-                    generation_kwargs.pop("temperature")
-                elif any(
-                    msg.lower() in str(e).lower()
-                    for msg in temperature_must_be_one_messages
-                ):
-                    log_once(
-                        f"The model {self.model_config.model_id!r} requires "
-                        "temperature to be set to 1, so setting it.",
-                        level=logging.DEBUG,
-                    )
-                    generation_kwargs["temperature"] = 1.0
-                elif any(msg.lower() in str(e).lower() for msg in max_items_messages):
-                    log_once(
-                        f"The model {self.model_config.model_id!r} does not support "
-                        "maxItems in the JSON schema, so disabling it.",
-                        level=logging.DEBUG,
-                    )
-                    ner_tag_names = list(
-                        self.dataset_config.prompt_label_mapping.values()
-                    )
-                    keys_and_their_types = {
-                        tag_name: (list[str], ...) for tag_name in ner_tag_names
-                    }
-                    pydantic_class = create_model(
-                        "AnswerFormat", **keys_and_their_types
-                    )
-                    generation_kwargs["response_format"] = pydantic_class
-                elif any(
-                    msg.lower() in str(e).lower() for msg in no_json_schema_messages
-                ):
-                    log_once(
-                        f"The model {self.model_config.model_id!r} does not support "
-                        "JSON schemas, so using the vanilla JSON format.",
-                        level=logging.DEBUG,
-                    )
-                    generation_kwargs["response_format"] = dict(type="json_object")
-                elif isinstance(e, RateLimitError):
-                    raise InvalidModel(
-                        "You have encountered your rate limit for model "
-                        f"{self.model_config.model_id!r}. Skipping."
-                    )
-                else:
-                    raise InvalidBenchmark(
-                        f"Failed to generate text. The error message was: {e}"
-                    )
-            except APIError as e:
-                raise InvalidBenchmark(
-                    f"Failed to generate text. The error message was: {e}"
-                )
-            except (
+        if any(msg.lower() in error_msg for msg in stop_messages):
+            log_once(
+                f"The model {model_id!r} does not support "
+                "stop sequences, so disabling them.",
+                level=logging.DEBUG,
+            )
+            generation_kwargs["stop"] = None
+            return
+        elif (
+            any(msg.lower() in error_msg for msg in logprobs_messages)
+            # Special case for Vertex AI models, since they have strict rate
+            # limits on using logprobs. They also have a cap of 5 logprobs, but
+            # we ignore this since the rate limiting makes it unusable anyway.
+            or (isinstance(error, VertexAIError) and "logprobs" in error_msg)
+        ):
+            log_once(
+                f"The model {model_id!r} does not support logprobs, so disabling it.",
+                level=logging.DEBUG,
+            )
+            generation_kwargs.pop("logprobs")
+            generation_kwargs.pop("top_logprobs")
+            return
+        elif any(msg.lower() in error_msg for msg in temperature_messages):
+            log_once(
+                f"The model {model_id!r} does not support "
+                "temperature, so disabling it.",
+                level=logging.DEBUG,
+            )
+            generation_kwargs.pop("temperature")
+            return
+        elif any(msg.lower() in error_msg for msg in temperature_must_be_one_messages):
+            log_once(
+                f"The model {model_id!r} requires "
+                "temperature to be set to 1, so setting it.",
+                level=logging.DEBUG,
+            )
+            generation_kwargs["temperature"] = 1.0
+            return
+        elif any(msg.lower() in error_msg for msg in max_items_messages):
+            log_once(
+                f"The model {model_id!r} does not support "
+                "maxItems in the JSON schema, so disabling it.",
+                level=logging.DEBUG,
+            )
+            ner_tag_names = list(self.dataset_config.prompt_label_mapping.values())
+            keys_and_their_types = {
+                tag_name: (list[str], ...) for tag_name in ner_tag_names
+            }
+            pydantic_class = create_model("AnswerFormat", **keys_and_their_types)
+            generation_kwargs["response_format"] = pydantic_class
+            return
+        elif any(msg.lower() in error_msg for msg in no_json_schema_messages):
+            log_once(
+                f"The model {self.model_config.model_id!r} does not support "
+                "JSON schemas, so using the vanilla JSON format.",
+                level=logging.DEBUG,
+            )
+            generation_kwargs["response_format"] = dict(type="json_object")
+            return
+        elif isinstance(
+            error,
+            (
                 APIConnectionError,
                 Timeout,
                 ServiceUnavailableError,
                 InternalServerError,
                 SystemError,
-            ) as e:
-                logger.debug(
-                    f"Service temporarily unavailable. The error message was: {e}. "
-                    f"Retrying in 5 seconds..."
+            ),
+        ):
+            logger.debug(
+                f"Service temporarily unavailable. The error message was: {error}. "
+                f"Retrying in 5 seconds..."
+            )
+            sleep(5)
+            return
+
+        if isinstance(error, RateLimitError):
+            raise InvalidModel(
+                f"You have encountered your rate limit for model {model_id!r}. "
+                "Skipping."
+            )
+
+        if isinstance(error, AuthenticationError):
+            raise NeedsAdditionalArgument(
+                cli_argument="--api-key",
+                script_argument="api_key=<your-api-key>",
+                run_with_cli=self.benchmark_config.run_with_cli,
+            )
+
+        raise InvalidBenchmark(
+            f"Failed to generate text. The error message was: {error}"
+        )
+
+    async def _generate_async(
+        self,
+        messages: list[dict[str, t.Any]],
+        generation_kwargs: dict[str, t.Any],
+        max_retries: int,
+        max_reruns: int,
+    ) -> tuple[list[ModelResponse], list[tuple[int, Exception]]]:
+        """Generate outputs from the model asynchronously.
+
+        Args:
+            messages:
+                The messages to pass to the model.
+            generation_kwargs:
+                The generation kwargs to pass to the model.
+            max_retries:
+                The maximum number of retries to make.
+            max_reruns:
+                The maximum number of reruns to make.
+
+        Returns:
+            A tuple containing the successful responses and the failed responses.
+        """
+        success = []
+        all_failures = {}
+        to_run = list(enumerate(messages))
+        prev_fail_count = len(to_run)
+        rerun_count = 0
+
+        while to_run and rerun_count < max_reruns and prev_fail_count > 0:
+            requests = [
+                litellm.acompletion(
+                    messages=msg, max_retries=max_retries, **generation_kwargs
                 )
-                sleep(5)
-            except AuthenticationError:
-                raise NeedsAdditionalArgument(
-                    cli_argument="--api-key",
-                    script_argument="api_key=<your-api-key>",
-                    run_with_cli=self.benchmark_config.run_with_cli,
+                for _, msg in to_run
+            ]
+            wrapped_requests = [
+                catch_coroutine_exception(request) for request in requests
+            ]
+            responses = await tqdm_async.gather(*wrapped_requests, leave=False)
+
+            next_to_run = []
+            current_fail_count = 0
+
+            for (orig_idx, _), response in zip(to_run, responses):
+                if isinstance(response, Exception):
+                    current_fail_count += 1
+                    all_failures[orig_idx] = response
+                    next_to_run.append((orig_idx, messages[orig_idx]))
+                else:
+                    success.append(response)
+
+            if current_fail_count >= prev_fail_count:
+                logger.warning(
+                    "Retry loop aborting due to no progress: "
+                    f"current_fail_count={current_fail_count}, "
+                    f"prev_fail_count={prev_fail_count}"
                 )
-        else:
-            raise InvalidBenchmark(
-                message=f"Failed to generate text, after {num_attempts} attempts."
-            )
+                break
+
+            prev_fail_count = current_fail_count
+            to_run = next_to_run
+            rerun_count += 1
+
+        failures = [(orig_idx, all_failures[orig_idx]) for orig_idx, _ in to_run]
+        return success, failures
+
+    @staticmethod
+    def _create_model_output(
+        model_responses: list[ModelResponse], model_id: str
+    ) -> GenerativeModelOutput:
+        """Create a GenerativeModelOutput object from a list of ModelResponse objects.
+
+        Args:
+            model_responses:
+                The list of ModelResponse objects to create the GenerativeModelOutput
+                object from.
+            model_id:
+                The ID of the model.
+
+        Returns:
+            A GenerativeModelOutput object.
+        """
+        sequences = []
+        scores = []
+        for model_response in model_responses:
+            if not model_response.choices:
+                # This happens for reasoning models, when they don't finish thinking
+                # and run out of tokens. Happens quite rarely, but we need to handle it.
+                logger.warning(
+                    f"The model {model_id!r} did not end up "
+                    "generating any text. This is likely because the model ran "
+                    "out of tokens while reasoning. Returning an empty string."
+                )
+                continue
+
+            model_response_choices = model_response.choices[0]
+            assert isinstance(model_response_choices, litellm.Choices)
+            generated_message: litellm.Message = model_response_choices.message
+            generation_output = generated_message.content or ""
+            generation_output = generation_output.strip()
+
+            # Structure the model output as a GenerativeModelOutput object
+            sequences.append(generation_output)
+            if hasattr(model_response_choices, "logprobs"):
+                logprobs_obj = model_response_choices.logprobs
+                if isinstance(logprobs_obj, ChoiceLogprobs):
+                    logprobs_list: list[list[tuple[str, float]]] = [
+                        [
+                            (top_logprob.token, top_logprob.logprob)
+                            for top_logprob in content.top_logprobs
+                        ]
+                        for content in model_response_choices.logprobs.content or list()
+                    ]
+                    scores.append(logprobs_list)
+                else:
+                    log_once(
+                        "The logprobs object is malformed, so we won't use logprobs to "
+                        "determine the labels.",
+                        level=logging.WARNING,
+                    )
 
-        assert isinstance(model_response, ModelResponse)
-        if not model_response.choices:
-            # This happens for reasoning models, when they don't finish thinking and run
-            # out of tokens. Happens quite rarely, but we need to handle it.
+        if not sequences:
             logger.warning(
-                f"The model {self.model_config.model_id!r} did not end up generating "
-                "any text. This is likely because the model ran out of tokens while "
-                "reasoning. Returning an empty string."
+                "No sequences were generated by the model "
+                f"{model_id!r}. This may be due to the "
+                "model running out of tokens or an issue with the input data. "
+                "Returning an empty GenerativeModelOutput."
             )
-            return GenerativeModelOutput(sequences=[""])
-
-        model_response_choices = model_response.choices[0]
-        assert isinstance(model_response_choices, litellm.Choices)
-        generated_message: litellm.Message = model_response_choices.message
-        generation_output = generated_message.content or ""
-        generation_output = generation_output.strip()
-
-        # Structure the model output as a GenerativeModelOutput object
-        model_output = GenerativeModelOutput(sequences=[generation_output])
-        if hasattr(model_response_choices, "logprobs"):
-            logprobs_obj = model_response_choices.logprobs
-            if isinstance(logprobs_obj, ChoiceLogprobs):
-                logprobs_list: list[list[tuple[str, float]]] = [
-                    [
-                        (top_logprob.token, top_logprob.logprob)
-                        for top_logprob in content.top_logprobs
-                    ]
-                    for content in model_response_choices.logprobs.content or list()
-                ]
-                model_output.scores = [logprobs_list]
-            else:
-                log_once(
-                    "The logprobs object is malformed, so we won't use logprobs to "
-                    "determine the labels.",
-                    level=logging.WARNING,
-                )
+            return GenerativeModelOutput(sequences=[], scores=None)
 
-        return model_output
+        if scores and len(sequences) != len(scores):
+            raise InvalidBenchmark(
+                "Sequences and scores must have the same length. "
+                f"Got {len(sequences)} sequences and {len(scores)} scores."
+            )
+
+        return GenerativeModelOutput(
+            sequences=sequences, scores=scores if scores else None
+        )
 
     @cached_property
     def num_params(self) -> int:

euroeval/benchmarker.py

@@ -11,6 +11,7 @@ from pathlib import Path
 from shutil import rmtree
 from time import sleep
 
+from huggingface_hub.constants import HF_HUB_ENABLE_HF_TRANSFER
 from torch.distributed import destroy_process_group
 
 from .benchmark_config_factory import build_benchmark_config
@@ -27,7 +28,7 @@ from .model_loading import load_model
 from .scores import log_scores
 from .speed_benchmark import benchmark_speed
 from .tasks import SPEED
-from .utils import enforce_reproducibility
+from .utils import enforce_reproducibility, get_package_version
 
 if t.TYPE_CHECKING:
     from .benchmark_modules import BenchmarkModule
@@ -164,6 +165,15 @@ class Benchmarker:
         if task is not None and dataset is not None:
             raise ValueError("Only one of `task` and `dataset` can be specified.")
 
+        # Bail early if hf_transfer is enabled but not installed.
+        if HF_HUB_ENABLE_HF_TRANSFER and get_package_version("hf_transfer") is None:
+            raise ImportError(
+                "Fast download using 'hf_transfer' is enabled "
+                "(HF_HUB_ENABLE_HF_TRANSFER=1) but the 'hf_transfer' "
+                "package is not available in your environment. "
+                "Try installing it with `pip install hf_transfer`."
+            )
+
         self.benchmark_config_default_params = BenchmarkConfigParams(
             progress_bar=progress_bar,
             save_results=save_results,

euroeval/data_models.py

@@ -529,12 +529,16 @@ class DatasetConfig:
         else:
             sep_word = main_language.or_separator
 
+        local_labels: list[str] = []
+        for label in self.labels:
+            if label not in self.prompt_label_mapping:
+                continue
+            local_label = self.prompt_label_mapping[label]
+            if local_label not in local_labels:
+                local_labels.append(local_label)
+
         # Convert labels to single-quoted labels - and remove duplicates
-        quoted_labels = [
-            f"'{self.prompt_label_mapping[label]}'"
-            for label in set(self.labels)
-            if label in self.prompt_label_mapping
-        ]
+        quoted_labels = [f"'{label}'" for label in local_labels]
 
         if not quoted_labels:
             return ""

euroeval/dataset_configs/english.py

@@ -79,7 +79,7 @@ ARC_CONFIG = DatasetConfig(
 )
 
 BELEBELE_CONFIG = DatasetConfig(
-    name="belebele",
+    name="belebele-en",
     pretty_name="the English multiple choice reading comprehension dataset BeleBele",
     huggingface_id="EuroEval/belebele-mini",
     task=MCRC,

euroeval/dataset_configs/finnish.py

@@ -2,7 +2,7 @@
 
 from ..data_models import DatasetConfig
 from ..languages import FI
-from ..tasks import LA, NER, RC, SENT, SUMM
+from ..tasks import COMMON_SENSE, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
@@ -40,16 +40,14 @@ XLSUM_FI_CONFIG = DatasetConfig(
     languages=[FI],
 )
 
-# TODO: Include when this issue has been resolved:
-# https://github.com/EuroEval/EuroEval/issues/158#issuecomment-2846664885
-# HELLASWAG_FI_CONFIG = DatasetConfig(
-#     name="hellaswag-fi",
-#     pretty_name="the truncated version of the Finnish common-sense reasoning dataset "
-#     "HellaSwag-fi, translated from the English HellaSwag dataset",
-#     huggingface_id="EuroEval/hellaswag-fi-mini",
-#     task=COMMON_SENSE,
-#     languages=[FI],
-# )
+HELLASWAG_FI_CONFIG = DatasetConfig(
+    name="hellaswag-fi",
+    pretty_name="the truncated version of the Finnish common-sense reasoning dataset "
+    "HellaSwag-fi, translated from the English HellaSwag dataset",
+    huggingface_id="EuroEval/hellaswag-fi-mini",
+    task=COMMON_SENSE,
+    languages=[FI],
+)
 
 SCALA_FI_CONFIG = DatasetConfig(
     name="scala-fi",
@@ -60,3 +58,13 @@ SCALA_FI_CONFIG = DatasetConfig(
 )
 
 ### Unofficial datasets ###
+
+BELEBELE_FI_CONFIG = DatasetConfig(
+    name="belebele-fi",
+    pretty_name="the Finnish multiple choice reading comprehension dataset "
+    "BeleBele-fi, translated from the English BeleBele dataset",
+    huggingface_id="EuroEval/belebele-fi-mini",
+    task=MCRC,
+    languages=[FI],
+    unofficial=True,
+)

euroeval/dataset_configs/italian.py

@@ -2,7 +2,7 @@
 
 from ..data_models import DatasetConfig
 from ..languages import IT
-from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT, SUMM
+from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
@@ -79,3 +79,13 @@ WIKINEURAL_IT_CONFIG = DatasetConfig(
     languages=[IT],
     unofficial=True,
 )
+
+BELEBELE_IT_CONFIG = DatasetConfig(
+    name="belebele-it",
+    pretty_name="the Italian multiple choice reading comprehension dataset "
+    "BeleBele-it, translated from the English BeleBele dataset",
+    huggingface_id="EuroEval/belebele-it-mini",
+    task=MCRC,
+    languages=[IT],
+    unofficial=True,
+)

euroeval/dataset_configs/spanish.py

@@ -2,7 +2,7 @@
 
 from ..data_models import DatasetConfig
 from ..languages import ES
-from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT, SUMM
+from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
@@ -76,3 +76,13 @@ XQUAD_ES_CONFIG = DatasetConfig(
     languages=[ES],
     unofficial=True,
 )
+
+BELEBELE_ES_CONFIG = DatasetConfig(
+    name="belebele-es",
+    pretty_name="the Spanish multiple choice reading comprehension dataset "
+    "BeleBele-es, translated from the English BeleBele dataset",
+    huggingface_id="EuroEval/belebele-es-mini",
+    task=MCRC,
+    languages=[ES],
+    unofficial=True,
+)

euroeval/finetuning.py

@@ -103,7 +103,6 @@ def finetune(
                 itr_scores = finetune_single_iteration(
                     model=model if model_already_initialized else None,
                     dataset=datasets[idx],
-                    iteration_idx=idx,
                     training_args=training_args,
                     model_config=model_config,
                     dataset_config=dataset_config,
@@ -158,7 +157,6 @@ def finetune(
 def finetune_single_iteration(
     model: BenchmarkModule | None,
     dataset: DatasetDict,
-    iteration_idx: int,
     training_args: TrainingArguments,
     model_config: "ModelConfig",
     dataset_config: "DatasetConfig",
@@ -171,8 +169,6 @@ def finetune_single_iteration(
             The model to use in the benchmark. If None then a new model will be loaded.
         dataset:
             The dataset to use for training and evaluation.
-        iteration_idx:
-            The index of the iteration.
         training_args:
             The training arguments.
         model_config:
@@ -213,41 +209,42 @@ def finetune_single_iteration(
 
         trainer.log = no_logging
 
-    # Re-block terminal output, as it gets unblocked by the `transformers`
-    # package before training
+    # Re-block terminal output, as it gets unblocked by the `transformers` package
+    # before training
     block_terminal_output()
 
-    # Sort out callbacks. We remove the callbacks that are producing unnecessary
-    # output, to avoid cluttering the terminal output
+    # Sort out callbacks. We remove the callbacks that are producing unnecessary output,
+    # to avoid cluttering the terminal output
     if not benchmark_config.verbose:
         trainer.remove_callback(PrinterCallback)
     trainer.remove_callback(ProgressCallback)
     if benchmark_config.progress_bar:
         trainer.add_callback(NeverLeaveProgressCallback)
 
-    try:
-        trainer.train()
-        with torch.inference_mode():
-            try:
-                test_scores = trainer.evaluate(
-                    eval_dataset=dataset["test"],
-                    orig_eval_dataset=dataset["original_test"],
-                    metric_key_prefix="test",
-                )
-            except TypeError:
-                test_scores = trainer.evaluate(
-                    eval_dataset=dataset["test"], metric_key_prefix="test"
-                )
-        return test_scores
-
-    except NaNValueInModelOutput as e:
-        del trainer
-        del model
-        clear_memory()
-        raise e
-
-    except (RuntimeError, ValueError, IndexError) as e:
-        raise InvalidBenchmark(str(e))
+    # Train the model
+    trainer.train()
+
+    # Evaluate the model
+    with torch.inference_mode():
+        try:
+            test_scores = trainer.evaluate(
+                eval_dataset=dataset["test"],
+                orig_eval_dataset=dataset["original_test"],
+                metric_key_prefix="test",
+            )
+        except TypeError:
+            test_scores = trainer.evaluate(
+                eval_dataset=dataset["test"], metric_key_prefix="test"
+            )
+        except NaNValueInModelOutput as e:
+            del trainer
+            del model
+            clear_memory()
+            raise e
+        except (RuntimeError, ValueError, IndexError) as e:
+            raise InvalidBenchmark(str(e))
+
+    return test_scores
 
 
 def get_training_args(
@@ -300,6 +297,7 @@ def get_training_args(
         save_total_limit=1,
         per_device_train_batch_size=batch_size,
         per_device_eval_batch_size=batch_size,
+        eval_accumulation_steps=32,
         optim=OptimizerNames.ADAMW_TORCH,
         learning_rate=2e-5,
         warmup_ratio=0.01,

euroeval/task_group_utils/sequence_classification.py

@@ -144,9 +144,27 @@ def extract_labels_from_generation(
         )
         if labels is not None:
             return labels
-    return get_closest_word_edit_labels(
-        generated_sequences=model_output.sequences, dataset_config=dataset_config
-    )
+
+    candidate_labels = [
+        dataset_config.prompt_label_mapping[lbl]
+        for lbl in dataset_config.id2label.values()
+    ]
+    new_predicted_labels: list[str] = list()
+    for predicted_label in model_output.sequences:
+        # If the prediction includes a boxed answer, use that instead of the full
+        # generation
+        if (m := re.search(r"boxed\{(.*?)\}", predicted_label)) is not None:
+            predicted_label = m.group(1)
+
+        # Pick the label with the smallest word edit distance to the predicted label
+        edit_distances = [
+            Levenshtein.distance(s1=predicted_label.lower(), s2=candidate_label.lower())
+            for candidate_label in candidate_labels
+        ]
+        predicted_label = candidate_labels[np.argmin(edit_distances).item()]
+        new_predicted_labels.append(predicted_label)
+
+    return new_predicted_labels
 
 
 def get_closest_logprobs_labels(
@@ -305,32 +323,3 @@ def get_closest_logprobs_labels(
 
     assert len(output_labels) == len(generation_logprobs)
     return output_labels
-
-
-def get_closest_word_edit_labels(
-    generated_sequences: list[str], dataset_config: "DatasetConfig"
-) -> list[str]:
-    """Get the labels with the smallest edit distance to the predicted labels.
-
-    Args:
-        generated_sequences:
-            The generated sequences from the model.
-        dataset_config:
-            The configuration of the dataset.
-
-    Returns:
-        The candidate labels with the smallest edit distance to the predicted labels.
-    """
-    candidate_labels = [
-        dataset_config.prompt_label_mapping[lbl]
-        for lbl in dataset_config.id2label.values()
-    ]
-    new_predicted_labels: list[str] = list()
-    for predicted_label in generated_sequences:
-        edit_distances = [
-            Levenshtein.distance(s1=predicted_label.lower(), s2=candidate_label.lower())
-            for candidate_label in candidate_labels
-        ]
-        closest_label = candidate_labels[np.argmin(edit_distances).item()]
-        new_predicted_labels.append(closest_label)
-    return new_predicted_labels

euroeval/tokenization_utils.py

@@ -169,7 +169,7 @@ def get_bos_token(tokenizer: "PreTrainedTokenizer") -> tuple[str, int]:
 
     vocab: dict[str, int] = tokenizer.get_vocab()
 
-    candidate_bos_tokens = ["<s>", "<|begin_of_text|>", "[CLS]"]
+    candidate_bos_tokens = ["<s>", "<|begin_of_text|>", "<|startoftext|>", "[CLS]"]
     for candidate_bos_token in candidate_bos_tokens:
         if candidate_bos_token in vocab:
             bos_token = candidate_bos_token
@@ -200,7 +200,7 @@ def get_eos_token(tokenizer: "PreTrainedTokenizer") -> tuple[str, int]:
 
     vocab: dict[str, int] = tokenizer.get_vocab()
 
-    candidate_eos_tokens = ["</s>", "<|end_of_text|>", "[SEP]"]
+    candidate_eos_tokens = ["</s>", "<|end_of_text|>", "<|endoftext|>", "[SEP]"]
     for candidate_eos_token in candidate_eos_tokens:
         if candidate_eos_token in vocab:
             eos_token = candidate_eos_token

euroeval/utils.py

@@ -1,5 +1,6 @@
 """Utility functions to be used in other scripts."""
 
+import asyncio
 import gc
 import importlib
 import importlib.metadata
@@ -327,3 +328,43 @@ def get_package_version(package_name: str) -> str | None:
         return importlib.metadata.version(package_name)
     except importlib.metadata.PackageNotFoundError:
         return None
+
+
+T = t.TypeVar("T", bound=object)
+
+
+def safe_run(coroutine: t.Coroutine[t.Any, t.Any, T]) -> T:
+    """Run a coroutine, ensuring that the event loop is always closed when we're done.
+
+    Args:
+        coroutine:
+            The coroutine to run.
+
+    Returns:
+        The result of the coroutine.
+    """
+    loop = asyncio.new_event_loop()
+    try:
+        asyncio.set_event_loop(loop)
+        return loop.run_until_complete(coroutine)
+    finally:
+        loop.close()
+        asyncio.set_event_loop(None)
+
+
+async def catch_coroutine_exception(
+    coroutine: t.Coroutine[t.Any, t.Any, T],
+) -> T | Exception:
+    """Run a coroutine, catching any exceptions and returning them.
+
+    Args:
+        coroutine:
+            The coroutine to run.
+
+    Returns:
+        The result of the coroutine, or the exception if it was raised.
+    """
+    try:
+        return await coroutine
+    except Exception as exc:
+        return exc

{euroeval-15.7.2.dist-info → euroeval-15.8.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: EuroEval
-Version: 15.7.2
+Version: 15.8.1
 Summary: The robust European language model benchmark.
 Project-URL: Repository, https://github.com/EuroEval/EuroEval
 Project-URL: Issues, https://github.com/EuroEval/EuroEval/issues

{euroeval-15.7.2.dist-info → euroeval-15.8.1.dist-info}/RECORD

@@ -1,14 +1,14 @@
 euroeval/__init__.py,sha256=NiT6S4II1YpnNl5KFHDNogE-rvVkOHQy5pR483eq_do,2581
 euroeval/benchmark_config_factory.py,sha256=RDYotoLcfNr3xU8Cw-G-Y8wLe6RSlJD1Ok9C97lWfOs,12553
-euroeval/benchmarker.py,sha256=4tCrs0CvKvQcMpJRtaonxELEDXkmY95stCGwht6wTGE,48649
+euroeval/benchmarker.py,sha256=EHoYilZ2Xx0-6_aEBlG84MsZbomJSiHNHc4wKOVVBB8,49199
 euroeval/callbacks.py,sha256=F1AJCLB8FJpxqYprwLi_PsH4Bc0x4lyR8UiTG-GlFLY,2452
 euroeval/cli.py,sha256=EMB6g6kRvxIqlfYLSoMzwLAtEd-fqXipo4A_HTkhjkA,8575
 euroeval/constants.py,sha256=p6kp_R6-Tq5LBvyXyT6Sa6N3SkjEElGS2LSZRBoQaYs,1985
 euroeval/data_loading.py,sha256=L_REtxefte5Ke4xE_Cz01zkfCyKlOYhSqT5ZXXulHPc,3992
-euroeval/data_models.py,sha256=t5FwpGxiSIMe7iKae-tT7usUWki-ILzAFFm7dPJoFsk,22973
+euroeval/data_models.py,sha256=59ca6gxmHwMdeIIU6f-gGXOVIXXDQPOt7m5nCXHK86E,23166
 euroeval/enums.py,sha256=L9LcNeruuhHvze9vKRogXY9vonRzoBqDzWSP6hxKQ7A,3195
 euroeval/exceptions.py,sha256=LRd7HoudupRp5-AX3L0X4hIAWCa6JVx-LViHPg7u7dg,5821
-euroeval/finetuning.py,sha256=IieAhgvxjeLHAHBief1Ay-STcCosQmrDHFTRTXFZX0Q,10743
+euroeval/finetuning.py,sha256=uuaUxNQJb7TivPQuI1OYQ_MIKbD-6-7mpkobLKsDefQ,10667
 euroeval/generation.py,sha256=LSsskfLjIJ-c3gQxmr7eiAobPOm-5bU9vnR7uHQ7XmU,10745
 euroeval/generation_utils.py,sha256=zRsaOHcbhysbMa983BZXxfd-qMe4NYts-ZbQxfvNTK4,13310
 euroeval/human_evaluation.py,sha256=VGvw1X6Mkdf22r-THSNWXMIqyJP44yh4rW53vq-0huo,27681
@@ -19,27 +19,27 @@ euroeval/model_loading.py,sha256=B6dyjYO0Dg7NOcUXls8Sjwe6W0c2UqJ1OGw-RkzoSSQ,223
 euroeval/scores.py,sha256=TovjCZD8wmGrIjA4v5oAQp18P5KVcHvakkByDh0Hstk,3059
 euroeval/speed_benchmark.py,sha256=J7VKWMf7GU_l0lRR8f0QeUr_vAaBQqTbgQ_yToHhp_0,3980
 euroeval/tasks.py,sha256=87gbe__K5KNIb1aBSuwGnMPmZgamJFecNNYmNgMxaVo,7069
-euroeval/tokenization_utils.py,sha256=RYTYbzCM9cryZ_w-_CzyN9Sbt47DbaGU5ukm-H38sHI,13871
+euroeval/tokenization_utils.py,sha256=kghOIZMM3H0P9YDv0VBSNI7drzgJXlkRtMwt3Cgeev8,13907
 euroeval/types.py,sha256=E0JhLfg-ek5pdFcYJbnGRUSodHxkuR3o8XGuIrBcuRM,2485
-euroeval/utils.py,sha256=DyWhtdFlAM1TZuiYXWNPN8KxNrZGNa-J3WfS6DGwkvM,10467
+euroeval/utils.py,sha256=e83OnWc0GJn0Tn_vP3tbqh1DAbLy2ky-LnIlTEOKzKU,11410
 euroeval/benchmark_modules/__init__.py,sha256=TNO-sNDwlXE-LMFXfwwqjQqUy55gywSmwRBcoPUFuaU,236
 euroeval/benchmark_modules/base.py,sha256=LcG46I2O5wcvu_3T_irBY6VkUhWVPKifBhcP-ln93TA,10798
 euroeval/benchmark_modules/fresh.py,sha256=_LWmpqiNGGTA-NoVC0v3-fS1sraDS9n-pgKUzz89jVk,9919
 euroeval/benchmark_modules/hf.py,sha256=yFApLL4_ia5Kw2iat5RSI8h5RhI4OP04HlzYidlhBCs,44012
-euroeval/benchmark_modules/litellm.py,sha256=_32H-M1L_TfW-opyaMLJFPxx0iOG8A8Zfq7uVGFKZdA,43005
+euroeval/benchmark_modules/litellm.py,sha256=dd7OqBvWA75zNrsEHtC3cx3rNpNJ-1QOL2arV_CqYG0,48231
 euroeval/benchmark_modules/vllm.py,sha256=DJyla0jr-DVMPPs4RBguxq1Xn5YguvyuAnIlgIOfFaw,39394
 euroeval/dataset_configs/__init__.py,sha256=kWKtlSAOY-olOQL3UtFqL6I3Tki3G3waMZSd2YChjCg,1895
 euroeval/dataset_configs/danish.py,sha256=MTt9EcriSer0QaFQ7_6evYxh-g9OPjroWegYdFpiKag,3395
 euroeval/dataset_configs/dutch.py,sha256=r21nxEvMmBkKqPXVW082batPsxJ9d0RB4DzngOTMJSk,3185
-euroeval/dataset_configs/english.py,sha256=yHw7D0zSNVbiSBAjR1mWX4V5FSkhqy4y-o-pnyWCLxE,2323
+euroeval/dataset_configs/english.py,sha256=-N85DiNVrZFqpahNUTfxaWy4vvdOWC8Bi0G4uAO4uDw,2326
 euroeval/dataset_configs/faroese.py,sha256=QQgLe5gv0f3AtXe5rV65xZ98gFgyITQPDr3UwO4Bnv4,1350
-euroeval/dataset_configs/finnish.py,sha256=lZA2bY_ul9qh3uGFrTNe7q15WyZ04EL9OYmrkcNjygY,1857
+euroeval/dataset_configs/finnish.py,sha256=_8YWIlZNpO8Qi233bH7cKwm3tq3WETLfC_6mzg7LLog,2045
 euroeval/dataset_configs/french.py,sha256=ATsj8_9_GxFTQgmfrniPQFZ1R9hoQCI1_ieWTnscFHU,2382
 euroeval/dataset_configs/german.py,sha256=QO6PrBQY6kyZeQMU1vg6KrC_sKyj9U2ukS9nbKO19is,2560
 euroeval/dataset_configs/icelandic.py,sha256=mncl7X4yO9gBmYqXMBfm7FKU1jcKryerSgd0dqlIA_4,4198
-euroeval/dataset_configs/italian.py,sha256=5yYMMBbxkfSDpLgJ9IH_pgkpzEp-74vMMvx-dT8x4WY,2345
+euroeval/dataset_configs/italian.py,sha256=KNjCvTzsEqH_EEk3At8slKqNwWWiIdbv_t5ke7n9nZI,2660
 euroeval/dataset_configs/norwegian.py,sha256=2SD5681gZFa1Ig-AEpnyStbivan_bq_Pada4qwE7tw0,5181
-euroeval/dataset_configs/spanish.py,sha256=fc0dHWU7-g_p6kaSGA8nD1vLVQF_yqR2PkixrYyWywc,2212
+euroeval/dataset_configs/spanish.py,sha256=NviL-FzJ5jq1bLTRvbtZBiGrAmZjxyijZNpKZFrnT-M,2527
 euroeval/dataset_configs/swedish.py,sha256=SOD2nKQTVwTpTvr362mDPHon42kr9vWs5C0mK02Fh-o,2811
 euroeval/prompt_templates/__init__.py,sha256=HWMZpybxs2xHPnVeJ43893conARahIVLWNXeRhXEGZw,357
 euroeval/prompt_templates/linguistic_acceptability.py,sha256=FAIJKS26EVRxlLHk1C3lN0GDtd5AM0MwvaMf-NNIxfU,6677
@@ -51,11 +51,11 @@ euroeval/prompt_templates/summarization.py,sha256=mcWeKNhGWmp7IG_iY64T-VOSabQg5w
 euroeval/task_group_utils/__init__.py,sha256=CorGVkixkoEDOQuDsrOGlTmF1zmM0wnGHs8psWTfD28,72
 euroeval/task_group_utils/multiple_choice_classification.py,sha256=nB78TzOgd0HBvTclmjOYJid9ZVAgu8IHZsqB_n1SAZU,6178
 euroeval/task_group_utils/question_answering.py,sha256=kZBABJ_WYNTH4Xgo2jIvfx7iYvfoGt0EUObSaXRCGmk,27700
-euroeval/task_group_utils/sequence_classification.py,sha256=MCdO5h3v_LWTkrvKAeefPq7rl1H5mFed50nAL4uZq0E,13837
+euroeval/task_group_utils/sequence_classification.py,sha256=Yqx0pUhuHYmSkv1ZUfOndSLTvpr0lWCk19oYITfSjV4,13555
 euroeval/task_group_utils/text_to_text.py,sha256=Nu1_qRPLbboCd9Q5rxqY4fQFJ_aGXu80aWQqoTG1cYc,5047
 euroeval/task_group_utils/token_classification.py,sha256=3idWB81Fcx9UhTuk-gxMfXENrCBmiWBDUWdULXoIhpw,17863
-euroeval-15.7.2.dist-info/METADATA,sha256=nCF9GI8kOoKP3Up_KgPSxe4pnomawC1rQqRGlYoEsIA,13669
-euroeval-15.7.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-euroeval-15.7.2.dist-info/entry_points.txt,sha256=tKQRxN0HX2mGtbZbZQdCRFUDZIecA_z4mZduueor3Ug,135
-euroeval-15.7.2.dist-info/licenses/LICENSE,sha256=oZp5fpOSQ7w-vFui8QNwrBIosrO7cnpArItdbvn52Ao,1082
-euroeval-15.7.2.dist-info/RECORD,,
+euroeval-15.8.1.dist-info/METADATA,sha256=Fdzj20PR6wWZUx_7f_bhPh8S4DF6ghZwMIDrJ4ozxFE,13669
+euroeval-15.8.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+euroeval-15.8.1.dist-info/entry_points.txt,sha256=tKQRxN0HX2mGtbZbZQdCRFUDZIecA_z4mZduueor3Ug,135
+euroeval-15.8.1.dist-info/licenses/LICENSE,sha256=oZp5fpOSQ7w-vFui8QNwrBIosrO7cnpArItdbvn52Ao,1082
+euroeval-15.8.1.dist-info/RECORD,,