EuroEval 15.8.0__tar.gz → 15.8.2__tar.gz

{euroeval-15.8.0 → euroeval-15.8.2}/.github/workflows/ci.yaml

@@ -10,6 +10,10 @@ on:
     branches:
       - main
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
 jobs:
   code-check:
     if: github.event.pull_request.draft == false

{euroeval-15.8.0 → euroeval-15.8.2}/.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
       - id: trailing-whitespace
       - id: debug-statements
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.8
+    rev: v0.11.9
     hooks:
       - id: ruff
         args:

{euroeval-15.8.0 → euroeval-15.8.2}/CHANGELOG.md

@@ -10,6 +10,23 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 
 
+## [v15.8.2] - 2025-05-12
+### Fixed
+- Catch error when caching generative model outputs, when the number of model inputs and
+  outputs do not match.
+- Disallow vLLM >=0.8.5, as it breaks generation output for several models.
+
+
+## [v15.8.1] - 2025-05-08
+### Fixed
+- NER labels were included twice in the prompt templates (which was due to there being
+  both, e.g., `B-ORG` and `I-ORG`). This caused models not using structured generation,
+  such as reasoning models, to sometimes output the wrong labels. This has been fixed
+  now.
+- If a model outputs a `\boxed{}` answer, we now extract and use that, rather than the
+  full generated answer.
+
+
 ## [v15.8.0] - 2025-05-07
 ### Added
 - Added the BeleBele datasets for Finnish, Italian and Spanish. They are listed as

{euroeval-15.8.0 → euroeval-15.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: EuroEval
-Version: 15.8.0
+Version: 15.8.2
 Summary: The robust European language model benchmark.
 Project-URL: Repository, https://github.com/EuroEval/EuroEval
 Project-URL: Issues, https://github.com/EuroEval/EuroEval/issues
@@ -62,12 +62,12 @@ Requires-Dist: bitsandbytes>=0.43.1; (platform_system == 'Linux') and extra == '
 Requires-Dist: fbgemm-gpu>=1.0.0; (platform_system == 'Linux') and extra == 'all'
 Requires-Dist: gradio>=4.26.0; extra == 'all'
 Requires-Dist: outlines>=0.1.11; extra == 'all'
-Requires-Dist: vllm>=0.8.3; (platform_system == 'Linux') and extra == 'all'
+Requires-Dist: vllm<0.8.5,>=0.8.3; (platform_system == 'Linux') and extra == 'all'
 Provides-Extra: generative
 Requires-Dist: bitsandbytes>=0.43.1; (platform_system == 'Linux') and extra == 'generative'
 Requires-Dist: fbgemm-gpu>=1.0.0; (platform_system == 'Linux') and extra == 'generative'
 Requires-Dist: outlines>=0.1.11; extra == 'generative'
-Requires-Dist: vllm>=0.8.3; (platform_system == 'Linux') and extra == 'generative'
+Requires-Dist: vllm<0.8.5,>=0.8.3; (platform_system == 'Linux') and extra == 'generative'
 Provides-Extra: human-evaluation
 Requires-Dist: gradio>=4.26.0; extra == 'human-evaluation'
 Provides-Extra: test

{euroeval-15.8.0 → euroeval-15.8.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "EuroEval"
-version = "15.8.0"
+version = "15.8.2"
 description = "The robust European language model benchmark."
 readme = "README.md"
 authors = [
@@ -46,7 +46,7 @@ dependencies = [
 generative = [
     "outlines>=0.1.11",
     "bitsandbytes>=0.43.1; platform_system == 'Linux'",
-    "vllm>=0.8.3; platform_system == 'Linux'",
+    "vllm>=0.8.3,<0.8.5; platform_system == 'Linux'",
     "fbgemm-gpu>=1.0.0; platform_system == 'Linux'",
 ]
 human_evaluation = [
@@ -55,7 +55,7 @@ human_evaluation = [
 all = [
     "outlines>=0.1.11",
     "bitsandbytes>=0.43.1; platform_system == 'Linux'",
-    "vllm>=0.8.3; platform_system == 'Linux'",
+    "vllm>=0.8.3,<0.8.5; platform_system == 'Linux'",
     "fbgemm-gpu>=1.0.0; platform_system == 'Linux'",
     "gradio>=4.26.0",
 ]

{euroeval-15.8.0 → euroeval-15.8.2}/src/euroeval/benchmark_modules/litellm.py

@@ -401,6 +401,12 @@ class LiteLLMModel(BenchmarkModule):
             model_responses=ordered_responses, model_id=self.model_config.model_id
         )
 
+        if len(messages) != len(model_output.sequences):
+            raise InvalidBenchmark(
+                f"Number of model inputs ({len(messages):,}) does not match the "
+                f"number of model outputs ({len(model_output.sequences):,})."
+            )
+
         return model_output
 
     def _handle_exception(
@@ -616,8 +622,7 @@ class LiteLLMModel(BenchmarkModule):
         scores = []
         for model_response in model_responses:
             if not model_response.choices:
-                # This happens for reasoning models, when they don't finish thinking
-                # and run out of tokens. Happens quite rarely, but we need to handle it.
+                sequences.append("")
                 logger.warning(
                     f"The model {model_id!r} did not end up "
                     "generating any text. This is likely because the model ran "

{euroeval-15.8.0 → euroeval-15.8.2}/src/euroeval/data_models.py

@@ -529,12 +529,16 @@ class DatasetConfig:
         else:
             sep_word = main_language.or_separator
 
+        local_labels: list[str] = []
+        for label in self.labels:
+            if label not in self.prompt_label_mapping:
+                continue
+            local_label = self.prompt_label_mapping[label]
+            if local_label not in local_labels:
+                local_labels.append(local_label)
+
         # Convert labels to single-quoted labels - and remove duplicates
-        quoted_labels = [
-            f"'{self.prompt_label_mapping[label]}'"
-            for label in set(self.labels)
-            if label in self.prompt_label_mapping
-        ]
+        quoted_labels = [f"'{label}'" for label in local_labels]
 
         if not quoted_labels:
             return ""

{euroeval-15.8.0 → euroeval-15.8.2}/src/euroeval/model_cache.py

@@ -168,6 +168,15 @@ class ModelCache:
         input_column = "messages" if "messages" in model_inputs else "text"
         model_inputs = model_inputs[input_column]
 
+        # Double check that the number of inputs and outputs match
+        if not len(model_inputs) == len(model_output.sequences):
+            logger.warning(
+                f"Number of model inputs ({len(model_inputs)}) does not match the "
+                f"number of model outputs ({len(model_output.sequences)}). We will not "
+                f"cache the model outputs."
+            )
+            return
+
         # Store the generated sequences in the cache, one by one
         with tqdm(
             iterable=model_inputs,

{euroeval-15.8.0 → euroeval-15.8.2}/src/euroeval/task_group_utils/sequence_classification.py

@@ -144,9 +144,27 @@ def extract_labels_from_generation(
         )
         if labels is not None:
             return labels
-    return get_closest_word_edit_labels(
-        generated_sequences=model_output.sequences, dataset_config=dataset_config
-    )
+
+    candidate_labels = [
+        dataset_config.prompt_label_mapping[lbl]
+        for lbl in dataset_config.id2label.values()
+    ]
+    new_predicted_labels: list[str] = list()
+    for predicted_label in model_output.sequences:
+        # If the prediction includes a boxed answer, use that instead of the full
+        # generation
+        if (m := re.search(r"boxed\{(.*?)\}", predicted_label)) is not None:
+            predicted_label = m.group(1)
+
+        # Pick the label with the smallest word edit distance to the predicted label
+        edit_distances = [
+            Levenshtein.distance(s1=predicted_label.lower(), s2=candidate_label.lower())
+            for candidate_label in candidate_labels
+        ]
+        predicted_label = candidate_labels[np.argmin(edit_distances).item()]
+        new_predicted_labels.append(predicted_label)
+
+    return new_predicted_labels
 
 
 def get_closest_logprobs_labels(
@@ -305,32 +323,3 @@ def get_closest_logprobs_labels(
 
     assert len(output_labels) == len(generation_logprobs)
     return output_labels
-
-
-def get_closest_word_edit_labels(
-    generated_sequences: list[str], dataset_config: "DatasetConfig"
-) -> list[str]:
-    """Get the labels with the smallest edit distance to the predicted labels.
-
-    Args:
-        generated_sequences:
-            The generated sequences from the model.
-        dataset_config:
-            The configuration of the dataset.
-
-    Returns:
-        The candidate labels with the smallest edit distance to the predicted labels.
-    """
-    candidate_labels = [
-        dataset_config.prompt_label_mapping[lbl]
-        for lbl in dataset_config.id2label.values()
-    ]
-    new_predicted_labels: list[str] = list()
-    for predicted_label in generated_sequences:
-        edit_distances = [
-            Levenshtein.distance(s1=predicted_label.lower(), s2=candidate_label.lower())
-            for candidate_label in candidate_labels
-        ]
-        closest_label = candidate_labels[np.argmin(edit_distances).item()]
-        new_predicted_labels.append(closest_label)
-    return new_predicted_labels

{euroeval-15.8.0 → euroeval-15.8.2}/uv.lock

@@ -906,7 +906,7 @@ wheels = [
 
 [[package]]
 name = "euroeval"
-version = "15.8.0"
+version = "15.8.2"
 source = { editable = "." }
 dependencies = [
     { name = "accelerate" },
@@ -1034,8 +1034,8 @@ requires-dist = [
     { name = "termcolor", specifier = ">=2.0.0" },
     { name = "torch", specifier = ">=2.6.0" },
     { name = "transformers", specifier = ">=4.51.0" },
-    { name = "vllm", marker = "sys_platform == 'linux' and extra == 'all'", specifier = ">=0.8.3" },
-    { name = "vllm", marker = "sys_platform == 'linux' and extra == 'generative'", specifier = ">=0.8.3" },
+    { name = "vllm", marker = "sys_platform == 'linux' and extra == 'all'", specifier = ">=0.8.3,<0.8.5" },
+    { name = "vllm", marker = "sys_platform == 'linux' and extra == 'generative'", specifier = ">=0.8.3,<0.8.5" },
 ]
 provides-extras = ["generative", "human-evaluation", "all", "test"]