EuroEval 15.8.1__tar.gz → 15.8.2__tar.gz

{euroeval-15.8.1 → euroeval-15.8.2}/.github/workflows/ci.yaml

@@ -10,6 +10,10 @@ on:
     branches:
       - main
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
 jobs:
   code-check:
     if: github.event.pull_request.draft == false

{euroeval-15.8.1 → euroeval-15.8.2}/.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
       - id: trailing-whitespace
       - id: debug-statements
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.8
+    rev: v0.11.9
     hooks:
       - id: ruff
         args:

{euroeval-15.8.1 → euroeval-15.8.2}/CHANGELOG.md

@@ -10,6 +10,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 
 
+## [v15.8.2] - 2025-05-12
+### Fixed
+- Catch error when caching generative model outputs, when the number of model inputs and
+  outputs do not match.
+- Disallow vLLM >=0.8.5, as it breaks generation output for several models.
+
+
 ## [v15.8.1] - 2025-05-08
 ### Fixed
 - NER labels were included twice in the prompt templates (which was due to there being

{euroeval-15.8.1 → euroeval-15.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: EuroEval
-Version: 15.8.1
+Version: 15.8.2
 Summary: The robust European language model benchmark.
 Project-URL: Repository, https://github.com/EuroEval/EuroEval
 Project-URL: Issues, https://github.com/EuroEval/EuroEval/issues
@@ -62,12 +62,12 @@ Requires-Dist: bitsandbytes>=0.43.1; (platform_system == 'Linux') and extra == '
 Requires-Dist: fbgemm-gpu>=1.0.0; (platform_system == 'Linux') and extra == 'all'
 Requires-Dist: gradio>=4.26.0; extra == 'all'
 Requires-Dist: outlines>=0.1.11; extra == 'all'
-Requires-Dist: vllm>=0.8.3; (platform_system == 'Linux') and extra == 'all'
+Requires-Dist: vllm<0.8.5,>=0.8.3; (platform_system == 'Linux') and extra == 'all'
 Provides-Extra: generative
 Requires-Dist: bitsandbytes>=0.43.1; (platform_system == 'Linux') and extra == 'generative'
 Requires-Dist: fbgemm-gpu>=1.0.0; (platform_system == 'Linux') and extra == 'generative'
 Requires-Dist: outlines>=0.1.11; extra == 'generative'
-Requires-Dist: vllm>=0.8.3; (platform_system == 'Linux') and extra == 'generative'
+Requires-Dist: vllm<0.8.5,>=0.8.3; (platform_system == 'Linux') and extra == 'generative'
 Provides-Extra: human-evaluation
 Requires-Dist: gradio>=4.26.0; extra == 'human-evaluation'
 Provides-Extra: test

{euroeval-15.8.1 → euroeval-15.8.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "EuroEval"
-version = "15.8.1"
+version = "15.8.2"
 description = "The robust European language model benchmark."
 readme = "README.md"
 authors = [
@@ -46,7 +46,7 @@ dependencies = [
 generative = [
     "outlines>=0.1.11",
     "bitsandbytes>=0.43.1; platform_system == 'Linux'",
-    "vllm>=0.8.3; platform_system == 'Linux'",
+    "vllm>=0.8.3,<0.8.5; platform_system == 'Linux'",
     "fbgemm-gpu>=1.0.0; platform_system == 'Linux'",
 ]
 human_evaluation = [
@@ -55,7 +55,7 @@ human_evaluation = [
 all = [
     "outlines>=0.1.11",
     "bitsandbytes>=0.43.1; platform_system == 'Linux'",
-    "vllm>=0.8.3; platform_system == 'Linux'",
+    "vllm>=0.8.3,<0.8.5; platform_system == 'Linux'",
     "fbgemm-gpu>=1.0.0; platform_system == 'Linux'",
     "gradio>=4.26.0",
 ]

{euroeval-15.8.1 → euroeval-15.8.2}/src/euroeval/benchmark_modules/litellm.py

@@ -401,6 +401,12 @@ class LiteLLMModel(BenchmarkModule):
             model_responses=ordered_responses, model_id=self.model_config.model_id
         )
 
+        if len(messages) != len(model_output.sequences):
+            raise InvalidBenchmark(
+                f"Number of model inputs ({len(messages):,}) does not match the "
+                f"number of model outputs ({len(model_output.sequences):,})."
+            )
+
         return model_output
 
     def _handle_exception(
@@ -616,8 +622,7 @@ class LiteLLMModel(BenchmarkModule):
         scores = []
         for model_response in model_responses:
             if not model_response.choices:
-                # This happens for reasoning models, when they don't finish thinking
-                # and run out of tokens. Happens quite rarely, but we need to handle it.
+                sequences.append("")
                 logger.warning(
                     f"The model {model_id!r} did not end up "
                     "generating any text. This is likely because the model ran "

{euroeval-15.8.1 → euroeval-15.8.2}/src/euroeval/model_cache.py

@@ -168,6 +168,15 @@ class ModelCache:
         input_column = "messages" if "messages" in model_inputs else "text"
         model_inputs = model_inputs[input_column]
 
+        # Double check that the number of inputs and outputs match
+        if not len(model_inputs) == len(model_output.sequences):
+            logger.warning(
+                f"Number of model inputs ({len(model_inputs)}) does not match the "
+                f"number of model outputs ({len(model_output.sequences)}). We will not "
+                f"cache the model outputs."
+            )
+            return
+
         # Store the generated sequences in the cache, one by one
         with tqdm(
             iterable=model_inputs,

{euroeval-15.8.1 → euroeval-15.8.2}/uv.lock

@@ -906,7 +906,7 @@ wheels = [
 
 [[package]]
 name = "euroeval"
-version = "15.8.1"
+version = "15.8.2"
 source = { editable = "." }
 dependencies = [
     { name = "accelerate" },
@@ -1034,8 +1034,8 @@ requires-dist = [
     { name = "termcolor", specifier = ">=2.0.0" },
     { name = "torch", specifier = ">=2.6.0" },
     { name = "transformers", specifier = ">=4.51.0" },
-    { name = "vllm", marker = "sys_platform == 'linux' and extra == 'all'", specifier = ">=0.8.3" },
-    { name = "vllm", marker = "sys_platform == 'linux' and extra == 'generative'", specifier = ">=0.8.3" },
+    { name = "vllm", marker = "sys_platform == 'linux' and extra == 'all'", specifier = ">=0.8.3,<0.8.5" },
+    { name = "vllm", marker = "sys_platform == 'linux' and extra == 'generative'", specifier = ">=0.8.3,<0.8.5" },
 ]
 provides-extras = ["generative", "human-evaluation", "all", "test"]