eval-framework 0.5.1__tar.gz → 0.5.2__tar.gz

{eval_framework-0.5.1 → eval_framework-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.5.1
+Version: 0.5.2
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -252,7 +252,7 @@ Requires-Dist: jinja2>=3.1.6,<4 ; extra == 'optional'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'transformers'
 Requires-Dist: torch>=2.5,<3 ; extra == 'transformers'
 Requires-Dist: accelerate>=1.14.0,<2 ; extra == 'transformers'
-Requires-Dist: vllm>=0.8.5,<0.9 ; extra == 'vllm'
+Requires-Dist: vllm>=0.22,<0.23 ; extra == 'vllm'
 Requires-Dist: torch>=2.5,<3 ; extra == 'vllm'
 Requires-Python: >=3.12, <3.13
 Project-URL: repository, https://github.com/Aleph-Alpha-Research/eval-framework

{eval_framework-0.5.1 → eval_framework-0.5.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.5.1"
+version = "0.5.2"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -66,7 +66,7 @@ transformers = [
 ]
 accelerate = ["accelerate"]
 vllm = [
-  "vllm>=0.8.5,<0.9",
+  "vllm>=0.22,<0.23",
   "torch>=2.5,<3"
 ]
 mistral = [
@@ -92,7 +92,7 @@ eval_framework = "eval_framework.run:run"
 [dependency-groups]
 dev = [
   "mypy>=2.1.0,<3",
-  "pytest>=9.1.0,<10",
+  "pytest>=9.1.1,<10",
   "pytest-mock>=3.15.1",
   "pytest-xdist>=3.8.0,<4",
   "pytest-sugar>1.1,<2",

{eval_framework-0.5.1 → eval_framework-0.5.2}/src/eval_framework/evaluation_generator.py

@@ -47,7 +47,7 @@ class EvaluationGenerator:
         else:
             raise NotImplementedError
 
-        self.task_name = eval_.task_class().NAME
+        self.task_name = eval_.display_name()
 
     def _run_metric_calculators(self, responses: list[Completion | Loglikelihood]) -> list[Result]:
         results: list[Result] = self.result_processor.load_metrics_results()

{eval_framework-0.5.1 → eval_framework-0.5.2}/src/eval_framework/llm/huggingface.py

@@ -6,7 +6,7 @@ import warnings
 from collections.abc import Callable, Sequence
 from functools import partial
 from pathlib import Path
-from typing import Any
+from typing import Any, cast
 
 import torch
 from tokenizers import Tokenizer
@@ -133,7 +133,7 @@ class BaseHFLLM(BaseLLM):
 
     def __del__(self) -> None:
         if hasattr(self, "model"):
-            num_gpus = len(self.model.hf_device_map)
+            num_gpus = len(cast(dict[str, Any], self.model.hf_device_map))
             del self.model
             if num_gpus > 1 and torch.distributed.is_initialized():
                 torch.distributed.destroy_process_group()

{eval_framework-0.5.1 → eval_framework-0.5.2}/src/eval_framework/llm/vllm.py

@@ -13,9 +13,9 @@ from typing import Any, Literal, Protocol, cast, override
 import torch
 from vllm import LLM, SamplingParams
 from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
-from vllm.inputs.data import TokensPrompt
+from vllm.inputs import TokensPrompt
 from vllm.outputs import RequestOutput
-from vllm.transformers_utils.tokenizer import get_tokenizer
+from vllm.tokenizers import get_tokenizer
 
 from eval_framework.llm.base import BaseLLM
 from eval_framework.shared.types import (

{eval_framework-0.5.1 → eval_framework-0.5.2}/src/eval_framework/llm/vllm_local_server.py

@@ -107,6 +107,7 @@ class VLLMLocalServerModel(BaseLLM):
         max_model_len: int | None = None,
         gpu_memory_utilization: float | None = None,
         enforce_eager: bool | None = None,
+        attention_backend: str | None = "TRITON_ATTN",
         # Escape hatch:
         vllm_command: str | None = None,
         vllm_extra_args: list[str] | None = None,
@@ -136,6 +137,10 @@ class VLLMLocalServerModel(BaseLLM):
             # vLLM exposes this as a boolean flag; passing a value breaks CLI parsing.
             if enforce_eager:
                 cmd += ["--enforce-eager"]
+        if attention_backend is not None:
+            # Force a specific attention backend. vLLM otherwise auto-selects FlashInfer,
+            # which requires nvcc at runtime (absent in CI), causing the server to fail to start.
+            cmd += ["--attention-backend", str(attention_backend)]
 
         if vllm_extra_args:
             cmd += list(vllm_extra_args)

{eval_framework-0.5.1 → eval_framework-0.5.2}/src/eval_framework/tasks/registry.py

@@ -53,6 +53,10 @@ class EvalFactory(ABC):
     def metrics(self) -> list[type["BaseMetric"]]:
         """The eval's metrics"""
 
+    @abstractmethod
+    def display_name(self) -> str:
+        """Human-readable display name. Is allowed to have special characters and whitespaces."""
+
     @abstractmethod
     def create(
         self, num_fewshot: int, custom_subjects: list[str] | None, custom_hf_revision: str | None
@@ -126,6 +130,10 @@ class _Lazy(EvalFactory):
         """The eval's metrics"""
         return self.task_class().get_metrics()
 
+    def display_name(self) -> str:
+        """The eval's human-readable display name (the task's ``NAME``)."""
+        return self.task_class().NAME
+
 
 class _Eager(EvalFactory):
     """Wraps an already-imported task class."""
@@ -167,6 +175,10 @@ class _Eager(EvalFactory):
         """The eval's metrics"""
         return self.task_class().get_metrics()
 
+    def display_name(self) -> str:
+        """The eval's human-readable display name (the task's ``NAME``)."""
+        return self.task_class().NAME
+
 
 class Registry:
     """A registry for tasks with support for lazy loading.