eval-framework 0.5.1__tar.gz → 0.5.3__tar.gz

{eval_framework-0.5.1 → eval_framework-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.5.1
+Version: 0.5.3
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -252,7 +252,7 @@ Requires-Dist: jinja2>=3.1.6,<4 ; extra == 'optional'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'transformers'
 Requires-Dist: torch>=2.5,<3 ; extra == 'transformers'
 Requires-Dist: accelerate>=1.14.0,<2 ; extra == 'transformers'
-Requires-Dist: vllm>=0.8.5,<0.9 ; extra == 'vllm'
+Requires-Dist: vllm>=0.22,<0.23 ; extra == 'vllm'
 Requires-Dist: torch>=2.5,<3 ; extra == 'vllm'
 Requires-Python: >=3.12, <3.13
 Project-URL: repository, https://github.com/Aleph-Alpha-Research/eval-framework

{eval_framework-0.5.1 → eval_framework-0.5.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.5.1"
+version = "0.5.3"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -66,7 +66,7 @@ transformers = [
 ]
 accelerate = ["accelerate"]
 vllm = [
-  "vllm>=0.8.5,<0.9",
+  "vllm>=0.22,<0.23",
   "torch>=2.5,<3"
 ]
 mistral = [
@@ -92,7 +92,7 @@ eval_framework = "eval_framework.run:run"
 [dependency-groups]
 dev = [
   "mypy>=2.1.0,<3",
-  "pytest>=9.1.0,<10",
+  "pytest>=9.1.1,<10",
   "pytest-mock>=3.15.1",
   "pytest-xdist>=3.8.0,<4",
   "pytest-sugar>1.1,<2",
@@ -100,7 +100,7 @@ dev = [
   "types-python-dateutil>=2.9.0.20260518,<3",
   "types-requests>=2.33.0.20260518,<3",
   "plotly>=6.8.0,<7",
-  "ruff>=0.15.18",
+  "ruff>=0.15.20",
   "pip-licenses>=5.5.5",
 ]
 flash-attn = [
@@ -109,7 +109,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.11.23,<0.11.24"]
+requires = ["uv_build>=0.11.24,<0.11.25"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]

{eval_framework-0.5.1 → eval_framework-0.5.3}/src/eval_framework/evaluation_generator.py

@@ -47,7 +47,7 @@ class EvaluationGenerator:
         else:
             raise NotImplementedError
 
-        self.task_name = eval_.task_class().NAME
+        self.task_name = eval_.display_name()
 
     def _run_metric_calculators(self, responses: list[Completion | Loglikelihood]) -> list[Result]:
         results: list[Result] = self.result_processor.load_metrics_results()

{eval_framework-0.5.1 → eval_framework-0.5.3}/src/eval_framework/llm/huggingface.py

@@ -6,7 +6,7 @@ import warnings
 from collections.abc import Callable, Sequence
 from functools import partial
 from pathlib import Path
-from typing import Any
+from typing import Any, cast
 
 import torch
 from tokenizers import Tokenizer
@@ -133,7 +133,7 @@ class BaseHFLLM(BaseLLM):
 
     def __del__(self) -> None:
         if hasattr(self, "model"):
-            num_gpus = len(self.model.hf_device_map)
+            num_gpus = len(cast(dict[str, Any], self.model.hf_device_map))
             del self.model
             if num_gpus > 1 and torch.distributed.is_initialized():
                 torch.distributed.destroy_process_group()

{eval_framework-0.5.1 → eval_framework-0.5.3}/src/eval_framework/llm/vllm.py

@@ -13,9 +13,9 @@ from typing import Any, Literal, Protocol, cast, override
 import torch
 from vllm import LLM, SamplingParams
 from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
-from vllm.inputs.data import TokensPrompt
+from vllm.inputs import TokensPrompt
 from vllm.outputs import RequestOutput
-from vllm.transformers_utils.tokenizer import get_tokenizer
+from vllm.tokenizers import get_tokenizer
 
 from eval_framework.llm.base import BaseLLM
 from eval_framework.shared.types import (

{eval_framework-0.5.1 → eval_framework-0.5.3}/src/eval_framework/llm/vllm_local_server.py

@@ -107,6 +107,7 @@ class VLLMLocalServerModel(BaseLLM):
         max_model_len: int | None = None,
         gpu_memory_utilization: float | None = None,
         enforce_eager: bool | None = None,
+        attention_backend: str | None = "TRITON_ATTN",
         # Escape hatch:
         vllm_command: str | None = None,
         vllm_extra_args: list[str] | None = None,
@@ -136,6 +137,10 @@ class VLLMLocalServerModel(BaseLLM):
             # vLLM exposes this as a boolean flag; passing a value breaks CLI parsing.
             if enforce_eager:
                 cmd += ["--enforce-eager"]
+        if attention_backend is not None:
+            # Force a specific attention backend. vLLM otherwise auto-selects FlashInfer,
+            # which requires nvcc at runtime (absent in CI), causing the server to fail to start.
+            cmd += ["--attention-backend", str(attention_backend)]
 
         if vllm_extra_args:
             cmd += list(vllm_extra_args)

{eval_framework-0.5.1 → eval_framework-0.5.3}/src/eval_framework/tasks/dataset_revisions.py

@@ -4,7 +4,7 @@ Overwrites ``task-dataset-revisions.json`` in this package with task class name
 
 Usage::
 
-    uv run python -m eval_framework.tasks.benchmarks.dataset_revisions
+    uv run python -m eval_framework.tasks.dataset_revisions
 """
 
 import json

{eval_framework-0.5.1 → eval_framework-0.5.3}/src/eval_framework/tasks/registry.py

@@ -53,6 +53,10 @@ class EvalFactory(ABC):
     def metrics(self) -> list[type["BaseMetric"]]:
         """The eval's metrics"""
 
+    @abstractmethod
+    def display_name(self) -> str:
+        """Human-readable display name. Is allowed to have special characters and whitespaces."""
+
     @abstractmethod
     def create(
         self, num_fewshot: int, custom_subjects: list[str] | None, custom_hf_revision: str | None
@@ -126,6 +130,10 @@ class _Lazy(EvalFactory):
         """The eval's metrics"""
         return self.task_class().get_metrics()
 
+    def display_name(self) -> str:
+        """The eval's human-readable display name (the task's ``NAME``)."""
+        return self.task_class().NAME
+
 
 class _Eager(EvalFactory):
     """Wraps an already-imported task class."""
@@ -167,6 +175,10 @@ class _Eager(EvalFactory):
         """The eval's metrics"""
         return self.task_class().get_metrics()
 
+    def display_name(self) -> str:
+        """The eval's human-readable display name (the task's ``NAME``)."""
+        return self.task_class().NAME
+
 
 class Registry:
     """A registry for tasks with support for lazy loading.

{eval_framework-0.5.1 → eval_framework-0.5.3}/src/eval_framework/tasks/task-dataset-revisions.json

@@ -14,6 +14,7 @@
     "GOLDENSWAG": "f98259e4185c9ab868cf695ac98e9d863eba7986",
     "GOLDENSWAG_IDK": "f98259e4185c9ab868cf695ac98e9d863eba7986",
     "GPQA_OLMES": "633f5ee89ab8ad4522a9f850766b73f62147ffdd",
+    "GSM8KBPB": "740312add88f781978c0658806c59bc2815b9866",
     "GSM8KReasoning": "740312add88f781978c0658806c59bc2815b9866",
     "GSM8K_OLMES": "740312add88f781978c0658806c59bc2815b9866",
     "GlobalMMLU": "0e619dbeb34206cd48705a1a0ea7fb21cae09993",
@@ -25,6 +26,7 @@
     "IFEval": "966cd89545d6b6acfd7638bc708b98261ca58e84",
     "IFEvalDe": "4f52d847003b3c83cc282e9d296853a24b34b19a",
     "MATH500": "6e4ed1a2a79af7d8630a6b768ec859cb5af4d3be",
+    "MATHMinervaBPB": "21a5633873b6a120296cce3e2df9d5550074f4a3",
     "MATHMinerva_OLMES": "21a5633873b6a120296cce3e2df9d5550074f4a3",
     "MBPPBPB": "4bb6404fdc6cacfda99d4ac4205087b89d32030c",
     "MBPP_OLMES": "4bb6404fdc6cacfda99d4ac4205087b89d32030c",
@@ -55,8 +57,10 @@
     "PIQA_IDK": "2e8ac2dffd59bac8c3c6714948f4c551a0848bb0",
     "PIQA_OLMES": "2e8ac2dffd59bac8c3c6714948f4c551a0848bb0",
     "SCIQ_OLMES": "2c94ad3e1aafab77146f384e23536f97a4849815",
+    "SQuAD2_MA": "3ffb306f725f7d2ce8394bc1873b24868140c412",
     "SQuAD_OLMES": "7b6d24c440a36b6815f21b70d25016731768db1f",
     "SocialIQAMC_OLMES": "8835ceb9141d7896d9d968634a9b21ae440e3ec5",
     "TRIVIAQA": "0f7faf33a3908546c6fd5b73a660e0f8ff173c2f",
+    "TriviaQA_MA": "0f7faf33a3908546c6fd5b73a660e0f8ff173c2f",
     "WINOGRANDECloze": "01e74176c63542e6b0bcb004dcdea22d94fb67b5"
 }

eval_framework-0.5.1/src/eval_framework/metrics/completion/cwe_accuracy.py

@@ -1,49 +0,0 @@
-import re
-
-from eval_framework.metrics.base import BaseMetric, MetricResult
-from eval_framework.shared.types import Completion, Error
-
-
-class CWEAccuracy(BaseMetric[Completion]):
-    """Metric for Common Word Extraction tasks"""
-
-    NAME = "CWEAccuracy"
-
-    def calculate(self, response: Completion) -> list[MetricResult]:
-        if response.error is not None:
-            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
-
-        ground_truths = [gt for gt in response.ground_truth_list if gt is not None]
-        if not ground_truths:
-            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
-
-        try:
-            # Get model's answer
-            model_answer = response.completion
-
-            # Check if all words in the correct answer are present in the model's answer
-            is_correct = self._is_answer_correct(ground_truths, model_answer)
-
-            return [
-                MetricResult(
-                    metric_name=self.NAME, value=1.0 if is_correct else 0.0, higher_is_better=True, error=response.error
-                )
-            ]
-        except Exception as e:
-            error = Error(error_class=e.__class__.__name__, message=str(e), traceback="")
-            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=error)]
-
-    def _is_answer_correct(self, correct_answer: list[str], model_answer: str) -> bool:
-        """Check if all words in correct_answer are present in model_answer as whole words"""
-        model_answer = model_answer.strip().lower()
-        correct_answer = [correct.strip().lower() for correct in correct_answer]
-
-        # For each word in the correct answer, check if it exists as a whole word in the model answer
-        for word in correct_answer:
-            # Create a regex pattern that matches the word as a whole word
-            # \b represents a word boundary
-            pattern = r"\b" + re.escape(word) + r"\b"
-            if not re.search(pattern, model_answer):
-                return False
-
-        return True

eval_framework-0.5.1/src/eval_framework/metrics/completion/niah_accuracy.py

@@ -1,163 +0,0 @@
-import re
-import unicodedata
-
-from eval_framework.metrics.base import (
-    BaseMetric,
-    MetricResult,
-)
-from eval_framework.shared.types import Completion, Error, LanguageMetricContext, extract_context_metric
-
-# Dictionary of "none" words in different languages
-NONE_DICT = {
-    "en": ["none"],
-    "ko": ["없음"],
-    "pl": ["brak"],
-    "zh": ["无"],
-    "vi": ["Không có"],
-    "ja": ["なし", "数字はありません"],
-    "ta": ["ஏதுமில்லை"],
-    "hu": ["nincs"],
-    "fr": ["aucun"],
-    "no": ["ingen"],
-    "uk": ["немає", "Нема"],
-    "ru": ["нет"],
-    "de": ["Keine vorhanden"],
-    "es": ["ninguno"],
-    "sv": ["inga"],
-    "fi": ["ei mikään"],
-    "cs": ["žádné", "žádná"],
-    "sr": ["nema"],
-    "pt": ["nenhum"],
-    "it": ["nessuno"],
-    "fa": ["هیچ کدام"],
-    "sw": ["hakuna"],
-    "nl": ["geen"],
-    "st": ["ha ho letho"],
-    "hi": ["कोई नहीं"],
-    "da": ["ingen"],
-}
-
-
-def clean_text(text: str) -> str:
-    """Clean text by removing spaces and normalizing"""
-    return text.strip().lower().replace("\u200c", "").replace(" ", "")
-
-
-class NIAHAccuracy(BaseMetric[Completion]):
-    """Metric for Needle in a Haystack tasks"""
-
-    NAME = "NIAHAccuracy"
-
-    def calculate(self, response: Completion) -> list[MetricResult]:
-        if response.error is not None:
-            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
-
-        context = extract_context_metric(response, LanguageMetricContext)
-
-        ground_truths = [gt for gt in response.ground_truth_list if gt is not None]
-
-        try:
-            # Extract task and language from metadata
-            assert response.context is not None
-            language = context.language
-
-            # Get model's answer
-            model_answer = response.completion
-
-            # Determine which comparison function to use based on the task
-            none_values = set(v for values in NONE_DICT.values() for v in values)
-            if ground_truths[0] in none_values:
-                is_correct = self._compare_none(language, model_answer)
-            else:
-                is_correct = self._compare_numbers(language, ground_truths, model_answer)
-
-            return [
-                MetricResult(
-                    metric_name=self.NAME, value=float(is_correct), higher_is_better=True, error=response.error
-                )
-            ]
-        except Exception as e:
-            error = Error(error_class=e.__class__.__name__, message=str(e), traceback="")
-            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=error)]
-
-    def _compare_numbers(self, lang: str, correct_answer: list[str], model_answer: str) -> bool:
-        """Compare numbers for regular NIAH tasks"""
-        if "-" in lang:
-            inst_lang = lang.split("-")[1]
-        else:
-            inst_lang = lang
-
-        if not model_answer:
-            return False
-
-        processed_model_answer = unicodedata.normalize("NFKC", model_answer)
-
-        none_words = NONE_DICT.get(inst_lang, ["none"])
-        # Check if any word in none_words is present in the processed answer; if yes, auto-fail
-        for word in none_words:
-            if word in processed_model_answer or clean_text(word) in processed_model_answer:
-                return False
-
-        # Extract all numeric substrings from the processed answer
-        numeric_strings = re.findall(r"\d+", processed_model_answer)
-
-        # Remove numbers that consist of a single digit
-        numeric_strings = [num for num in numeric_strings if len(num) > 1]
-
-        # Remove duplicates while preserving the original order
-        numeric_strings = list(dict.fromkeys(numeric_strings))
-
-        # If no numerics are found after processing, return False
-        if not numeric_strings:
-            return False
-
-        # Convert the extracted number strings to integers
-        try:
-            extracted_numbers = [int(num) for num in numeric_strings]
-        except Exception:
-            return False
-
-        # Convert correct_answers elements to integers to ensure numeric comparison
-        try:
-            correct_converted = [int(item) for item in correct_answer]
-        except Exception:
-            return False
-
-        # Check that the number of extracted numbers matches the length of correct_answers
-        if len(extracted_numbers) != len(correct_converted):
-            return False
-
-        # Compare the extracted numbers with the correct answers
-        if set(extracted_numbers) == set(correct_converted):
-            return True
-        else:
-            return False
-
-    def _compare_none(self, lang: str, model_answer: str) -> bool:
-        """Compare for NIAH none tasks"""
-        # Lower-case all inputs for consistent, case-insensitive processing
-        if "-" in lang:
-            inst_lang = lang.split("-")[1]
-        else:
-            inst_lang = lang
-
-        processed_model_answer = clean_text(unicodedata.normalize("NFKC", model_answer))
-        none_words = [clean_text(word) for word in NONE_DICT[inst_lang]]
-
-        # Remove single digit numbers from the processed answer
-        processed_model_answer = re.sub(r"\b\d\b", "", processed_model_answer)
-
-        # Extract all multi-digit numeric substrings from the processed answer
-        numeric_strings = re.findall(r"\d\d+", processed_model_answer)
-
-        # If any multi-digit numbers are found, return False
-        if numeric_strings:
-            return False
-
-        # Check if any of the words in none_words are present
-        for word in none_words:
-            if word in processed_model_answer:
-                return True
-
-        # If none of the none_words are found, return False
-        return False