eval-framework 0.7.2__tar.gz → 0.8.0__tar.gz

{eval_framework-0.7.2 → eval_framework-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.7.2
+Version: 0.8.0
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -236,13 +236,10 @@ Requires-Dist: numpy>=2.2.6
 Requires-Dist: antlr4-python3-runtime==4.11.0
 Requires-Dist: scipy>=1.18.0,<2
 Requires-Dist: accelerate ; extra == 'accelerate'
-Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,vllm,optional,mistral] ; extra == 'all'
+Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,optional] ; extra == 'all'
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: mistral-common>=1.11.6,<2 ; extra == 'mistral'
-Requires-Dist: huggingface-hub>=0.36.2,<0.37 ; extra == 'mistral'
-Requires-Dist: eval-framework[vllm] ; extra == 'mistral'
 Requires-Dist: openai>=2.46.0,<3 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
@@ -251,19 +248,15 @@ Requires-Dist: jinja2>=3.1.6,<4 ; extra == 'optional'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'transformers'
 Requires-Dist: torch>=2.5,<3 ; extra == 'transformers'
 Requires-Dist: accelerate>=1.14.0,<2 ; extra == 'transformers'
-Requires-Dist: vllm>=0.22,<0.23 ; extra == 'vllm'
-Requires-Dist: torch>=2.5,<3 ; extra == 'vllm'
 Requires-Python: >=3.12, <3.13
 Project-URL: repository, https://github.com/Aleph-Alpha-Research/eval-framework
 Provides-Extra: accelerate
 Provides-Extra: all
 Provides-Extra: api
 Provides-Extra: determined
-Provides-Extra: mistral
 Provides-Extra: openai
 Provides-Extra: optional
 Provides-Extra: transformers
-Provides-Extra: vllm
 Description-Content-Type: text/markdown
 
 <!-- Badges -->
@@ -316,12 +309,13 @@ pip install eval_framework
 There are optional extras available to unlock specific features of the library:
 - `api` for inference using the aleph-alpha client.
 - `determined` for running jobs via determined.
-- `mistral` for inference on Mistral models.
+- `openai` for inference against OpenAI-compatible HTTP endpoints.
 - `transformers` for inference using the transformers library.
-- `vllm` for inference via VLLM.
 
 As a short hand, the `all` extra installs all of the above.
 
+For local vLLM inference, see the sibling [`local-vllm-server/`](./local-vllm-server/README.md) package.
+
 We use `uv` to better resolve dependencies when downloading the extras. You can install uv with:
 ```bash
 curl -LsSf https://astral.sh/uv/install.sh | sh

{eval_framework-0.7.2 → eval_framework-0.8.0}/README.md

@@ -48,12 +48,13 @@ pip install eval_framework
 There are optional extras available to unlock specific features of the library:
 - `api` for inference using the aleph-alpha client.
 - `determined` for running jobs via determined.
-- `mistral` for inference on Mistral models.
+- `openai` for inference against OpenAI-compatible HTTP endpoints.
 - `transformers` for inference using the transformers library.
-- `vllm` for inference via VLLM.
 
 As a short hand, the `all` extra installs all of the above.
 
+For local vLLM inference, see the sibling [`local-vllm-server/`](./local-vllm-server/README.md) package.
+
 We use `uv` to better resolve dependencies when downloading the extras. You can install uv with:
 ```bash
 curl -LsSf https://astral.sh/uv/install.sh | sh

{eval_framework-0.7.2 → eval_framework-0.8.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.7.2"
+version = "0.8.0"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -64,22 +64,15 @@ transformers = [
   "accelerate>=1.14.0,<2",
 ]
 accelerate = ["accelerate"]
-vllm = [
-  "vllm>=0.22,<0.23",
-  "torch>=2.5,<3"
-]
-mistral = [
-  "mistral-common>=1.11.6,<2",
-  "huggingface-hub>=0.36.2,<0.37",
-  "eval_framework[vllm]",
-]
+
+
 # from template-formatting
 optional = [
   "transformers>=4.45.2,<5",
   "jinja2>=3.1.6,<4"
 ]
 all = [
-  "eval_framework[determined,api,openai,transformers,accelerate,vllm,optional,mistral]"
+  "eval_framework[determined,api,openai,transformers,accelerate,optional]"
 ]
 
 [project.urls]
@@ -98,7 +91,7 @@ dev = [
   "types-pyyaml>=6.0.12.20260518,<7",
   "types-python-dateutil>=2.9.0.20260716,<3",
   "types-requests>=2.33.0.20260712,<3",
-  "plotly>=6.8.0,<7",
+  "plotly>=6.9.0,<7",
   "ruff>=0.15.22",
   "pip-licenses>=5.5.5",
 ]
@@ -160,7 +153,6 @@ markers = [
     "cpu_slow: runs for a long time (on CPU)",
     "slow_download: smoke tests that download large datasets (>15s); excluded from CI, run manually with -m slow_download",
     "external_api: needs external services for execution",
-    "vllm: tests that specifically require vLLM functionality",
     "formatter_hash: formatter consistency tests using hash comparisons",
 ]
 filterwarnings = [

{eval_framework-0.7.2 → eval_framework-0.8.0}/src/eval_framework/llm/models.py

@@ -18,11 +18,6 @@ if is_extra_installed(extra="transformers"):
         Qwen3_0_6B,
     )
 
-if is_extra_installed("mistral"):
-    from eval_framework.llm.mistral import MagistralVLLM  # noqa F401
 
 if is_extra_installed("openai"):
     from eval_framework.llm.openai import OpenAIModel  # noqa F401
-
-if is_extra_installed("vllm"):
-    from eval_framework.llm.vllm import VLLMRegistryModel, Qwen3_0_6B_VLLM, Qwen3_0_6B_VLLM_No_Thinking  # noqa F401

{eval_framework-0.7.2 → eval_framework-0.8.0}/src/eval_framework/run.py

@@ -13,6 +13,7 @@ except ImportError:
 from eval_framework.context.local import LocalContext
 from eval_framework.main import main
 from eval_framework.tasks.task_loader import load_extra_tasks
+from eval_framework.tasks.utils import close_pools
 from eval_framework.utils.logging import setup_logging
 
 logger = logging.getLogger(__name__)
@@ -392,19 +393,22 @@ def _run_single_task(kwargs: dict) -> None:
 def run_with_kwargs(kwargs: dict) -> None:
     task_suite_path = kwargs.pop("task_suite", None)
 
-    if task_suite_path is not None:
-        from eval_framework.suite import TaskSuite, run_suite
-
-        output_dir = kwargs.get("output_dir", "outputs")
-        log_level = kwargs.get("verbosity", 1)
-        setup_logging(output_dir, log_level=log_level)
-
-        suite = TaskSuite.load(task_suite_path)
-        logger.info(f"Loaded task suite '{suite.name}' from {task_suite_path}")
-        result = run_suite(suite, kwargs)
-        logger.info(f"Suite '{suite.name}' completed. Aggregates: {result.aggregates}")
-    else:
-        _run_single_task(kwargs)
+    try:
+        if task_suite_path is not None:
+            from eval_framework.suite import TaskSuite, run_suite
+
+            output_dir = kwargs.get("output_dir", "outputs")
+            log_level = kwargs.get("verbosity", 1)
+            setup_logging(output_dir, log_level=log_level)
+
+            suite = TaskSuite.load(task_suite_path)
+            logger.info(f"Loaded task suite '{suite.name}' from {task_suite_path}")
+            result = run_suite(suite, kwargs)
+            logger.info(f"Suite '{suite.name}' completed. Aggregates: {result.aggregates}")
+        else:
+            _run_single_task(kwargs)
+    finally:
+        close_pools()
 
 
 def run() -> None:

{eval_framework-0.7.2 → eval_framework-0.8.0}/src/eval_framework/tasks/utils.py

@@ -78,11 +78,14 @@ def get_or_create_pool(
 
 
 def close_pools() -> None:
-    for pool in _pools.values():
+    with _pools_lock:
+        pools = list(_pools.values())
+        _pools.clear()
+    for pool in pools:
         try:
             pool.close()
         except Exception:
-            pass
+            logger.exception("Error closing sandbox container pool")
 
 
 atexit.register(close_pools)

eval_framework-0.7.2/src/eval_framework/llm/mistral.py

@@ -1,88 +0,0 @@
-from functools import partial
-from pathlib import Path
-from typing import Any, Literal, override
-
-from vllm import SamplingParams
-
-from eval_framework.llm.vllm import TokenizedContainer, VLLMModel, VLLMTokenizerAPI
-from template_formatting.formatter import BaseFormatter, Message
-from template_formatting.mistral_formatter import MagistralFormatter, MistralSerializer
-
-__all__ = [
-    "MistralAdapter",
-    "MistralVLLM",
-]
-
-
-class MistralAdapter(VLLMTokenizerAPI[list[Message]]):
-    def __init__(self, target_mdl: str) -> None:
-        self.serializer = MistralSerializer(llm_target=target_mdl)
-        self.tokenizer = self.serializer.get_tokenizer()
-
-    def encode_formatted_struct(self, struct: list[Message]) -> TokenizedContainer:
-        mistral_msg_lst = self.serializer.convert_from_aa(msg_lst=struct)
-        mistral_request = self.serializer.build_mistral_request(mistral_msg_lst=mistral_msg_lst)
-        mistral_tokenized_obj = self.tokenizer.encode_instruct(mistral_request)
-        return TokenizedContainer(tokens=mistral_tokenized_obj.tokens, text=mistral_tokenized_obj.text)
-
-    def encode_plain_text(self, text: str) -> TokenizedContainer:
-        choice_tokens = self.tokenizer.tokenizer.encode(text, False, False)
-        return TokenizedContainer(tokens=choice_tokens, text=text)
-
-
-class MistralVLLM(VLLMModel):
-    def __init__(
-        self,
-        # Model source (3 options: file path, HuggingFace model name, Wandb artifact name):
-        checkpoint_path: str | Path | None = None,
-        model_name: str | None = None,
-        artifact_name: str | None = None,
-        # Formatter (2 options):
-        formatter: BaseFormatter | None = None,
-        formatter_name: str | None = None,
-        formatter_kwargs: dict[str, Any] | None = None,
-        # Explicit name for the `name` property:
-        checkpoint_name: str | None = None,
-        # VLLM parameters (not complete):
-        max_model_len: int | None = None,
-        tensor_parallel_size: int = 1,
-        gpu_memory_utilization: float = 0.9,
-        batch_size: int = 1,
-        sampling_params: SamplingParams | dict[str, Any] | None = None,
-        bytes_per_token: float | None = None,
-        **kwargs: Any,
-    ) -> None:
-        model_args = {"tokenizer_mode": "mistral", "config_format": "mistral", "load_format": "mistral"}
-        super().__init__(
-            checkpoint_path=checkpoint_path,
-            model_name=model_name,
-            artifact_name=artifact_name,
-            formatter=formatter,
-            formatter_name=formatter_name,
-            formatter_kwargs=formatter_kwargs,
-            checkpoint_name=checkpoint_name,
-            max_model_len=max_model_len,
-            tensor_parallel_size=tensor_parallel_size,
-            gpu_memory_utilization=gpu_memory_utilization,
-            batch_size=batch_size,
-            sampling_params=sampling_params,
-            bytes_per_token=bytes_per_token,
-            **{**model_args, **kwargs},
-        )
-
-    @override
-    @property
-    def tokenizer(self) -> VLLMTokenizerAPI:
-        if self._tokenizer is None:
-            self._tokenizer = MistralAdapter(target_mdl=self.LLM_NAME)
-        return self._tokenizer
-
-    @property
-    def formatter_output_mode(self) -> Literal["string", "list"]:
-        """Determine the correct output mode for the formatter based on tokenizer type."""
-        return "list"
-
-
-class MagistralVLLM(MistralVLLM):
-    LLM_NAME = "mistralai/Magistral-Small-2506"
-    DEFAULT_FORMATTER = partial(MagistralFormatter, "mistralai/Magistral-Small-2506")