eval-framework 0.3.7__tar.gz → 0.5.0__tar.gz

{eval_framework-0.3.7 → eval_framework-0.5.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.3.7
-Summary: Evalulation Framework
+Version: 0.5.0
+Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
                                     Version 2.0, January 2004
@@ -211,49 +211,47 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Typing :: Typed
-Requires-Dist: pyyaml>=6.0.1,<7
-Requires-Dist: xmltodict>=0.13.0,<0.16
-Requires-Dist: pydantic>=2.7,<3
-Requires-Dist: datasets>=4.0.0,<5
-Requires-Dist: sacrebleu>=2.4.3,<3
-Requires-Dist: pycountry>=24.6.1,<25
-Requires-Dist: nltk>=3.9.1,<4
-Requires-Dist: python-dotenv>=1.0.1,<2
-Requires-Dist: lingua-language-detector>=2.0.2,<3
-Requires-Dist: google-crc32c>=1.5.0,<2
-Requires-Dist: kubernetes>=31.0.0,<32
+Requires-Dist: pyyaml>=6.0.3,<7
+Requires-Dist: xmltodict>=1.0.4,<1.1
+Requires-Dist: pydantic>=2.13.4,<3
+Requires-Dist: datasets>=5.0.0,<6
+Requires-Dist: sacrebleu>=2.6.0,<3
+Requires-Dist: pycountry>=26.2.16,<27
+Requires-Dist: nltk>=3.9.4,<4
+Requires-Dist: python-dotenv>=1.2.2,<2
+Requires-Dist: lingua-language-detector>=2.2.0,<3
+Requires-Dist: google-crc32c>=1.8.0,<2
 Requires-Dist: langdetect>=1.0.9,<2
-Requires-Dist: spacy>=3.8.3,<4
-Requires-Dist: jsonschema>=4.23.0,<5
-Requires-Dist: mysql-connector-python>=9.0.0,<10
-Requires-Dist: psycopg2-binary>=2.9.9,<3
+Requires-Dist: spacy>=3.8.14,<4
+Requires-Dist: jsonschema>=4.26.0,<5
+Requires-Dist: mysql-connector-python>=9.7.0,<10
+Requires-Dist: psycopg2-binary>=2.9.12,<3
 Requires-Dist: sympy>=1.13.1,<2
-Requires-Dist: llm-sandbox[docker]==0.3.37
+Requires-Dist: llm-sandbox[docker]==0.3.39
 Requires-Dist: jsonlines>=4,<5
-Requires-Dist: lxml>=6,<7
-Requires-Dist: python-iso639>=2025.2.18
-Requires-Dist: wandb>=0.23.0,<1
-Requires-Dist: boto3>=1.40.54,<2
-Requires-Dist: numpy>=1.26.4
+Requires-Dist: lxml>=6.1.1,<7
+Requires-Dist: python-iso639>=2026.4.20
+Requires-Dist: wandb>=0.27.2,<1
+Requires-Dist: boto3>=1.43.19,<2
+Requires-Dist: numpy>=2.2.6
 Requires-Dist: antlr4-python3-runtime==4.11.0
-Requires-Dist: scipy>=1.14.0,<2
+Requires-Dist: scipy>=1.17.1,<2
 Requires-Dist: accelerate ; extra == 'accelerate'
-Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,vllm,comet,optional,mistral] ; extra == 'all'
+Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,vllm,optional,mistral] ; extra == 'all'
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
-Requires-Dist: unbabel-comet>=2.2.6,<3 ; extra == 'comet'
-Requires-Dist: determined>=0.38,<0.39 ; extra == 'determined'
-Requires-Dist: tensorboard==2.19.0 ; extra == 'determined'
-Requires-Dist: mistral-common>=1.7,<2 ; extra == 'mistral'
-Requires-Dist: huggingface-hub>=0.33.2,<0.34 ; extra == 'mistral'
+Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
+Requires-Dist: tensorboard==2.20.0 ; extra == 'determined'
+Requires-Dist: mistral-common>=1.11.3,<2 ; extra == 'mistral'
+Requires-Dist: huggingface-hub>=0.36.2,<0.37 ; extra == 'mistral'
 Requires-Dist: eval-framework[vllm] ; extra == 'mistral'
 Requires-Dist: openai>=1.62,<3 ; extra == 'openai'
-Requires-Dist: tiktoken>=0.9,<1 ; extra == 'openai'
+Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'
 Requires-Dist: jinja2>=3.1.6,<4 ; extra == 'optional'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'transformers'
 Requires-Dist: torch>=2.5,<3 ; extra == 'transformers'
-Requires-Dist: accelerate>=0.30.0,<1 ; extra == 'transformers'
+Requires-Dist: accelerate>=1.14.0,<2 ; extra == 'transformers'
 Requires-Dist: vllm>=0.8.5,<0.9 ; extra == 'vllm'
 Requires-Dist: torch>=2.5,<3 ; extra == 'vllm'
 Requires-Python: >=3.12, <3.13
@@ -261,7 +259,6 @@ Project-URL: repository, https://github.com/Aleph-Alpha-Research/eval-framework
 Provides-Extra: accelerate
 Provides-Extra: all
 Provides-Extra: api
-Provides-Extra: comet
 Provides-Extra: determined
 Provides-Extra: mistral
 Provides-Extra: openai
@@ -319,7 +316,6 @@ pip install eval_framework
 
 There are optional extras available to unlock specific features of the library:
 - `api` for inference using the aleph-alpha client.
-- `comet` for the COMET metric.
 - `determined` for running jobs via determined.
 - `mistral` for inference on Mistral models.
 - `transformers` for inference using the transformers library.

{eval_framework-0.3.7 → eval_framework-0.5.0}/README.md

@@ -47,7 +47,6 @@ pip install eval_framework
 
 There are optional extras available to unlock specific features of the library:
 - `api` for inference using the aleph-alpha client.
-- `comet` for the COMET metric.
 - `determined` for running jobs via determined.
 - `mistral` for inference on Mistral models.
 - `transformers` for inference using the transformers library.

{eval_framework-0.3.7 → eval_framework-0.5.0}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "eval-framework"
-version = "0.3.7"
-description = "Evalulation Framework"
+version = "0.5.0"
+description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
 requires-python = ">=3.12,<3.13"
@@ -18,53 +18,51 @@ classifiers = [
   "Typing :: Typed",
 ]
 dependencies = [
-  "pyyaml>=6.0.1,<7",
-  "xmltodict>=0.13.0,<0.16",
-  "pydantic>=2.7,<3",
-  "datasets>=4.0.0,<5",
-  "sacrebleu>=2.4.3,<3",
-  "pycountry>=24.6.1,<25",
-  "nltk>=3.9.1,<4",
-  "python-dotenv>=1.0.1,<2",
-  "lingua-language-detector>=2.0.2,<3",
-  "google-crc32c>=1.5.0,<2",
-  "kubernetes>=31.0.0,<32", # required by llm-sandbox though actually not needed
+  "pyyaml>=6.0.3,<7",
+  "xmltodict>=1.0.4,<1.1",
+  "pydantic>=2.13.4,<3",
+  "datasets>=5.0.0,<6",
+  "sacrebleu>=2.6.0,<3",
+  "pycountry>=26.2.16,<27",
+  "nltk>=3.9.4,<4",
+  "python-dotenv>=1.2.2,<2",
+  "lingua-language-detector>=2.2.0,<3",
+  "google-crc32c>=1.8.0,<2",
   "langdetect>=1.0.9,<2", # required by the original ifeval implementation
-  "spacy>=3.8.3,<4",
-  "jsonschema>=4.23.0,<5",
-  "mysql-connector-python>=9.0.0,<10", # required for sql-related tasks
-  "psycopg2-binary>=2.9.9,<3", # required for sql-related tasks
+  "spacy>=3.8.14,<4",
+  "jsonschema>=4.26.0,<5",
+  "mysql-connector-python>=9.7.0,<10", # required for sql-related tasks
+  "psycopg2-binary>=2.9.12,<3", # required for sql-related tasks
   "sympy>=1.13.1,<2",
-  "llm-sandbox[docker]==0.3.37",
+  "llm-sandbox[docker]==0.3.39",
   "jsonlines>=4,<5",
-  "lxml>=6,<7",
-  "python-iso639>=2025.2.18",
-  "wandb>=0.23.0,<1",
-  "boto3>=1.40.54,<2",
-  "numpy>=1.26.4",
+  "lxml>=6.1.1,<7",
+  "python-iso639>=2026.4.20",
+  "wandb>=0.27.2,<1",
+  "boto3>=1.43.19,<2",
+  "numpy>=2.2.6",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
   # https://github.com/sympy/sympy/blob/0204fa34e8f6f6f8ccb4de01209be9a2345c9d6e/doc/src/contributing/dependencies.md?plain=1#L125
   "antlr4-python3-runtime==4.11.0",
-  "scipy>=1.14.0,<2",  # required for the aggregation of pass@k metrics
-
+  "scipy>=1.17.1,<2",  # required for the aggregation of pass@k metrics
 ]
 
 [project.optional-dependencies]
 # Model-specific extras
 determined = [
-  "determined>=0.38,<0.39",
-  "tensorboard==2.19.0"
+  "determined>=0.38.1,<0.39",
+  "tensorboard==2.20.0"
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
   "openai>=1.62,<3",
-  "tiktoken>=0.9,<1",
+  "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]
 transformers = [
   "transformers>=4.45.2,<5",
   "torch>=2.5,<3",
-  "accelerate>=0.30.0,<1",
+  "accelerate>=1.14.0,<2",
 ]
 accelerate = ["accelerate"]
 vllm = [
@@ -72,21 +70,17 @@ vllm = [
   "torch>=2.5,<3"
 ]
 mistral = [
-  "mistral-common>=1.7,<2",
-  "huggingface-hub>=0.33.2,<0.34",
+  "mistral-common>=1.11.3,<2",
+  "huggingface-hub>=0.36.2,<0.37",
   "eval_framework[vllm]",
 ]
-# Benchmark/metric specific extras
-comet = [
-  "unbabel-comet>=2.2.6,<3",
-]
 # from template-formatting
 optional = [
   "transformers>=4.45.2,<5",
   "jinja2>=3.1.6,<4"
 ]
 all = [
-  "eval_framework[determined,api,openai,transformers,accelerate,vllm,comet,optional,mistral]"
+  "eval_framework[determined,api,openai,transformers,accelerate,vllm,optional,mistral]"
 ]
 
 [project.urls]
@@ -97,24 +91,25 @@ eval_framework = "eval_framework.run:run"
 
 [dependency-groups]
 dev = [
-  "mypy>=1.10,<2",
-  "pytest>=8.3.3,<9",
-  "pytest-mock>=3.14.1",
-  "pytest-xdist>=3.6.1,<4",
+  "mypy>=2.1.0,<3",
+  "pytest>=9.1.0,<10",
+  "pytest-mock>=3.15.1",
+  "pytest-xdist>=3.8.0,<4",
   "pytest-sugar>1.1,<2",
-  "types-pyyaml>=6.0.12.20240917,<7",
-  "types-python-dateutil>=2.9.0.20241206,<3",
-  "types-requests>=2.32.0.20250328,<3",
-  "plotly>=5.24.1,<6",
-  "ruff>=0.12.8",
+  "types-pyyaml>=6.0.12.20260518,<7",
+  "types-python-dateutil>=2.9.0.20260518,<3",
+  "types-requests>=2.33.0.20260518,<3",
+  "plotly>=6.8.0,<7",
+  "ruff>=0.15.18",
+  "pip-licenses>=5.5.5",
 ]
 flash-attn = [
-  "flash-attn>=2.7.2.post1,<2.8",
+  "flash-attn>=2.8.3,<2.9",
   "torch"
 ]
 
 [build-system]
-requires = ["uv_build>=0.9.0,<0.10.0"]
+requires = ["uv_build>=0.11.22,<0.11.23"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]
@@ -125,22 +120,6 @@ override-dependencies = [
   "requests>=2.32,<3",  # fix for determined
 ]
 
-[tool.uv.sources]
-torch = [
-  { index = "pytorch-default", marker = "sys_platform != 'linux'" },
-  { index = "pytorch-cu124", marker = "sys_platform == 'linux'" },
-]
-
-[[tool.uv.index]]
-name = "pytorch-cu124"
-url = "https://download.pytorch.org/whl/cu124"
-explicit = true
-
-[[tool.uv.index]]
-name = "pytorch-default"
-url = "https://pypi.org/simple"
-explicit = true
-
 [tool.uv.extra-build-dependencies]
 # Build flash-attn with the same torch version as in the container. Details at:
 # https://docs.astral.sh/uv/concepts/projects/config/#augmenting-build-dependencies
@@ -166,6 +145,7 @@ known-third-party = ["wandb"]
 
 [tool.ruff.lint.extend-per-file-ignores]
 "__init__.py" = ["F401"]
+"tests/tests_eval_framework/tasks/benchmarks/test_mmlu_de.py" = ["E501"]
 
 [tool.mypy]
 plugins = "pydantic.mypy"

{eval_framework-0.3.7 → eval_framework-0.5.0}/src/eval_framework/evaluation_generator.py

@@ -80,6 +80,7 @@ class EvaluationGenerator:
                 )
             else:
                 metric = metric_class()
+            metric.fail_on_error = self.config.fail_on_error
 
             logger.info(f"Starting calculation of {metric.NAME}")
             safe_tqdm_write(f"INFO: Calculating {metric.NAME}")

{eval_framework-0.3.7 → eval_framework-0.5.0}/src/eval_framework/llm/openai.py

@@ -55,7 +55,7 @@ class OpenAIModel(BaseLLM):
         formatter: BaseFormatter | None = None,
         temperature: float | None = None,
         top_p: float | None = None,
-        api_key: str | None = os.getenv("OPENAI_API_KEY", ""),
+        api_key: str | None = None,
         organization: str | None = None,
         base_url: str | None = None,
         bytes_per_token: float | None = None,
@@ -86,13 +86,12 @@ class OpenAIModel(BaseLLM):
         self._top_p = top_p
 
         self._client = OpenAI(
-            api_key=api_key,
+            api_key=api_key if api_key is not None else os.getenv("OPENAI_API_KEY", ""),
             organization=organization,
             base_url=base_url,
         )
 
-        # Initialize tokenizer for the model
-        self._encoder = self._get_encoder()
+        self._encoder: tiktoken.Encoding | Tokenizer | None = self._get_encoder_or_none()
 
         # set bytes_per_token_scalar for non-standard models
         if bytes_per_token is not None and bytes_per_token <= 0:
@@ -101,9 +100,23 @@ class OpenAIModel(BaseLLM):
             4.0 / bytes_per_token if bytes_per_token is not None else 4.0 / self.BYTES_PER_TOKEN
         )
 
-    def _get_encoder(self) -> tiktoken.Encoding:
+    def _get_encoder_or_none(self) -> tiktoken.Encoding | None:
         assert self._model_name is not None
-        return tiktoken.encoding_for_model(self._model_name)
+        try:
+            return tiktoken.encoding_for_model(self._model_name)
+        except KeyError:
+            logger.info(
+                "tiktoken could not map model_name=%r. Disabling token counting for this model.",
+                self._model_name,
+            )
+            return None
+        except Exception as e:
+            logger.warning(
+                "Failed to initialize tiktoken encoder for model_name=%r (%s). Disabling token counting.",
+                self._model_name,
+                e.__class__.__name__,
+            )
+            return None
 
     def _count_tokens(self, text: str) -> int:
         """
@@ -115,6 +128,8 @@ class OpenAIModel(BaseLLM):
         Returns:
             Number of tokens.
         """
+        if self._encoder is None:
+            raise RuntimeError("Token counting is not available (no encoder configured).")
         return len(self._encoder.encode(text))
 
     def generate_from_messages(
@@ -166,14 +181,31 @@ class OpenAIModel(BaseLLM):
                     stop=stop_sequences,
                 )
                 completion = response.choices[0].text
+                usage = getattr(response, "usage", None)
+                prompt_tokens = getattr(usage, "prompt_tokens", None) if usage is not None else None
+                completion_tokens = getattr(usage, "completion_tokens", None) if usage is not None else None
                 return RawCompletion(
                     prompt=prompt,
-                    prompt_sequence_positions=self._count_tokens(prompt),
-                    concat_compression=ConcatCompression.calculate(
-                        single_messages, count_tokens=self._count_tokens, completion=completion
+                    prompt_sequence_positions=(
+                        prompt_tokens
+                        if prompt_tokens is not None
+                        else (self._count_tokens(prompt) if self._encoder is not None else None)
+                    ),
+                    concat_compression=(
+                        ConcatCompression.calculate(
+                            single_messages,
+                            count_tokens=self._count_tokens,
+                            completion=completion,
+                        )
+                        if self._encoder is not None
+                        else None
                     ),
                     completion=completion,
-                    completion_sequence_positions=self._count_tokens(completion),
+                    completion_sequence_positions=(
+                        completion_tokens
+                        if completion_tokens is not None
+                        else (self._count_tokens(completion) if self._encoder is not None else None)
+                    ),
                 )
 
             else:
@@ -190,15 +222,26 @@ class OpenAIModel(BaseLLM):
                 )
                 prompt = "\n".join([f"{m.get('role', '')}: {m.get('content', '')}" for m in chat_messages])
                 prompt_tokens = getattr(chat_response.usage, "prompt_tokens", None)
+                completion_tokens = getattr(chat_response.usage, "completion_tokens", None)
                 completion = chat_response.choices[0].message.content or ""
                 return RawCompletion(
                     prompt=prompt,
                     prompt_sequence_positions=prompt_tokens,
-                    concat_compression=ConcatCompression.calculate(
-                        single_messages, count_tokens=self._count_tokens, completion=completion
+                    concat_compression=(
+                        ConcatCompression.calculate(
+                            single_messages,
+                            count_tokens=self._count_tokens,
+                            completion=completion,
+                        )
+                        if self._encoder is not None
+                        else None
                     ),
                     completion=completion,
-                    completion_sequence_positions=self._count_tokens(completion),
+                    completion_sequence_positions=(
+                        completion_tokens
+                        if completion_tokens is not None
+                        else (self._count_tokens(completion) if self._encoder is not None else None)
+                    ),
                 )
 
         with concurrent.futures.ThreadPoolExecutor() as executor:
@@ -218,6 +261,10 @@ class OpenAIModel(BaseLLM):
         Note:
             Uses the OpenAI completions API with echo=True; chat logprobs are not supported.
         """
+        if self._encoder is None:
+            raise NotImplementedError(
+                "OpenAIModel.logprobs() requires a local tokenizer/encoder, but none is available."
+            )
         assert self._model_name in ["babbage-002", "davinci-002"], (
             "Log-probs for prompt tokens are only supported for a limited set of models."
         )
@@ -383,12 +430,16 @@ class DeepseekModel(OpenAIModel):
             base_url="https://api.deepseek.com/beta",
         )
         self._tokenizer_name = tokenizer_name if tokenizer_name is not None else "deepseek-ai/DeepSeek-V3.2-Exp"
+        # DeepSeek uses HF tokenization; override the base encoder (which may be None).
+        self._encoder = self._get_encoder()
 
     def _get_encoder(self) -> Tokenizer:
         return AutoTokenizer.from_pretrained(self._tokenizer_name)
 
     def _count_tokens(self, text: str) -> int:
-        return len(self._encoder.encode(text))
+        encoder = self._encoder
+        assert encoder is not None
+        return len(encoder.encode(text))  # type: ignore[union-attr]
 
 
 ### Model Aliases ###

eval_framework-0.5.0/src/eval_framework/llm/vllm_local_server.py

@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+import atexit
+import logging
+import os
+import signal
+import socket
+import subprocess
+import time
+import urllib.error
+import urllib.request
+from collections.abc import Sequence
+
+from eval_framework.llm.base import BaseLLM
+from eval_framework.llm.openai import OpenAIModel
+from eval_framework.shared.types import RawCompletion, RawLoglikelihood
+from eval_framework.tasks.base import Sample
+from template_formatting.formatter import BaseFormatter, Message
+
+logger = logging.getLogger(__name__)
+
+
+def _pick_free_port(host: str) -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind((host, 0))
+        return int(s.getsockname()[1])
+
+
+def _wait_for_http_ready(url: str, *, timeout_s: float) -> None:
+    deadline = time.time() + timeout_s
+    last_err: Exception | None = None
+    while time.time() < deadline:
+        try:
+            with urllib.request.urlopen(url, timeout=2) as resp:
+                if 200 <= resp.status < 500:
+                    return
+        except (urllib.error.URLError, TimeoutError, ConnectionError, OSError) as e:
+            last_err = e
+            time.sleep(0.25)
+    raise RuntimeError(f"Timed out waiting for server readiness at {url}. Last error: {last_err}")
+
+
+def _wait_for_http_ready_or_proc_exit(url: str, *, timeout_s: float, proc: subprocess.Popen[str]) -> None:
+    """
+    Like `_wait_for_http_ready`, but fail fast if the server process exits.
+
+    This avoids long timeouts that hide the real root cause (e.g. invalid CLI flags,
+    missing dependencies, CUDA issues).
+    """
+    deadline = time.time() + timeout_s
+    last_err: Exception | None = None
+
+    while time.time() < deadline:
+        if proc.poll() is not None:
+            out = ""
+            try:
+                if proc.stdout is not None:
+                    out = proc.stdout.read() or ""
+            except Exception:
+                out = ""
+            tail = out.strip()
+            if len(tail) > 8000:
+                tail = tail[-8000:]
+            raise RuntimeError(
+                f"vLLM server process exited before becoming ready. exit_code={proc.returncode}. Output (tail):\n{tail}"
+            )
+
+        try:
+            with urllib.request.urlopen(url, timeout=2) as resp:
+                if 200 <= resp.status < 500:
+                    return
+        except (urllib.error.URLError, TimeoutError, ConnectionError, OSError) as e:
+            last_err = e
+            time.sleep(0.25)
+
+    raise RuntimeError(f"Timed out waiting for server readiness at {url}. Last error: {last_err}")
+
+
+class VLLMLocalServerModel(BaseLLM):
+    """
+    Provider-style model: start a local vLLM OpenAI-compatible server, then talk to it via `OpenAIModel(base_url=...)`.
+
+    This gives you a stable HTTP boundary (good for VCR cassettes) while keeping "local vLLM" as a selectable backend.
+
+    Notes:
+    - The server is started in a subprocess using `vllm serve`.
+    - Cleanup is best-effort (SIGTERM then SIGKILL).
+    - Not all OpenAI API features are guaranteed to be supported by the local server (e.g. logprobs).
+    """
+
+    def __init__(
+        self,
+        *,
+        model_name: str,
+        host: str = "127.0.0.1",
+        port: int | None = None,
+        startup_timeout_s: float = 120.0,
+        # `OpenAIModel` parameters:
+        formatter: BaseFormatter | None = None,
+        temperature: float | None = None,
+        top_p: float | None = None,
+        api_key: str | None = None,
+        bytes_per_token: float | None = None,
+        # vLLM "serve" parameters (subset, passed through):
+        tensor_parallel_size: int | None = None,
+        dtype: str | None = None,
+        max_model_len: int | None = None,
+        gpu_memory_utilization: float | None = None,
+        enforce_eager: bool | None = None,
+        # Escape hatch:
+        vllm_command: str | None = None,
+        vllm_extra_args: list[str] | None = None,
+        env: dict[str, str] | None = None,
+    ) -> None:
+        self._model_name = model_name
+        self._host = host
+        self._port = port if port is not None else _pick_free_port(host)
+        self._startup_timeout_s = float(startup_timeout_s)
+
+        self._proc: subprocess.Popen[str] | None = None
+
+        self._server_url = f"http://{self._host}:{self._port}/v1"
+
+        cmd = [vllm_command or "vllm", "serve", self._model_name, "--host", self._host, "--port", str(self._port)]
+
+        # A small, intentionally conservative subset of flags.
+        if tensor_parallel_size is not None:
+            cmd += ["--tensor-parallel-size", str(tensor_parallel_size)]
+        if dtype is not None:
+            cmd += ["--dtype", str(dtype)]
+        if max_model_len is not None:
+            cmd += ["--max-model-len", str(max_model_len)]
+        if gpu_memory_utilization is not None:
+            cmd += ["--gpu-memory-utilization", str(gpu_memory_utilization)]
+        if enforce_eager is not None:
+            # vLLM exposes this as a boolean flag; passing a value breaks CLI parsing.
+            if enforce_eager:
+                cmd += ["--enforce-eager"]
+
+        if vllm_extra_args:
+            cmd += list(vllm_extra_args)
+
+        merged_env = os.environ.copy()
+        if env:
+            merged_env.update(env)
+
+        logger.info("Starting local vLLM server: %s", " ".join(cmd))
+        self._proc = subprocess.Popen(
+            cmd,
+            env=merged_env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+        )
+
+        # Ensure we don't leave it around if the process exits abruptly.
+        atexit.register(self._cleanup)
+
+        # Wait until the OpenAI-compatible endpoints respond.
+        if self._proc is None:
+            raise RuntimeError("Failed to start vLLM server process.")
+        _wait_for_http_ready_or_proc_exit(
+            f"{self._server_url}/models",
+            timeout_s=self._startup_timeout_s,
+            proc=self._proc,
+        )
+
+        # Configure client to talk to the local server.
+        # For local servers, any non-empty API key typically works; allow explicit override.
+        effective_api_key = api_key if api_key is not None else os.getenv("OPENAI_API_KEY") or "local-vllm"
+
+        self._client = OpenAIModel(
+            model_name=self._model_name,
+            formatter=formatter,
+            temperature=temperature,
+            top_p=top_p,
+            api_key=effective_api_key,
+            base_url=self._server_url,
+            bytes_per_token=bytes_per_token,
+        )
+
+    @property
+    def name(self) -> str:
+        return f"vllm_local::{self._model_name}"
+
+    def generate_from_messages(
+        self,
+        messages: list[Sequence[Message]],
+        stop_sequences: list[str] | None = None,
+        max_tokens: int | None = None,
+        temperature: float | None = None,
+        top_p: float | None = None,
+    ) -> list[RawCompletion]:
+        return self._client.generate_from_messages(messages, stop_sequences, max_tokens, temperature, top_p)
+
+    def logprobs(self, samples: list[Sample]) -> list[RawLoglikelihood]:
+        return self._client.logprobs(samples)
+
+    def _cleanup(self) -> None:
+        proc = self._proc
+        self._proc = None
+        if proc is None:
+            return
+        if proc.poll() is not None:
+            return
+
+        try:
+            proc.terminate()
+            proc.wait(timeout=10)
+        except Exception:
+            try:
+                proc.send_signal(signal.SIGKILL)
+            except Exception:
+                pass
+
+    def __del__(self) -> None:
+        self._cleanup()