vllm-gguf-plugin 0.0.1__tar.gz

vllm_gguf_plugin-0.0.1/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vllm_gguf_plugin-0.0.1/MANIFEST.in

@@ -0,0 +1,8 @@
+include LICENSE
+include README.md
+
+recursive-include vllm_gguf_plugin/csrc *.h *.cpp *.cu *.cuh
+
+global-exclude __pycache__
+global-exclude *.py[cod]
+global-exclude *.so

vllm_gguf_plugin-0.0.1/PKG-INFO

@@ -0,0 +1,70 @@
+Metadata-Version: 2.4
+Name: vllm-gguf-plugin
+Version: 0.0.1
+Summary: Out-of-tree GGUF quantization plugin for vLLM
+License-Expression: Apache-2.0
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: gguf>=0.17.0
+Requires-Dist: vllm
+Requires-Dist: torch>=2.9
+Provides-Extra: dev
+Requires-Dist: pre-commit; extra == "dev"
+Dynamic: license-file
+
+# vLLM GGUF Quantization Plugin
+
+This plugin provides out-of-tree GGUF quantization support for vLLM after
+in-tree support deprecation
+([vllm-project/vllm#39583](https://github.com/vllm-project/vllm/issues/39583)).
+
+## Installation
+
+### Prerequisites
+
+- CUDA toolkit or ROCm toolkit
+
+We recommend [uv](https://docs.astral.sh/uv/) for package management. If you
+don't have it installed:
+
+```bash
+curl -LsSf https://astral.sh/uv/install.sh | sh
+```
+
+### From Source
+
+1. Clone this repository:
+
+   ```bash
+   git clone https://github.com/vllm-project/vllm-gguf-plugin
+   cd vllm-gguf-plugin
+   ```
+
+2. Install the plugin in development mode:
+
+   ```bash
+   uv pip install -e . --torch-backend=auto
+   ```
+
+Or install directly:
+
+```bash
+uv pip install . --torch-backend=auto
+```
+
+## Development
+
+```bash
+uv pip install -e .[dev] --torch-backend=auto
+pre-commit install
+pre-commit run --all-files
+```
+
+The same hooks also run in GitHub Actions on every push and pull request.
+
+## Usage
+
+```bash
+vllm serve Qwen/Qwen3-0.6B-GGUF:Q8_0 --tokenizer Qwen/Qwen3-0.6B
+```

vllm_gguf_plugin-0.0.1/README.md

@@ -0,0 +1,55 @@
+# vLLM GGUF Quantization Plugin
+
+This plugin provides out-of-tree GGUF quantization support for vLLM after
+in-tree support deprecation
+([vllm-project/vllm#39583](https://github.com/vllm-project/vllm/issues/39583)).
+
+## Installation
+
+### Prerequisites
+
+- CUDA toolkit or ROCm toolkit
+
+We recommend [uv](https://docs.astral.sh/uv/) for package management. If you
+don't have it installed:
+
+```bash
+curl -LsSf https://astral.sh/uv/install.sh | sh
+```
+
+### From Source
+
+1. Clone this repository:
+
+   ```bash
+   git clone https://github.com/vllm-project/vllm-gguf-plugin
+   cd vllm-gguf-plugin
+   ```
+
+2. Install the plugin in development mode:
+
+   ```bash
+   uv pip install -e . --torch-backend=auto
+   ```
+
+Or install directly:
+
+```bash
+uv pip install . --torch-backend=auto
+```
+
+## Development
+
+```bash
+uv pip install -e .[dev] --torch-backend=auto
+pre-commit install
+pre-commit run --all-files
+```
+
+The same hooks also run in GitHub Actions on every push and pull request.
+
+## Usage
+
+```bash
+vllm serve Qwen/Qwen3-0.6B-GGUF:Q8_0 --tokenizer Qwen/Qwen3-0.6B
+```

vllm_gguf_plugin-0.0.1/pyproject.toml

@@ -0,0 +1,81 @@
+[build-system]
+requires = ["setuptools>=77.0.3,<81.0.0", "wheel", "torch>=2.9"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "vllm-gguf-plugin"
+dynamic = ["version"]
+description = "Out-of-tree GGUF quantization plugin for vLLM"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.10"
+dependencies = ["gguf>=0.17.0", "vllm", "torch>=2.9"]
+
+[project.optional-dependencies]
+dev = ["pre-commit"]
+
+[project.entry-points."vllm.general_plugins"]
+gguf = "vllm_gguf_plugin:register"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["vllm_gguf_plugin*"]
+
+[tool.vllm_gguf_plugin]
+base_version = "0.0.1"
+
+[tool.ruff.lint]
+select = [
+    # pycodestyle
+    "E",
+    # Pyflakes
+    "F",
+    # pyupgrade
+    "UP",
+    # flake8-bugbear
+    "B",
+    # flake8-implicit-str-concat
+    "ISC",
+    # flake8-simplify
+    "SIM",
+    # isort
+    "I",
+    # flake8-logging-format
+    "G",
+]
+ignore = [
+    # star imports
+    "F405", "F403",
+    # lambda expression assignment
+    "E731",
+    # zip without `strict=`
+    "B905",
+    # loop control variable not used within loop body
+    "B007",
+    # f-string format
+    "UP032",
+]
+
+[tool.ruff.format]
+docstring-code-format = true
+
+[tool.typos.files]
+extend-exclude = [
+    "build/*",
+    "dist/*",
+    ".git/*",
+    "vllm_gguf_plugin/csrc/gguf/ggml-common.h",
+    "vllm_gguf_plugin/csrc/gguf/dequantize.cuh",
+    "vllm_gguf_plugin/csrc/gguf/vecdotq.cuh",
+    "vllm_gguf_plugin/csrc/gguf/mmq.cuh",
+    "vllm_gguf_plugin/csrc/gguf/mmvq.cuh",
+]
+ignore-hidden = false
+
+[tool.typos.default.extend-identifiers]
+iy = "iy"
+
+[tool.pytest.ini_options]
+markers = [
+    "slow: marks tests as slow (large model downloads, requires GPU)",
+]

vllm_gguf_plugin-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

vllm_gguf_plugin-0.0.1/setup.py

@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import pathlib
+import sys
+
+import tomllib
+from setuptools import setup
+
+
+def _package_version() -> str:
+    project = tomllib.loads(pathlib.Path("pyproject.toml").read_text())
+    version = project["tool"]["vllm_gguf_plugin"]["base_version"]
+    suffix = os.environ.get("VLLM_GGUF_PLUGIN_LOCAL_VERSION_SUFFIX")
+    if not suffix:
+        return version
+    normalized_suffix = suffix if suffix.startswith("+") else f"+{suffix}"
+    return f"{version}{normalized_suffix}"
+
+
+def _should_build_extension() -> bool:
+    packaging_commands = {"sdist", "egg_info", "dist_info"}
+    return not any(command in packaging_commands for command in sys.argv[1:])
+
+
+setup_kwargs: dict = {"version": _package_version()}
+
+if _should_build_extension():
+    from torch.utils.cpp_extension import BuildExtension, CUDAExtension
+
+    setup_kwargs.update(
+        ext_modules=[
+            CUDAExtension(
+                name="vllm_gguf_plugin._C_gguf",
+                sources=[
+                    "vllm_gguf_plugin/csrc/torch_bindings.cpp",
+                    "vllm_gguf_plugin/csrc/gguf/gguf_kernel.cu",
+                ],
+                include_dirs=[
+                    "vllm_gguf_plugin/csrc",
+                    "vllm_gguf_plugin/csrc/gguf",
+                ],
+                py_limited_api=True,
+                extra_compile_args={
+                    "cxx": ["-O3", "-std=c++17"],
+                    "nvcc": [
+                        "-O3",
+                        "-std=c++17",
+                        "--use_fast_math",
+                        # Exposes aoti_torch_get_current_cuda_stream in the AOTI shim.
+                        "-DUSE_CUDA",
+                    ],
+                },
+            )
+        ],
+        cmdclass={"build_ext": BuildExtension},
+        options={"bdist_wheel": {"py_limited_api": "cp310"}},
+    )
+
+setup(**setup_kwargs)

vllm_gguf_plugin-0.0.1/tests/test_gguf_download.py

@@ -0,0 +1,169 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+from vllm.config.load import LoadConfig
+
+from vllm_gguf_plugin.loader import GGUFModelLoader
+from vllm_gguf_plugin.weight_utils import download_gguf
+
+
+class TestGGUFDownload:
+    """Test GGUF model downloading functionality."""
+
+    @patch("vllm_gguf_plugin.weight_utils.snapshot_download")
+    def test_download_gguf_single_file(self, mock_download):
+        """Test downloading a single GGUF file."""
+        mock_folder = "/tmp/mock_cache"
+        mock_download.return_value = mock_folder
+
+        with patch("glob.glob") as mock_glob:
+            mock_glob.side_effect = lambda pattern, **kwargs: (
+                [f"{mock_folder}/model-IQ1_S.gguf"] if "IQ1_S" in pattern else []
+            )
+
+            result = download_gguf("unsloth/Qwen3-0.6B-GGUF", "IQ1_S")
+
+            mock_download.assert_called_once_with(
+                repo_id="unsloth/Qwen3-0.6B-GGUF",
+                cache_dir=None,
+                allow_patterns=[
+                    "*.IQ1_S-*.gguf",
+                    "*.IQ1_S.gguf",
+                    "*-IQ1_S-*.gguf",
+                    "*-IQ1_S.gguf",
+                    "*.iq1_s-*.gguf",
+                    "*.iq1_s.gguf",
+                    "*-iq1_s-*.gguf",
+                    "*-iq1_s.gguf",
+                ],
+                revision=None,
+                ignore_patterns=None,
+            )
+
+            assert result == f"{mock_folder}/model-IQ1_S.gguf"
+
+    @patch("vllm_gguf_plugin.weight_utils.snapshot_download")
+    def test_download_gguf_sharded_files(self, mock_download):
+        """Test downloading sharded GGUF files."""
+        mock_folder = "/tmp/mock_cache"
+        mock_download.return_value = mock_folder
+
+        with patch("glob.glob") as mock_glob:
+            mock_glob.side_effect = lambda pattern, **kwargs: (
+                [
+                    f"{mock_folder}/model-Q2_K-00001-of-00002.gguf",
+                    f"{mock_folder}/model-Q2_K-00002-of-00002.gguf",
+                ]
+                if "Q2_K" in pattern
+                else []
+            )
+
+            result = download_gguf("unsloth/gpt-oss-120b-GGUF", "Q2_K")
+
+            assert result == f"{mock_folder}/model-Q2_K-00001-of-00002.gguf"
+
+    @patch("vllm_gguf_plugin.weight_utils.snapshot_download")
+    def test_download_gguf_subdir(self, mock_download):
+        """Test downloading GGUF files from subdirectory."""
+        mock_folder = "/tmp/mock_cache"
+        mock_download.return_value = mock_folder
+
+        with patch("glob.glob") as mock_glob:
+            mock_glob.side_effect = lambda pattern, **kwargs: (
+                [f"{mock_folder}/Q2_K/model-Q2_K.gguf"]
+                if "Q2_K" in pattern or "**/*.gguf" in pattern
+                else []
+            )
+
+            result = download_gguf("unsloth/gpt-oss-120b-GGUF", "Q2_K")
+
+            assert result == f"{mock_folder}/Q2_K/model-Q2_K.gguf"
+
+    @patch("vllm_gguf_plugin.weight_utils.snapshot_download")
+    @patch("glob.glob", return_value=[])
+    def test_download_gguf_no_files_found(self, mock_glob, mock_download):
+        """Test error when no GGUF files are found."""
+        mock_folder = "/tmp/mock_cache"
+        mock_download.return_value = mock_folder
+
+        with pytest.raises(ValueError, match="Downloaded GGUF files not found"):
+            download_gguf("unsloth/Qwen3-0.6B-GGUF", "IQ1_S")
+
+
+class TestGGUFModelLoader:
+    """Test GGUFModelLoader class methods."""
+
+    @patch("os.path.isfile", return_value=True)
+    def test_prepare_weights_local_file(self, mock_isfile):
+        """Test _prepare_weights with local file."""
+        load_config = LoadConfig(load_format="gguf")
+        loader = GGUFModelLoader(load_config)
+
+        model_config = MagicMock()
+        model_config.model_weights = "/path/to/model.gguf"
+        model_config.model = "/path/to/hf"
+
+        result = loader._prepare_weights(model_config)
+        assert result == "/path/to/model.gguf"
+        mock_isfile.assert_called_once_with("/path/to/model.gguf")
+
+    @patch("vllm_gguf_plugin.loader.hf_hub_download")
+    @patch("os.path.isfile", return_value=False)
+    def test_prepare_weights_repo_filename(self, mock_isfile, mock_hf_download):
+        """Test _prepare_weights with repo_id/filename.gguf format."""
+        load_config = LoadConfig(load_format="gguf")
+        loader = GGUFModelLoader(load_config)
+
+        mock_hf_download.return_value = "/downloaded/model.gguf"
+
+        model_config = MagicMock()
+        model_config.model_weights = "unsloth/Qwen3-0.6B-GGUF/model.gguf"
+        model_config.model = "unsloth/Qwen3-0.6B-GGUF"
+
+        result = loader._prepare_weights(model_config)
+        assert result == "/downloaded/model.gguf"
+        mock_hf_download.assert_called_once_with(
+            repo_id="unsloth/Qwen3-0.6B-GGUF", filename="model.gguf"
+        )
+
+    @patch("vllm_gguf_plugin.weight_utils.snapshot_download")
+    @patch("glob.glob")
+    @patch("os.path.isdir", return_value=False)
+    @patch("os.path.isfile", return_value=False)
+    def test_prepare_weights_remote_repo_quant_type(
+        self, mock_isfile, mock_isdir, mock_glob, mock_download
+    ):
+        """Test _prepare_weights with remote repo_id:quant_type format."""
+        mock_folder = "/tmp/mock_cache"
+        mock_download.return_value = mock_folder
+        mock_glob.side_effect = lambda pattern, **kwargs: (
+            [f"{mock_folder}/model-IQ1_S.gguf"] if "IQ1_S" in pattern else []
+        )
+
+        load_config = LoadConfig(load_format="gguf")
+        loader = GGUFModelLoader(load_config)
+
+        model_config = MagicMock()
+        model_config.model_weights = "unsloth/Qwen3-0.6B-GGUF:IQ1_S"
+        model_config.model = "unsloth/Qwen3-0.6B-GGUF"
+        model_config.revision = None
+
+        result = loader._prepare_weights(model_config)
+        assert result == f"{mock_folder}/model-IQ1_S.gguf"
+        mock_download.assert_called_once()
+
+    @patch("os.path.isfile", return_value=False)
+    def test_prepare_weights_invalid_format(self, mock_isfile):
+        """Test _prepare_weights with invalid format."""
+        load_config = LoadConfig(load_format="gguf")
+        loader = GGUFModelLoader(load_config)
+
+        model_config = MagicMock()
+        model_config.model_weights = "invalid-format"
+        model_config.model = "invalid-format"
+
+        with pytest.raises(ValueError, match="Unrecognised GGUF reference"):
+            loader._prepare_weights(model_config)