lemonade-sdk 7.0.1__py3-none-any.whl → 7.0.3__py3-none-any.whl

lemonade_server/model_manager.py

@@ -1,7 +1,8 @@
 import json
 import os
 import huggingface_hub
-import pkg_resources
+from importlib.metadata import distributions
+from lemonade_server.pydantic_models import LoadConfig
 
 
 class ModelManager:
@@ -64,16 +65,45 @@ class ModelManager:
         """
         return self.filter_models_by_backend(self.downloaded_models)
 
-    def download_gguf(self, checkpoint) -> str:
-        # The colon after the checkpoint name indicates which
-        # specific GGUF to download
-        repo_id = checkpoint.split(":")[0]
-        pattern_to_match = f'*{checkpoint.split(":")[1]}.gguf'
-        return huggingface_hub.snapshot_download(
-            repo_id=repo_id,
-            allow_patterns=[pattern_to_match],
+    def download_gguf(self, model_config: LoadConfig) -> dict:
+        """
+        Downloads the GGUF file for the given model configuration.
+        """
+
+        # The variant parameter can be either:
+        # 1. A full GGUF filename (e.g. "model-Q4_0.gguf")
+        # 2. A quantization variant (e.g. "Q4_0")
+        # This code handles both cases by constructing the appropriate filename
+        checkpoint, variant = model_config.checkpoint.split(":")
+        hf_base_name = checkpoint.split("/")[-1].replace("-GGUF", "")
+        variant_name = (
+            variant if variant.endswith(".gguf") else f"{hf_base_name}-{variant}.gguf"
+        )
+
+        # If there is a mmproj file, add it to the patterns
+        expected_files = {"variant": variant_name}
+        if model_config.mmproj:
+            expected_files["mmproj"] = model_config.mmproj
+
+        # Download the files
+        snapshot_folder = huggingface_hub.snapshot_download(
+            repo_id=checkpoint,
+            allow_patterns=list(expected_files.values()),
         )
 
+        # Ensure we downloaded all expected files while creating a dict of the downloaded files
+        snapshot_files = {}
+        for file in expected_files:
+            snapshot_files[file] = os.path.join(snapshot_folder, expected_files[file])
+            if expected_files[file] not in os.listdir(snapshot_folder):
+                raise ValueError(
+                    f"Hugging Face snapshot download for {model_config.checkpoint} "
+                    f"expected file {expected_files[file]} not found in {snapshot_folder}"
+                )
+
+        # Return a dict that points to the snapshot path of the downloaded GGUF files
+        return snapshot_files
+
     def download_models(self, models: list[str]):
         """
         Downloads the specified models from Hugging Face.
@@ -88,7 +118,8 @@ class ModelManager:
             print(f"Downloading {model} ({checkpoint})")
 
             if "gguf" in checkpoint.lower():
-                self.download_gguf(checkpoint)
+                model_config = LoadConfig(**self.supported_models[model])
+                self.download_gguf(model_config)
             else:
                 huggingface_hub.snapshot_download(repo_id=checkpoint)
 
@@ -97,9 +128,11 @@ class ModelManager:
         Returns a filtered dict of models that are enabled by the
         current environment.
         """
+        installed_packages = {dist.metadata["Name"].lower() for dist in distributions()}
+
         hybrid_installed = (
-            "onnxruntime-vitisai" in pkg_resources.working_set.by_key
-            and "onnxruntime-genai-directml-ryzenai" in pkg_resources.working_set.by_key
+            "onnxruntime-vitisai" in installed_packages
+            and "onnxruntime-genai-directml-ryzenai" in installed_packages
         )
         filtered = {}
         for model, value in models.items():

{lemonade/tools/server → lemonade_server}/pydantic_models.py

@@ -24,6 +24,8 @@ class LoadConfig(BaseModel):
     max_prompt_length: Optional[int] = None
     # Indicates whether the model is a reasoning model, like DeepSeek
     reasoning: Optional[bool] = False
+    # Indicates which Multimodal Projector (mmproj) file to use
+    mmproj: Optional[str] = None
 
 
 class CompletionRequest(BaseModel):

lemonade_server/server_models.json

@@ -9,13 +9,13 @@
         "checkpoint": "amd/Llama-3.2-1B-Instruct-awq-uint4-float16-cpu-onnx",
         "recipe": "oga-cpu",
         "reasoning": false,
-        "suggested": true
+        "suggested": false
     },
     "Llama-3.2-3B-Instruct-CPU": {
         "checkpoint": "amd/Llama-3.2-3B-Instruct-awq-uint4-float16-cpu-onnx",
         "recipe": "oga-cpu",
         "reasoning": false,
-        "suggested": true
+        "suggested": false
     },
     "Phi-3-Mini-Instruct-CPU": {
         "checkpoint": "amd/Phi-3-mini-4k-instruct_int4_float16_onnx_cpu",
@@ -103,6 +103,13 @@
         "max_prompt_length": 2000,
         "suggested": true
     },
+    "Llama-xLAM-2-8b-fc-r-Hybrid": {
+        "checkpoint": "amd/Llama-xLAM-2-8b-fc-r-awq-g128-int4-asym-bfp16-onnx-hybrid",
+        "recipe": "oga-hybrid",
+        "reasoning": false,
+        "max_prompt_length": 2000,
+        "suggested": true
+    },
     "Llama-3.2-1B-Instruct-DirectML": {
         "checkpoint": "amd/Llama-3.2-1B-Instruct-dml-int4-awq-block-128-directml",
         "recipe": "oga-igpu",
@@ -158,7 +165,7 @@
         "suggested": true
     },
     "Qwen3-8B-GGUF": {
-        "checkpoint": "unsloth/Qwen3-8B-GGUF:Q4_0",
+        "checkpoint": "unsloth/Qwen3-8B-GGUF:Q4_1",
         "recipe": "llamacpp",
         "reasoning": true,
         "suggested": true
@@ -180,5 +187,19 @@
         "recipe": "llamacpp",
         "reasoning": true,
         "suggested": true
+    },
+    "Gemma-3-4b-it-GGUF": {
+        "checkpoint": "ggml-org/gemma-3-4b-it-GGUF:Q4_K_M",
+        "mmproj": "mmproj-model-f16.gguf",
+        "recipe": "llamacpp",
+        "reasoning": false,
+        "suggested": true
+    },
+    "Qwen2.5-VL-7B-Instruct": {
+        "checkpoint": "ggml-org/Qwen2.5-VL-7B-Instruct-GGUF:Q4_K_M",
+        "mmproj": "mmproj-Qwen2.5-VL-7B-Instruct-f16.gguf",
+        "recipe": "llamacpp",
+        "reasoning": false,
+        "suggested": true
     }
-}
+}