lemonade-sdk 7.0.4__py3-none-any.whl → 8.0.1__py3-none-any.whl

lemonade_server/model_manager.py

@@ -1,24 +1,60 @@
 import json
 import os
+from typing import Optional
+import shutil
 import huggingface_hub
 from importlib.metadata import distributions
-from lemonade_server.pydantic_models import LoadConfig
+from lemonade_server.pydantic_models import PullConfig
+from lemonade.cache import DEFAULT_CACHE_DIR
+
+USER_MODELS_FILE = os.path.join(DEFAULT_CACHE_DIR, "user_models.json")
 
 
 class ModelManager:
 
+    @staticmethod
+    def parse_checkpoint(checkpoint: str) -> tuple[str, str | None]:
+        """
+        Parse a checkpoint string that may contain a variant separated by a colon.
+
+        For GGUF models, the format is "repository:variant" (e.g., "unsloth/Qwen3-0.6B-GGUF:Q4_0").
+        For other models, there is no variant.
+
+        Args:
+            checkpoint: The checkpoint string, potentially with variant
+
+        Returns:
+            tuple: (base_checkpoint, variant) where variant is None if no colon is present
+        """
+        if ":" in checkpoint:
+            base_checkpoint, variant = checkpoint.split(":", 1)
+            return base_checkpoint, variant
+        return checkpoint, None
+
     @property
     def supported_models(self) -> dict:
         """
         Returns a dictionary of supported models.
         Note: Models must be downloaded before they are locally available.
         """
-        # Load the models dictionary from the JSON file
+        # Load the models dictionary from the built-in JSON file
         server_models_file = os.path.join(
             os.path.dirname(__file__), "server_models.json"
         )
         with open(server_models_file, "r", encoding="utf-8") as file:
-            models = json.load(file)
+            models: dict = json.load(file)
+
+        # Load the user's JSON file, if it exists, and merge into the models dict
+        if os.path.exists(USER_MODELS_FILE):
+            with open(USER_MODELS_FILE, "r", encoding="utf-8") as file:
+                user_models: dict = json.load(file)
+            # Prepend the user namespace to the model names
+            user_models = {
+                f"user.{model_name}": model_info
+                for model_name, model_info in user_models.items()
+            }
+
+            models.update(user_models)
 
         # Add the model name as a key in each entry, to make it easier
         # to access later
@@ -49,11 +85,12 @@ class ModelManager:
         Returns a dictionary of locally available models.
         """
         downloaded_models = {}
+        downloaded_checkpoints = self.downloaded_hf_checkpoints
         for model in self.supported_models:
-            if (
-                self.supported_models[model]["checkpoint"].split(":")[0]
-                in self.downloaded_hf_checkpoints
-            ):
+            base_checkpoint = self.parse_checkpoint(
+                self.supported_models[model]["checkpoint"]
+            )[0]
+            if base_checkpoint in downloaded_checkpoints:
                 downloaded_models[model] = self.supported_models[model]
         return downloaded_models
 
@@ -65,7 +102,7 @@ class ModelManager:
         """
         return self.filter_models_by_backend(self.downloaded_models)
 
-    def download_gguf(self, model_config: LoadConfig) -> dict:
+    def download_gguf(self, model_config: PullConfig) -> dict:
         """
         Downloads the GGUF file for the given model configuration.
         """
@@ -74,7 +111,7 @@ class ModelManager:
         # 1. A full GGUF filename (e.g. "model-Q4_0.gguf")
         # 2. A quantization variant (e.g. "Q4_0")
         # This code handles both cases by constructing the appropriate filename
-        checkpoint, variant = model_config.checkpoint.split(":")
+        checkpoint, variant = self.parse_checkpoint(model_config.checkpoint)
         hf_base_name = checkpoint.split("/")[-1].replace("-GGUF", "")
         variant_name = (
             variant if variant.endswith(".gguf") else f"{hf_base_name}-{variant}.gguf"
@@ -91,11 +128,24 @@ class ModelManager:
             allow_patterns=list(expected_files.values()),
         )
 
+        # Make sure we downloaded something
+        # If we didn't that can indicate that no patterns from allow_patterns match
+        # any files in the HF repo
+        if not os.path.exists(snapshot_folder):
+            raise ValueError(
+                "No patterns matched the variant parameter (CHECKPOINT:VARIANT). "
+                "Try again, providing the full filename of your target .gguf file as the variant."
+                " For example: Qwen/Qwen2.5-Coder-3B-Instruct-GGUF:"
+                "qwen2.5-coder-3b-instruct-q4_0.gguf"
+            )
+
         # Ensure we downloaded all expected files while creating a dict of the downloaded files
         snapshot_files = {}
         for file in expected_files:
             snapshot_files[file] = os.path.join(snapshot_folder, expected_files[file])
-            if expected_files[file] not in os.listdir(snapshot_folder):
+            if expected_files[file].lower() not in [
+                name.lower() for name in os.listdir(snapshot_folder)
+            ]:
                 raise ValueError(
                     f"Hugging Face snapshot download for {model_config.checkpoint} "
                     f"expected file {expected_files[file]} not found in {snapshot_folder}"
@@ -104,24 +154,103 @@ class ModelManager:
         # Return a dict that points to the snapshot path of the downloaded GGUF files
         return snapshot_files
 
-    def download_models(self, models: list[str]):
+    def download_models(
+        self,
+        models: list[str],
+        checkpoint: Optional[str] = None,
+        recipe: Optional[str] = None,
+        reasoning: bool = False,
+        mmproj: str = "",
+    ):
         """
         Downloads the specified models from Hugging Face.
         """
         for model in models:
             if model not in self.supported_models:
-                raise ValueError(
-                    f"Model {model} is not supported. Please choose from the following: "
-                    f"{list(self.supported_models.keys())}"
+                # Register the model as a user model if the model name
+                # is not already registered
+
+                # Ensure the model name includes the `user` namespace
+                model_parsed = model.split(".", 1)
+                if len(model_parsed) != 2 or model_parsed[0] != "user":
+                    raise ValueError(
+                        f"When registering a new model, the model name must "
+                        "include the `user` namespace, for example "
+                        f"`user.Phi-4-Mini-GGUF`. Received: {model}"
+                    )
+
+                model_name = model_parsed[1]
+
+                # Check that required arguments are provided
+                if not recipe or not checkpoint:
+                    raise ValueError(
+                        f"Model {model} is not registered with Lemonade Server. "
+                        "To register and install it, provide the `checkpoint` "
+                        "and `recipe` arguments, as well as the optional "
+                        "`reasoning` and `mmproj` arguments as appropriate. "
+                    )
+
+                # JSON content that will be used for registration if the download succeeds
+                new_user_model = {
+                    "checkpoint": checkpoint,
+                    "recipe": recipe,
+                    "reasoning": reasoning,
+                    "suggested": True,
+                    "labels": ["custom"],
+                }
+
+                if mmproj:
+                    new_user_model["mmproj"] = mmproj
+
+                # Make sure that a variant is provided for GGUF models before registering the model
+                if "gguf" in checkpoint.lower() and ":" not in checkpoint.lower():
+                    raise ValueError(
+                        "You are required to provide a 'variant' in the checkpoint field when "
+                        "registering a GGUF model. The variant is provided "
+                        "as CHECKPOINT:VARIANT. For example: "
+                        "Qwen/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_0 or "
+                        "Qwen/Qwen2.5-Coder-3B-Instruct-GGUF:"
+                        "qwen2.5-coder-3b-instruct-q4_0.gguf"
+                    )
+
+                # Create a PullConfig we will use to download the model
+                new_registration_model_config = PullConfig(
+                    model_name=model_name,
+                    checkpoint=checkpoint,
+                    recipe=recipe,
+                    reasoning=reasoning,
                 )
-            checkpoint = self.supported_models[model]["checkpoint"]
-            print(f"Downloading {model} ({checkpoint})")
+            else:
+                new_registration_model_config = None
 
-            if "gguf" in checkpoint.lower():
-                model_config = LoadConfig(**self.supported_models[model])
-                self.download_gguf(model_config)
+            # Download the model
+            if new_registration_model_config:
+                checkpoint_to_download = checkpoint
+                gguf_model_config = new_registration_model_config
             else:
-                huggingface_hub.snapshot_download(repo_id=checkpoint)
+                checkpoint_to_download = self.supported_models[model]["checkpoint"]
+                gguf_model_config = PullConfig(**self.supported_models[model])
+            print(f"Downloading {model} ({checkpoint_to_download})")
+
+            if "gguf" in checkpoint_to_download.lower():
+                self.download_gguf(gguf_model_config)
+            else:
+                huggingface_hub.snapshot_download(repo_id=checkpoint_to_download)
+
+            # Register the model in user_models.json, creating that file if needed
+            # We do this registration after the download so that we don't register
+            # any incorrectly configured models where the download would fail
+            if new_registration_model_config:
+                if os.path.exists(USER_MODELS_FILE):
+                    with open(USER_MODELS_FILE, "r", encoding="utf-8") as file:
+                        user_models: dict = json.load(file)
+                else:
+                    user_models = {}
+
+                user_models[model_name] = new_user_model
+
+                with open(USER_MODELS_FILE, mode="w", encoding="utf-8") as file:
+                    json.dump(user_models, fp=file)
 
     def filter_models_by_backend(self, models: dict) -> dict:
         """
@@ -143,6 +272,49 @@ class ModelManager:
                 filtered[model] = value
         return filtered
 
+    def delete_model(self, model_name: str):
+        """
+        Deletes the specified model from local storage.
+        """
+        if model_name not in self.supported_models:
+            raise ValueError(
+                f"Model {model_name} is not supported. Please choose from the following: "
+                f"{list(self.supported_models.keys())}"
+            )
+
+        checkpoint = self.supported_models[model_name]["checkpoint"]
+        print(f"Deleting {model_name} ({checkpoint})")
+
+        # Handle GGUF models that have the format "checkpoint:variant"
+        base_checkpoint = self.parse_checkpoint(checkpoint)[0]
+
+        try:
+            # Get the local path using snapshot_download with local_files_only=True
+            snapshot_path = huggingface_hub.snapshot_download(
+                repo_id=base_checkpoint, local_files_only=True
+            )
+
+            # Navigate up to the model directory (parent of snapshots directory)
+            model_path = os.path.dirname(os.path.dirname(snapshot_path))
+
+            # Delete the entire model directory (including all snapshots)
+            if os.path.exists(model_path):
+                shutil.rmtree(model_path)
+                print(f"Successfully deleted model {model_name} from {model_path}")
+            else:
+                raise ValueError(
+                    f"Model {model_name} not found locally at {model_path}"
+                )
+
+        except Exception as e:
+            if (
+                "not found in cache" in str(e).lower()
+                or "no such file" in str(e).lower()
+            ):
+                raise ValueError(f"Model {model_name} is not installed locally")
+            else:
+                raise ValueError(f"Failed to delete model {model_name}: {str(e)}")
+
 
 # This file was originally licensed under Apache 2.0. It has been modified.
 # Modifications Copyright (c) 2025 AMD

lemonade_server/pydantic_models.py

@@ -15,9 +15,8 @@ class LoadConfig(BaseModel):
     and hardware/framework configuration (recipe) for model loading.
     """
 
-    model_name: Optional[str] = None
+    model_name: str
     checkpoint: Optional[str] = None
-    max_new_tokens: int = DEFAULT_MAX_NEW_TOKENS
     recipe: Optional[str] = None
     # Indicates the maximum prompt length allowed for that specific
     # checkpoint + recipe combination
@@ -77,9 +76,15 @@ class ResponsesRequest(BaseModel):
     stream: bool = False
 
 
-class PullConfig(BaseModel):
+class PullConfig(LoadConfig):
+    """
+    Pull and load have the same fields.
+    """
+
+
+class DeleteConfig(BaseModel):
     """
-    Configurating for installing a supported LLM.
+    Configuration for deleting a supported LLM.
     """
 
     model_name: str

lemonade_server/server_models.json

@@ -193,13 +193,15 @@
         "mmproj": "mmproj-model-f16.gguf",
         "recipe": "llamacpp",
         "reasoning": false,
-        "suggested": true
+        "suggested": true,
+        "labels": ["vision"]
     },
-    "Qwen2.5-VL-7B-Instruct": {
+    "Qwen2.5-VL-7B-Instruct-GGUF": {
         "checkpoint": "ggml-org/Qwen2.5-VL-7B-Instruct-GGUF:Q4_K_M",
         "mmproj": "mmproj-Qwen2.5-VL-7B-Instruct-f16.gguf",
         "recipe": "llamacpp",
         "reasoning": false,
-        "suggested": true
+        "suggested": true,
+        "labels": ["vision"]
     }
 }

lemonade/common/analyze_model.py

@@ -1,26 +0,0 @@
-import numpy as np
-import torch
-import onnx
-
-
-def count_parameters(model: torch.nn.Module) -> int:
-    """
-    Returns the number of parameters of a given model
-    """
-    if isinstance(model, (torch.nn.Module, torch.jit.ScriptModule)):
-        return sum([parameter.numel() for _, parameter in model.named_parameters()])
-    elif isinstance(model, str) and model.endswith(".onnx"):
-        onnx_model = onnx.load(model)
-        return int(
-            sum(
-                np.prod(tensor.dims, dtype=np.int64)
-                for tensor in onnx_model.graph.initializer
-                if tensor.name not in onnx_model.graph.input
-            )
-        )
-    else:
-        return None
-
-
-# This file was originally licensed under Apache 2.0. It has been modified.
-# Modifications Copyright (c) 2025 AMD

lemonade/common/labels.py

@@ -1,61 +0,0 @@
-from typing import Dict, List
-import turnkeyml.common.printing as printing
-
-
-def to_dict(label_list: List[str]) -> Dict[str, List[str]]:
-    """
-    Convert label list into a dictionary of labels
-    """
-    label_dict = {}
-    for item in label_list:
-        try:
-            label_key, label_value = item.split("::")
-            label_value = label_value.split(",")
-            label_dict[label_key] = label_value
-        except ValueError:
-            printing.log_warning(
-                (
-                    f"Malformed label {item} found. "
-                    "Each label must have the format key::value1,value2,... "
-                )
-            )
-    return label_dict
-
-
-def load_from_file(file_path: str) -> Dict[str, List[str]]:
-    """
-    This function extracts labels from a Python file.
-    Labels must be in the first line of a Python file and start with "# labels: "
-    Each label must have the format "key::value1,value2,..."
-
-    Example:
-        "# labels: author::google test_group::daily,monthly"
-    """
-    # Open file
-    with open(file_path, encoding="utf-8") as f:
-        first_line = f.readline()
-
-    # Return label dict
-    if "# labels:" in first_line:
-        label_list = first_line.replace("\n", "").split(" ")[2:]
-        return to_dict(label_list)
-    else:
-        return {}
-
-
-def is_subset(label_dict_a: Dict[str, List[str]], label_dict_b: Dict[str, List[str]]):
-    """
-    This function returns True if label_dict_a is a subset of label_dict_b.
-    More specifically, we return True if:
-        * All keys of label_dict_a are also keys of label_dict_b AND,
-        * All values of label_dict_a[key] are values of label_dict_b[key]
-    """
-    for key in label_dict_a:
-        # Skip benchmarking if the label_dict_a key is not a key of label_dict_b
-        if key not in label_dict_b:
-            return False
-        # A label key may point to multiple label values
-        # Skip if not all values of label_dict_a[key] are in label_dict_b[key]
-        elif not all(elem in label_dict_a[key] for elem in label_dict_b[key]):
-            return False
-    return True

lemonade/common/onnx_helpers.py

@@ -1,176 +0,0 @@
-"""
-Helper functions for dealing with ONNX files and ONNX models
-"""
-
-import os
-from typing import Tuple, Union
-import re
-import math
-import numpy as np
-import onnx
-import onnxruntime as ort
-import lemonade.common.exceptions as exp
-from lemonade.state import State
-import lemonade.common.build as build
-
-
-def check_model(onnx_file, success_message, fail_message) -> bool:
-    if os.path.isfile(onnx_file):
-        print(success_message)
-    else:
-        print(fail_message)
-        return False
-    try:
-        onnx.checker.check_model(onnx_file)
-        print("\tSuccessfully checked onnx file")
-        return True
-    except onnx.checker.ValidationError as e:
-        print("\tError while checking generated ONNX file")
-        print(e)
-        return False
-
-
-def original_inputs_file(cache_dir: str, build_name: str):
-    return os.path.join(build.output_dir(cache_dir, build_name), "inputs.npy")
-
-
-def onnx_dir(state: State):
-    return os.path.join(build.output_dir(state.cache_dir, state.build_name), "onnx")
-
-
-def get_output_names(
-    onnx_model: Union[str, onnx.ModelProto],
-):  # pylint: disable=no-member
-    # Get output names of ONNX file/model
-    if not isinstance(onnx_model, onnx.ModelProto):  # pylint: disable=no-member
-        onnx_model = onnx.load(onnx_model)
-    return [node.name for node in onnx_model.graph.output]  # pylint: disable=no-member
-
-
-def parameter_count(model):
-    weights = model.graph.initializer
-    parameter_count = 0
-
-    for w in weights:
-        weight = onnx.numpy_helper.to_array(w)
-        parameter_count += np.prod(weight.shape)
-    return parameter_count
-
-
-def io_bytes(onnx_path: str) -> Tuple[int, int]:
-    """Return the number of bytes of each of the inputs and outputs"""
-    # pylint: disable = no-member
-
-    def elem_type_to_bytes(elem_type) -> int:
-        """
-        Convert ONNX's elem_type to the number of bytes used by
-        hardware to send that specific datatype through PCIe
-        """
-        if (
-            elem_type == onnx.TensorProto.DataType.UINT8
-            or elem_type == onnx.TensorProto.DataType.INT8
-            or elem_type == onnx.TensorProto.DataType.BOOL
-        ):
-            # Each bool requires an entire byte
-            return 1
-        elif (
-            elem_type == onnx.TensorProto.DataType.UINT16
-            or elem_type == onnx.TensorProto.DataType.INT16
-            or elem_type == onnx.TensorProto.DataType.FLOAT16
-        ):
-            return 2
-        if (
-            elem_type == onnx.TensorProto.DataType.FLOAT
-            or elem_type == onnx.TensorProto.DataType.INT32
-            or elem_type == onnx.TensorProto.DataType.INT64
-            or elem_type == onnx.TensorProto.DataType.DOUBLE
-            or elem_type == onnx.TensorProto.DataType.UINT64
-        ):
-            # 64 bit ints are treated as 32 bits everywhere
-            # Doubles are treated as floats
-            return 4
-        elif (
-            elem_type == onnx.TensorProto.DataType.COMPLEX64
-            or elem_type == onnx.TensorProto.DataType.COMPLEX128
-            or elem_type == onnx.TensorProto.DataType.STRING
-            or elem_type == onnx.TensorProto.DataType.UNDEFINED
-        ):
-            raise exp.Error("Unsupported data type")
-        else:
-            raise exp.Error("Unsupported data type (unknown to ONNX)")
-
-    def get_nodes_bytes(nodes):
-        nodes_bytes = {}
-        for node in nodes:
-
-            # Get the number of the data type
-            dtype_bytes = elem_type_to_bytes(node.type.tensor_type.elem_type)
-
-            # Calculate the total number of elements based on the shape
-            shape = str(node.type.tensor_type.shape.dim)
-            num_elements = np.prod([int(s) for s in shape.split() if s.isdigit()])
-
-            # Assign a total number of bytes to each node
-            nodes_bytes[node.name] = num_elements * dtype_bytes
-
-        return nodes_bytes
-
-    # Get the number of bytes of each of the inputs and outputs
-    model = onnx.load(onnx_path)
-    onnx_input_bytes = get_nodes_bytes(model.graph.input)
-    onnx_output_bytes = get_nodes_bytes(model.graph.output)
-
-    return int(sum(onnx_input_bytes.values())), int(sum(onnx_output_bytes.values()))
-
-
-def dtype_ort2str(dtype_str: str):
-    if dtype_str == "float16":
-        datatype = "float16"
-    elif dtype_str == "float":
-        datatype = "float32"
-    elif dtype_str == "double":
-        datatype = "float64"
-    elif dtype_str == "long":
-        datatype = "int64"
-    else:
-        datatype = dtype_str
-    return datatype
-
-
-def dummy_inputs(onnx_file: str) -> dict:
-    # Generate dummy inputs of the expected shape and type for the input model
-    sess_options = ort.SessionOptions()
-    sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
-    onnx_session = ort.InferenceSession(onnx_file, sess_options)
-    sess_input = onnx_session.get_inputs()
-
-    input_stats = []
-    for _idx, input_ in enumerate(range(len(sess_input))):
-        input_name = sess_input[input_].name
-        input_shape = sess_input[input_].shape
-
-        # TODO: Use onnx update_inputs_outputs_dims to automatically freeze models
-        for dim in input_shape:
-            if isinstance(dim, str) is True or math.isnan(dim) is True:
-                raise AssertionError(
-                    "Error: Model has dynamic inputs. Freeze the graph and try again"
-                )
-
-        input_type = sess_input[input_].type
-        input_stats.append([input_name, input_shape, input_type])
-
-    input_feed = {}
-    for stat in input_stats:
-        dtype_str = re.search(r"\((.*)\)", stat[2])
-        assert dtype_str is not None
-        datatype = dtype_ort2str(dtype_str.group(1))
-        input_feed[stat[0]] = np.random.rand(*stat[1]).astype(datatype)
-    return input_feed
-
-
-def get_opset(model: onnx.ModelProto) -> int:
-    return getattr(model.opset_import[0], "version", None)
-
-
-# This file was originally licensed under Apache 2.0. It has been modified.
-# Modifications Copyright (c) 2025 AMD

lemonade/common/plugins.py

@@ -1,10 +0,0 @@
-import pkgutil
-import importlib
-
-
-def discover():
-    return {
-        name: importlib.import_module(name)
-        for _, name, _ in pkgutil.iter_modules()
-        if name.startswith("turnkeyml_plugin_")
-    }