lemonade-sdk 8.1.1__tar.gz → 8.1.3__tar.gz

{lemonade_sdk-8.1.1/src/lemonade_sdk.egg-info → lemonade_sdk-8.1.3}/PKG-INFO

@@ -1,18 +1,18 @@
 Metadata-Version: 2.4
 Name: lemonade-sdk
-Version: 8.1.1
+Version: 8.1.3
 Summary: Lemonade SDK: Your LLM Aide for Validation and Deployment
 Author-email: lemonade@amd.com
-Requires-Python: >=3.10, <3.13
+Requires-Python: >=3.10, <3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: NOTICE.md
 Requires-Dist: invoke>=2.0.0
-Requires-Dist: onnx<1.18.0,>=1.11.0
+Requires-Dist: onnx==1.18.0
 Requires-Dist: pyyaml>=5.4
 Requires-Dist: typeguard>=2.3.13
 Requires-Dist: packaging>=20.9
-Requires-Dist: numpy<2.0.0
+Requires-Dist: numpy
 Requires-Dist: fasteners
 Requires-Dist: GitPython>=3.1.40
 Requires-Dist: psutil>=6.1.1
@@ -41,9 +41,10 @@ Requires-Dist: accelerate; extra == "dev"
 Requires-Dist: datasets; extra == "dev"
 Requires-Dist: pandas>=1.5.3; extra == "dev"
 Requires-Dist: matplotlib; extra == "dev"
-Requires-Dist: model-generate==1.5.0; (platform_system == "Windows" and python_version == "3.10") and extra == "dev"
 Requires-Dist: human-eval-windows==1.0.4; extra == "dev"
 Requires-Dist: lm-eval[api]; extra == "dev"
+Provides-Extra: model-generate
+Requires-Dist: model-generate==1.5.0; (platform_system == "Windows" and python_version == "3.10") and extra == "model-generate"
 Provides-Extra: oga-hybrid
 Requires-Dist: lemonade-sdk[oga-ryzenai]; extra == "oga-hybrid"
 Provides-Extra: oga-unified
@@ -105,7 +106,7 @@ Dynamic: summary
     <img src="https://img.shields.io/badge/Ubuntu-24.04%20%7C%2025.04-E95420?logo=ubuntu&logoColor=white" alt="Ubuntu 24.04 | 25.04" />
   </a>
   <a href="docs/README.md#installation" title="Check out our instructions">
-    <img src="https://img.shields.io/badge/Python-3.10%20%7C%203.12-blue?logo=python&logoColor=white" alt="Made with Python" />
+    <img src="https://img.shields.io/badge/Python-3.10--3.13-blue?logo=python&logoColor=white" alt="Made with Python" />
   </a>
   <a href="https://github.com/lemonade-sdk/lemonade/blob/main/docs/contribute.md" title="Contribution Guide">
     <img src="https://img.shields.io/badge/PRs-welcome-brightgreen.svg" alt="PRs Welcome" />

{lemonade_sdk-8.1.1 → lemonade_sdk-8.1.3}/README.md

@@ -14,7 +14,7 @@
     <img src="https://img.shields.io/badge/Ubuntu-24.04%20%7C%2025.04-E95420?logo=ubuntu&logoColor=white" alt="Ubuntu 24.04 | 25.04" />
   </a>
   <a href="docs/README.md#installation" title="Check out our instructions">
-    <img src="https://img.shields.io/badge/Python-3.10%20%7C%203.12-blue?logo=python&logoColor=white" alt="Made with Python" />
+    <img src="https://img.shields.io/badge/Python-3.10--3.13-blue?logo=python&logoColor=white" alt="Made with Python" />
   </a>
   <a href="https://github.com/lemonade-sdk/lemonade/blob/main/docs/contribute.md" title="Contribution Guide">
     <img src="https://img.shields.io/badge/PRs-welcome-brightgreen.svg" alt="PRs Welcome" />

lemonade_sdk-8.1.3/pyproject.toml

@@ -0,0 +1,8 @@
+[build-system]
+requires = [
+  "setuptools>=68",
+  "wheel"
+]
+build-backend = "setuptools.build_meta"
+
+

{lemonade_sdk-8.1.1 → lemonade_sdk-8.1.3}/setup.py

@@ -28,13 +28,11 @@ setup(
         # Minimal dependencies required for end-users who are running
         # apps deployed on Lemonade SDK
         "invoke>=2.0.0",
-        "onnx>=1.11.0,<1.18.0",
+        "onnx==1.18.0",
         "pyyaml>=5.4",
         "typeguard>=2.3.13",
         "packaging>=20.9",
-        # Necessary until upstream packages account for the breaking
-        # change to numpy
-        "numpy<2.0.0",
+        "numpy",
         "fasteners",
         "GitPython>=3.1.40",
         "psutil>=6.1.1",
@@ -74,12 +72,14 @@ setup(
             "datasets",
             "pandas>=1.5.3",
             "matplotlib",
-            "model-generate==1.5.0; platform_system=='Windows' and python_version=='3.10'",
             # Install human-eval from a forked repo with Windows support until the
             # PR (https://github.com/openai/human-eval/pull/53) is merged
             "human-eval-windows==1.0.4",
             "lm-eval[api]",
         ],
+        "model-generate": [
+            "model-generate==1.5.0; platform_system=='Windows' and python_version=='3.10'",
+        ],
         # Keep backwards compatibility for old extras names
         "oga-hybrid": ["lemonade-sdk[oga-ryzenai]"],
         "oga-unified": ["lemonade-sdk[oga-ryzenai]"],
@@ -125,15 +125,16 @@ setup(
             "lemonade=lemonade:lemonadecli",
             "lemonade-install=lemonade_install:installcli",
             "lemonade-server-dev=lemonade_server.cli:main",
+            "lsdev=lemonade_server.cli:developer_entrypoint",
         ]
     },
-    python_requires=">=3.10, <3.13",
+    python_requires=">=3.10, <3.14",
     long_description=open("README.md", "r", encoding="utf-8").read(),
     long_description_content_type="text/markdown",
     include_package_data=True,
     package_data={
         "lemonade_server": ["server_models.json"],
-        "lemonade": ["tools/server/static/*"],
+        "lemonade": ["tools/server/static/**/*"],
     },
 )
 

{lemonade_sdk-8.1.1 → lemonade_sdk-8.1.3}/src/lemonade/common/inference_engines.py

@@ -5,7 +5,6 @@ import importlib.metadata
 import subprocess
 from abc import ABC, abstractmethod
 from typing import Dict, Optional
-import transformers
 
 
 class InferenceEngineDetector:
@@ -352,6 +351,7 @@ class TransformersDetector(BaseEngineDetector):
 
         try:
             import torch
+            import transformers
 
             if device_type == "cpu":
                 result = {

{lemonade_sdk-8.1.1 → lemonade_sdk-8.1.3}/src/lemonade/tools/llamacpp/utils.py

@@ -57,7 +57,7 @@ def identify_rocm_arch_from_name(device_name: str) -> str | None:
     return None
 
 
-def identify_rocm_arch_and_hip_id() -> tuple[str, str]:
+def identify_rocm_arch() -> str:
     """
     Identify the appropriate ROCm target architecture based on the device info
     Returns tuple of (architecture, gpu_type) where gpu_type is 'igpu' or 'dgpu'
@@ -68,21 +68,54 @@ def identify_rocm_arch_and_hip_id() -> tuple[str, str]:
     amd_igpu = system_info.get_amd_igpu_device()
     amd_dgpu = system_info.get_amd_dgpu_devices()
     target_arch = None
-    gpu_count = 0
     for gpu in [amd_igpu] + amd_dgpu:
         if gpu.get("available") and gpu.get("name"):
-            gpu_count += 1
             target_arch = identify_rocm_arch_from_name(gpu["name"].lower())
             if target_arch:
                 break
 
-    # Get HIP ID based on the number of GPUs available
-    # Here, we assume that the iGPU will always show up before the dGPUs (if available)
-    # We also assume that selecting the dGPU is preferred over the iGPU
-    # Multiple GPUs are not supported at the moment
-    hip_id = str(gpu_count - 1)
+    return target_arch
 
-    return target_arch, hip_id
+
+def identify_hip_id() -> str:
+    """
+    Identify the HIP ID
+    """
+    # Get HIP devices
+    hip_devices = get_hip_devices()
+    logging.debug(f"HIP devices found: {hip_devices}")
+    if len(hip_devices) == 0:
+        raise ValueError("No HIP devices found when identifying HIP ID")
+
+    # Identify HIP devices that are compatible with our ROCm builds
+    rocm_devices = []
+    for device in hip_devices:
+        device_id, device_name = device
+        if identify_rocm_arch_from_name(device_name):
+            rocm_devices.append([device_id, device_name])
+    logging.debug(f"ROCm devices found: {rocm_devices}")
+
+    # If no ROCm devices are found, use the last HIP device
+    # This might be needed in some scenarios where HIP reports generic device names
+    # Example: "AMD Radeon Graphics" for STX Halo iGPU on Ubuntu 24.04
+    if len(rocm_devices) == 0:
+        rocm_devices = [hip_devices[-1]]
+        logging.warning(
+            "No ROCm devices found when identifying HIP ID. "
+            f"Falling back to the following device: {rocm_devices[0]}"
+        )
+    elif len(rocm_devices) > 1:
+        logging.warning(
+            f"Multiple ROCm devices found when identifying HIP ID: {rocm_devices}"
+            "The last device will be used."
+        )
+
+    # Select the last device
+    device_selected = rocm_devices[-1]
+    logging.debug(f"Selected ROCm device: {device_selected}")
+
+    # Return the device ID
+    return device_selected[0]
 
 
 def get_llama_version(backend: str) -> str:
@@ -277,7 +310,7 @@ def install_llamacpp(backend):
         target_arch = None
         if backend == "rocm":
             # Identify the target architecture
-            target_arch, hip_id = identify_rocm_arch_and_hip_id()
+            target_arch = identify_rocm_arch()
             if not target_arch:
                 system = platform.system().lower()
                 if system == "linux":
@@ -293,10 +326,6 @@ def install_llamacpp(backend):
                     f"for supported configurations. {hint}"
                 )
 
-            # Set HIP_VISIBLE_DEVICES=0 for igpu, =1 for dgpu
-            env_file_path = os.path.join(llama_server_exe_dir, ".env")
-            set_key(env_file_path, "HIP_VISIBLE_DEVICES", hip_id)
-
         # Direct download for Vulkan/ROCm
         llama_archive_url, filename = get_binary_url_and_filename(backend, target_arch)
         llama_archive_path = os.path.join(llama_server_exe_dir, filename)
@@ -315,6 +344,12 @@ def install_llamacpp(backend):
         else:
             raise NotImplementedError(f"Unsupported archive format: {filename}")
 
+        # Identify and set HIP ID
+        if backend == "rocm":
+            hip_id = identify_hip_id()
+            env_file_path = os.path.join(llama_server_exe_dir, ".env")
+            set_key(env_file_path, "HIP_VISIBLE_DEVICES", str(hip_id))
+
         # Make executable on Linux - need to update paths after extraction
         if platform.system().lower() == "linux":
             # Re-get the paths since extraction might have changed the directory structure
@@ -778,3 +813,68 @@ class LlamaCppAdapter(ModelAdapter):
             error_msg = f"Failed to run llama.cpp command: {str(e)}\n"
             error_msg += f"Command: {' '.join(cmd)}"
             raise Exception(error_msg)
+
+
+def get_hip_devices():
+    """Get list of HIP devices with their IDs and names."""
+    import ctypes
+    import sys
+    import os
+    import glob
+    from ctypes import c_int, POINTER
+    from ctypes.util import find_library
+
+    # Get llama.cpp path
+    rocm_path = get_llama_folder_path("rocm")
+
+    # Load HIP library
+    hip_library_pattern = (
+        "amdhip64*.dll" if sys.platform.startswith("win") else "libamdhip64*.so"
+    )
+    search_pattern = os.path.join(rocm_path, hip_library_pattern)
+    matching_files = glob.glob(search_pattern)
+    if not matching_files:
+        raise RuntimeError(
+            f"Could not find HIP runtime library matching pattern: {search_pattern}"
+        )
+    try:
+        libhip = ctypes.CDLL(matching_files[0])
+    except OSError:
+        raise RuntimeError(f"Could not load HIP runtime library from {path}")
+
+    # Setup function signatures
+    hipError_t = c_int
+    hipDeviceProp_t = ctypes.c_char * 2048
+    libhip.hipGetDeviceCount.restype = hipError_t
+    libhip.hipGetDeviceCount.argtypes = [POINTER(c_int)]
+    libhip.hipGetDeviceProperties.restype = hipError_t
+    libhip.hipGetDeviceProperties.argtypes = [POINTER(hipDeviceProp_t), c_int]
+    libhip.hipGetErrorString.restype = ctypes.c_char_p
+    libhip.hipGetErrorString.argtypes = [hipError_t]
+
+    # Get device count
+    device_count = c_int()
+    err = libhip.hipGetDeviceCount(ctypes.byref(device_count))
+    if err != 0:
+        logging.error(
+            "hipGetDeviceCount failed:", libhip.hipGetErrorString(err).decode()
+        )
+        return []
+
+    # Get device properties
+    devices = []
+    for i in range(device_count.value):
+        prop = hipDeviceProp_t()
+        err = libhip.hipGetDeviceProperties(ctypes.byref(prop), i)
+        if err != 0:
+            logging.error(
+                f"hipGetDeviceProperties failed for device {i}:",
+                libhip.hipGetErrorString(err).decode(),
+            )
+            continue
+
+        # Extract device name from HIP device properties
+        device_name = ctypes.string_at(prop, 256).decode("utf-8").rstrip("\x00")
+        devices.append([i, device_name])
+
+    return devices

{lemonade_sdk-8.1.1 → lemonade_sdk-8.1.3}/src/lemonade/tools/management_tools.py

@@ -109,7 +109,7 @@ class Cache(ManagementTool):
     # pylint: disable=pointless-statement,f-string-without-interpolation
     f"""
     A set of functions for managing the lemonade build cache. The default
-    cache location is {lemonade_cache.DEFAULT_CACHE_DIR}, and can also be 
+    cache location is {lemonade_cache.DEFAULT_CACHE_DIR}, and can also be
     selected with
     the global --cache-dir option or the LEMONADE_CACHE_DIR environment variable.
 

{lemonade_sdk-8.1.1 → lemonade_sdk-8.1.3}/src/lemonade/tools/oga/utils.py

@@ -100,9 +100,10 @@ class OrtGenaiModel(ModelAdapter):
         max_new_tokens=512,
         min_new_tokens=0,
         do_sample=True,
-        top_k=50,
-        top_p=1.0,
-        temperature=0.7,
+        top_k=None,
+        top_p=None,
+        temperature=None,
+        repeat_penalty=None,
         streamer: OrtGenaiStreamer = None,
         pad_token_id=None,
         stopping_criteria=None,
@@ -154,38 +155,58 @@ class OrtGenaiModel(ModelAdapter):
         if random_seed is None:
             random_seed = -1  # In og.Generator, -1 = seed with random device
 
+        # Get search config if available, otherwise use empty dict
+        # Thanks to the empty dict, if the model doesn't have a built-in search
+        #   config, the .get() calls will all just use the default values
+        search_config = {}
         if self.config and "search" in self.config:
             search_config = self.config["search"]
-            params.set_search_options(
-                do_sample=search_config.get("do_sample", do_sample),
-                top_k=search_config.get("top_k", top_k),
-                top_p=search_config.get("top_p", top_p),
-                temperature=search_config.get("temperature", temperature),
-                max_length=max_length_to_use,
-                min_length=min_length,
-                early_stopping=search_config.get("early_stopping", False),
-                length_penalty=search_config.get("length_penalty", 1.0),
-                num_beams=search_config.get("num_beams", 1),
-                num_return_sequences=search_config.get("num_return_sequences", 1),
-                repetition_penalty=search_config.get("repetition_penalty", 1.0),
-                past_present_share_buffer=search_config.get(
-                    "past_present_share_buffer", True
-                ),
-                random_seed=random_seed,
-                # Not currently supported by OGA
-                # diversity_penalty=search_config.get('diversity_penalty', 0.0),
-                # no_repeat_ngram_size=search_config.get('no_repeat_ngram_size', 0),
-            )
-        else:
-            params.set_search_options(
-                do_sample=do_sample,
-                top_k=top_k,
-                top_p=top_p,
-                temperature=temperature,
-                max_length=max_length_to_use,
-                min_length=min_length,
-                random_seed=random_seed,
-            )
+
+        # Apply parameter hierarchy: user provided > search config > defaults
+        default_top_k = 50
+        default_top_p = 1.0
+        default_temperature = 0.7
+        default_repetition_penalty = 1.0
+
+        top_k_to_use = (
+            top_k if top_k is not None else search_config.get("top_k", default_top_k)
+        )
+        top_p_to_use = (
+            top_p if top_p is not None else search_config.get("top_p", default_top_p)
+        )
+        temperature_to_use = (
+            temperature
+            if temperature is not None
+            else search_config.get("temperature", default_temperature)
+        )
+        # Map the llamacpp name, `repeat_penalty`, to the OGA name, `repetition_penalty`
+        repetition_penalty_to_use = (
+            repeat_penalty
+            if repeat_penalty is not None
+            else search_config.get("repetition_penalty", default_repetition_penalty)
+        )
+
+        # Set search options once with all parameters
+        params.set_search_options(
+            do_sample=search_config.get("do_sample", do_sample),
+            top_k=top_k_to_use,
+            top_p=top_p_to_use,
+            temperature=temperature_to_use,
+            repetition_penalty=repetition_penalty_to_use,
+            max_length=max_length_to_use,
+            min_length=min_length,
+            early_stopping=search_config.get("early_stopping", False),
+            length_penalty=search_config.get("length_penalty", 1.0),
+            num_beams=search_config.get("num_beams", 1),
+            num_return_sequences=search_config.get("num_return_sequences", 1),
+            past_present_share_buffer=search_config.get(
+                "past_present_share_buffer", True
+            ),
+            random_seed=random_seed,
+            # Not currently supported by OGA
+            # diversity_penalty=search_config.get('diversity_penalty', 0.0),
+            # no_repeat_ngram_size=search_config.get('no_repeat_ngram_size', 0),
+        )
         params.try_graph_capture_with_max_batch_size(1)
 
         generator = og.Generator(self.model, params)

{lemonade_sdk-8.1.1 → lemonade_sdk-8.1.3}/src/lemonade/tools/server/llamacpp.py

@@ -43,6 +43,72 @@ def llamacpp_address(port: int) -> str:
     return f"http://127.0.0.1:{port}/v1"
 
 
+def _separate_openai_params(request_dict: dict, endpoint_type: str = "chat") -> dict:
+    """
+    Separate standard OpenAI parameters from custom llama.cpp parameters.
+
+    Args:
+        request_dict: Dictionary of all request parameters
+        endpoint_type: Type of endpoint ("chat" or "completion")
+
+    Returns:
+        Dictionary with parameters properly separated for OpenAI client
+    """
+    openai_client_params = {}
+    extra_params = {}
+
+    # Common OpenAI parameters for both endpoint types
+    common_params = {
+        "model",
+        "frequency_penalty",
+        "logit_bias",
+        "logprobs",
+        "max_tokens",
+        "n",
+        "presence_penalty",
+        "seed",
+        "stop",
+        "stream",
+        "temperature",
+        "top_p",
+        "user",
+    }
+
+    # Standard OpenAI parameters by endpoint type
+    if endpoint_type == "chat":
+        chat_specific_params = {
+            "messages",
+            "top_logprobs",
+            "response_format",
+            "service_tier",
+            "stream_options",
+            "tools",
+            "tool_choice",
+            "parallel_tool_calls",
+        }
+        openai_params = common_params | chat_specific_params
+    else:  # completion
+        completion_specific_params = {
+            "prompt",
+            "best_of",
+            "echo",
+            "suffix",
+        }
+        openai_params = common_params | completion_specific_params
+
+    for key, value in request_dict.items():
+        if key in openai_params:
+            openai_client_params[key] = value
+        else:
+            extra_params[key] = value
+
+    # If there are custom parameters, use extra_body to pass them through
+    if extra_params:
+        openai_client_params["extra_body"] = extra_params
+
+    return openai_client_params
+
+
 class LlamaTelemetry:
     """
     Manages telemetry data collection and display for llama server.
@@ -226,6 +292,11 @@ def _launch_llama_subprocess(
         "--ctx-size",
         str(ctx_size),
     ]
+
+    # Lock random seed for deterministic behavior in CI
+    if os.environ.get("LEMONADE_CI_MODE"):
+        base_command.extend(["--seed", "42"])
+
     if "mmproj" in snapshot_files:
         base_command.extend(["--mmproj", snapshot_files["mmproj"]])
         if not use_gpu:
@@ -238,6 +309,15 @@ def _launch_llama_subprocess(
     # Add port and jinja to enable tool use
     base_command.extend(["--port", str(telemetry.port), "--jinja"])
 
+    # Disable jinja for gpt-oss-120b on Vulkan
+    if backend == "vulkan" and "gpt-oss-120b" in snapshot_files["variant"].lower():
+        base_command.remove("--jinja")
+        logging.warning(
+            "Jinja is disabled for gpt-oss-120b on Vulkan due to a llama.cpp bug "
+            "(see https://github.com/ggml-org/llama.cpp/issues/15274). "
+            "The model cannot use tools. If needed, use the ROCm backend instead."
+        )
+
     # Use legacy reasoning formatting, since not all apps support the new
     # reasoning_content field
     base_command.extend(["--reasoning-format", "none"])
@@ -384,13 +464,17 @@ def chat_completion(
         exclude_unset=True, exclude_none=True
     )
 
+    # Separate standard OpenAI parameters from custom llama.cpp parameters
+    openai_client_params = _separate_openai_params(request_dict, "chat")
+
     # Check if streaming is requested
     if chat_completion_request.stream:
 
         def event_stream():
             try:
                 # Enable streaming
-                for chunk in client.chat.completions.create(**request_dict):
+                # pylint: disable=missing-kwoa
+                for chunk in client.chat.completions.create(**openai_client_params):
                     yield f"data: {chunk.model_dump_json()}\n\n"
                 yield "data: [DONE]\n\n"
 
@@ -412,7 +496,8 @@ def chat_completion(
         # Non-streaming response
         try:
             # Disable streaming for non-streaming requests
-            response = client.chat.completions.create(**request_dict)
+            # pylint: disable=missing-kwoa
+            response = client.chat.completions.create(**openai_client_params)
 
             # Show telemetry after completion
             telemetry.show_telemetry()
@@ -420,6 +505,7 @@ def chat_completion(
             return response
 
         except Exception as e:  # pylint: disable=broad-exception-caught
+            logging.error("Error during chat completion: %s", str(e))
             raise HTTPException(
                 status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                 detail=f"Chat completion error: {str(e)}",
@@ -446,13 +532,17 @@ def completion(completion_request: CompletionRequest, telemetry: LlamaTelemetry)
     # Convert Pydantic model to dict and remove unset/null values
     request_dict = completion_request.model_dump(exclude_unset=True, exclude_none=True)
 
+    # Separate standard OpenAI parameters from custom llama.cpp parameters
+    openai_client_params = _separate_openai_params(request_dict, "completion")
+
     # Check if streaming is requested
     if completion_request.stream:
 
         def event_stream():
             try:
                 # Enable streaming
-                for chunk in client.completions.create(**request_dict):
+                # pylint: disable=missing-kwoa
+                for chunk in client.completions.create(**openai_client_params):
                     yield f"data: {chunk.model_dump_json()}\n\n"
                 yield "data: [DONE]\n\n"
 
@@ -474,7 +564,8 @@ def completion(completion_request: CompletionRequest, telemetry: LlamaTelemetry)
         # Non-streaming response
         try:
             # Disable streaming for non-streaming requests
-            response = client.completions.create(**request_dict)
+            # pylint: disable=missing-kwoa
+            response = client.completions.create(**openai_client_params)
 
             # Show telemetry after completion
             telemetry.show_telemetry()
@@ -482,6 +573,7 @@ def completion(completion_request: CompletionRequest, telemetry: LlamaTelemetry)
             return response
 
         except Exception as e:  # pylint: disable=broad-exception-caught
+            logging.error("Error during completion: %s", str(e))
             raise HTTPException(
                 status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                 detail=f"Completion error: {str(e)}",