lemonade-sdk 8.1.10__tar.gz → 8.1.12__tar.gz

{lemonade_sdk-8.1.10/src/lemonade_sdk.egg-info → lemonade_sdk-8.1.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lemonade-sdk
-Version: 8.1.10
+Version: 8.1.12
 Summary: Lemonade SDK: Your LLM Aide for Validation and Deployment
 Author-email: lemonade@amd.com
 Requires-Python: >=3.10, <3.14
@@ -29,6 +29,7 @@ Requires-Dist: tabulate
 Requires-Dist: sentencepiece
 Requires-Dist: huggingface-hub[hf_xet]==0.33.0
 Requires-Dist: python-dotenv
+Requires-Dist: rumps>=0.4.0; sys_platform == "darwin"
 Provides-Extra: oga-ryzenai
 Requires-Dist: onnxruntime-genai-directml-ryzenai==0.7.0.2.1; extra == "oga-ryzenai"
 Requires-Dist: protobuf>=6.30.1; extra == "oga-ryzenai"
@@ -65,6 +66,8 @@ Dynamic: summary
     <img src="https://img.shields.io/badge/Windows-11-0078D6?logo=windows&logoColor=white" alt="Windows 11" /></a>
   <a href="https://lemonade-server.ai/#linux" title="Ubuntu 24.04 & 25.04 Supported">
     <img src="https://img.shields.io/badge/Ubuntu-24.04%20%7C%2025.04-E95420?logo=ubuntu&logoColor=white" alt="Ubuntu 24.04 | 25.04" /></a>
+  <a href="https://lemonade-server.ai/" title="macOS 14+ with Apple Silicon">
+    <img src="https://img.shields.io/badge/macOS-14%2B-000000?logo=apple&logoColor=white" alt="macOS 14+" /></a>
   <a href="docs/README.md#installation" title="Check out our instructions">
     <img src="https://img.shields.io/badge/Python-3.10--3.13-blue?logo=python&logoColor=white" alt="Made with Python" /></a>
   <a href="https://github.com/lemonade-sdk/lemonade/blob/main/docs/contribute.md" title="Contribution Guide">
@@ -152,11 +155,11 @@ You can also import custom GGUF and ONNX models from Hugging Face by using our [
 
 Lemonade supports the following configurations, while also making it easy to switch between them at runtime. Find more information about it [here](./docs/README.md#software-and-hardware-overview).
 
-| Hardware | Engine: OGA | Engine: llamacpp | Engine: HF | Windows | Linux |
-|----------|-------------|------------------|------------|---------|-------|
-| **🧠 CPU** | All platforms | All platforms | All platforms | ✅ | ✅ |
-| **🎮 GPU** | — | Vulkan: All platforms<br>ROCm: Selected AMD platforms* | — | ✅ | ✅ |
-| **🤖 NPU** | AMD Ryzen™ AI 300 series | — | — | ✅ | — |
+| Hardware | Engine: OGA | Engine: llamacpp | Engine: FLM | Windows | Linux | macOS |
+|----------|-------------|------------------|------------|---------|-------|-------|
+| **🧠 CPU** | All platforms | All platforms | - | ✅ | ✅ | ✅ |
+| **🎮 GPU** | — | Vulkan: All platforms<br>ROCm: Selected AMD platforms*<br>Metal: Apple Silicon | — | ✅ | ✅ | ✅ |
+| **🤖 NPU** | AMD Ryzen™ AI 300 series | — | Ryzen™ AI 300 series | ✅ | — | — |
 
 <details>
 <summary><small><i>* See supported AMD ROCm platforms</i></small></summary>
@@ -261,6 +264,7 @@ This project is:
   - [OnnxRuntime GenAI](https://github.com/microsoft/onnxruntime-genai)
   - [Hugging Face Hub](https://github.com/huggingface/huggingface_hub)
   - [OpenAI API](https://github.com/openai/openai-python)
+  - [IRON/MLIR-AIE](https://github.com/Xilinx/mlir-aie)
   - and more...
 - Accelerated by mentorship from the OCV Catalyst program.
 - Licensed under the [Apache 2.0 License](https://github.com/lemonade-sdk/lemonade/blob/main/LICENSE).

{lemonade_sdk-8.1.10 → lemonade_sdk-8.1.12}/README.md

@@ -9,6 +9,8 @@
     <img src="https://img.shields.io/badge/Windows-11-0078D6?logo=windows&logoColor=white" alt="Windows 11" /></a>
   <a href="https://lemonade-server.ai/#linux" title="Ubuntu 24.04 & 25.04 Supported">
     <img src="https://img.shields.io/badge/Ubuntu-24.04%20%7C%2025.04-E95420?logo=ubuntu&logoColor=white" alt="Ubuntu 24.04 | 25.04" /></a>
+  <a href="https://lemonade-server.ai/" title="macOS 14+ with Apple Silicon">
+    <img src="https://img.shields.io/badge/macOS-14%2B-000000?logo=apple&logoColor=white" alt="macOS 14+" /></a>
   <a href="docs/README.md#installation" title="Check out our instructions">
     <img src="https://img.shields.io/badge/Python-3.10--3.13-blue?logo=python&logoColor=white" alt="Made with Python" /></a>
   <a href="https://github.com/lemonade-sdk/lemonade/blob/main/docs/contribute.md" title="Contribution Guide">
@@ -96,11 +98,11 @@ You can also import custom GGUF and ONNX models from Hugging Face by using our [
 
 Lemonade supports the following configurations, while also making it easy to switch between them at runtime. Find more information about it [here](./docs/README.md#software-and-hardware-overview).
 
-| Hardware | Engine: OGA | Engine: llamacpp | Engine: HF | Windows | Linux |
-|----------|-------------|------------------|------------|---------|-------|
-| **🧠 CPU** | All platforms | All platforms | All platforms | ✅ | ✅ |
-| **🎮 GPU** | — | Vulkan: All platforms<br>ROCm: Selected AMD platforms* | — | ✅ | ✅ |
-| **🤖 NPU** | AMD Ryzen™ AI 300 series | — | — | ✅ | — |
+| Hardware | Engine: OGA | Engine: llamacpp | Engine: FLM | Windows | Linux | macOS |
+|----------|-------------|------------------|------------|---------|-------|-------|
+| **🧠 CPU** | All platforms | All platforms | - | ✅ | ✅ | ✅ |
+| **🎮 GPU** | — | Vulkan: All platforms<br>ROCm: Selected AMD platforms*<br>Metal: Apple Silicon | — | ✅ | ✅ | ✅ |
+| **🤖 NPU** | AMD Ryzen™ AI 300 series | — | Ryzen™ AI 300 series | ✅ | — | — |
 
 <details>
 <summary><small><i>* See supported AMD ROCm platforms</i></small></summary>
@@ -205,6 +207,7 @@ This project is:
   - [OnnxRuntime GenAI](https://github.com/microsoft/onnxruntime-genai)
   - [Hugging Face Hub](https://github.com/huggingface/huggingface_hub)
   - [OpenAI API](https://github.com/openai/openai-python)
+  - [IRON/MLIR-AIE](https://github.com/Xilinx/mlir-aie)
   - and more...
 - Accelerated by mentorship from the OCV Catalyst program.
 - Licensed under the [Apache 2.0 License](https://github.com/lemonade-sdk/lemonade/blob/main/LICENSE).

{lemonade_sdk-8.1.10 → lemonade_sdk-8.1.12}/setup.py

@@ -17,6 +17,7 @@ setup(
         "lemonade.tools.huggingface",
         "lemonade.tools.oga",
         "lemonade.tools.llamacpp",
+        "lemonade.tools.flm",
         "lemonade.tools.report",
         "lemonade.tools.server.utils",
         "lemonade.tools.server",
@@ -48,6 +49,8 @@ setup(
         "sentencepiece",
         "huggingface-hub[hf_xet]==0.33.0",
         "python-dotenv",
+        # macOS-specific dependencies
+        "rumps>=0.4.0; sys_platform == 'darwin'",
     ],
     extras_require={
         # The non-dev extras are meant to deploy specific backends into end-user

{lemonade_sdk-8.1.10 → lemonade_sdk-8.1.12}/src/lemonade/cache.py

@@ -43,7 +43,11 @@ def build_name(input_name):
     """
 
     if os.path.isdir(input_name):
+        # Input is a folder so no good way to determine a model name
         input_name_sanitized = "local_model"
+    elif os.path.isfile(input_name):
+        # Use the filename without its extension
+        input_name_sanitized = os.path.splitext(os.path.basename(input_name))[0]
     else:
         # Sanitize the input name
         input_name_sanitized = input_name.replace("/", "_")
@@ -63,8 +67,9 @@ class Keys:
     TOKEN_GENERATION_TOKENS_PER_SECOND = "token_generation_tokens_per_second"
     STD_DEV_TOKENS_PER_SECOND = "std_dev_tokens_per_second"
     SECONDS_TO_FIRST_TOKEN = "seconds_to_first_token"
-    PREFILL_TOKENS_PER_SECOND = "prefill_tokens_per_second"
     STD_DEV_SECONDS_TO_FIRST_TOKEN = "std_dev_seconds_to_first_token"
+    PREFILL_TOKENS_PER_SECOND = "prefill_tokens_per_second"
+    STD_DEV_PREFILL_TOKENS_PER_SECOND = "std_dev_prefill_tokens_per_second"
     CHECKPOINT = "checkpoint"
     DTYPE = "dtype"
     PROMPT = "prompt"

{lemonade_sdk-8.1.10 → lemonade_sdk-8.1.12}/src/lemonade/common/status.py

@@ -112,10 +112,10 @@ class UniqueInvocationInfo(BasicInfo):
         if print_file_name:
             print(f"{self.script_name}{self.extension}:")
 
-        # Print invocation about the model (only applies to scripts, not ONNX files or
+        # Print invocation about the model (only applies to scripts, not ONNX or GGUF files, nor
         # LLMs, which have no extension)
         if not (
-            self.extension == ".onnx"
+            self.extension in [".onnx", ".gguf"]
             or self.extension == build.state_file_name
             or self.extension == ""
         ):
@@ -138,7 +138,7 @@ class UniqueInvocationInfo(BasicInfo):
 
         if self.depth == 0:
             print(f"{self.indent}\tLocation:\t{self.file}", end="")
-            if self.extension == ".onnx":
+            if self.extension in [".onnx", ".gguf"]:
                 print()
             else:
                 print(f", line {self.line}")
@@ -314,7 +314,7 @@ class UniqueInvocationInfo(BasicInfo):
         Print information about a given model or submodel.
         """
 
-        if self.extension == ".onnx" or self.extension == "":
+        if self.extension in [".onnx", ".gguf"] or self.extension == "":
             self.indent = "\t" * (2 * self.depth)
         else:
             self.indent = "\t" * (2 * self.depth + 1)

{lemonade_sdk-8.1.10 → lemonade_sdk-8.1.12}/src/lemonade/tools/bench.py

@@ -29,7 +29,9 @@ class Bench(Tool, ABC):
             Keys.SECONDS_TO_FIRST_TOKEN,
             Keys.STD_DEV_SECONDS_TO_FIRST_TOKEN,
             Keys.TOKEN_GENERATION_TOKENS_PER_SECOND,
+            Keys.STD_DEV_TOKENS_PER_SECOND,
             Keys.PREFILL_TOKENS_PER_SECOND,
+            Keys.STD_DEV_PREFILL_TOKENS_PER_SECOND,
             Keys.PROMPT_TOKENS,
             Keys.RESPONSE_TOKENS,
             Keys.MAX_MEMORY_USED_GBYTE,
@@ -42,7 +44,9 @@ class Bench(Tool, ABC):
         self.mean_time_to_first_token_list = []
         self.std_dev_time_to_first_token_list = []
         self.prefill_tokens_per_second_list = []
+        self.std_dev_prefill_tokens_per_second_list = []
         self.token_generation_tokens_per_second_list = []
+        self.std_dev_token_generation_tokens_per_second_list = []
         self.max_memory_used_gb_list = []
 
         # Max memory used can only be measured on Windows systems
@@ -88,7 +92,7 @@ class Bench(Tool, ABC):
             default=[str(default_prompt_length)],
             metavar="PROMPT",
             help="Input one or more prompts to the LLM. Three formats are supported. "
-            "1) integer: use a synthetic prompt with the specified length "
+            "1) integer: use a synthetic prompt with the specified token length "
             "2) str: use a user-provided prompt string "
             "3) path/to/prompt.txt: load the prompt from a text file. "
             f"(default: {default_prompt_length}) ",
@@ -246,10 +250,27 @@ class Bench(Tool, ABC):
             Keys.PREFILL_TOKENS_PER_SECOND,
             self.get_item_or_list(self.prefill_tokens_per_second_list),
         )
+        if not all(
+            element is None for element in self.std_dev_prefill_tokens_per_second_list
+        ):
+            state.save_stat(
+                Keys.STD_DEV_PREFILL_TOKENS_PER_SECOND,
+                self.get_item_or_list(self.std_dev_prefill_tokens_per_second_list),
+            )
         state.save_stat(
             Keys.TOKEN_GENERATION_TOKENS_PER_SECOND,
             self.get_item_or_list(self.token_generation_tokens_per_second_list),
         )
+        if not all(
+            element is None
+            for element in self.std_dev_token_generation_tokens_per_second_list
+        ):
+            state.save_stat(
+                Keys.STD_DEV_TOKENS_PER_SECOND,
+                self.get_item_or_list(
+                    self.std_dev_token_generation_tokens_per_second_list
+                ),
+            )
         if self.save_max_memory_used:
             state.save_stat(
                 Keys.MAX_MEMORY_USED_GBYTE,

lemonade_sdk-8.1.12/src/lemonade/tools/flm/__init__.py

@@ -0,0 +1 @@
+# FLM (FastFlowLM) utilities for Lemonade SDK

lemonade_sdk-8.1.12/src/lemonade/tools/flm/utils.py

@@ -0,0 +1,255 @@
+"""
+FLM (FastFlowLM) utilities for installation, version checking, and model management.
+"""
+
+import os
+import logging
+import subprocess
+import tempfile
+import time
+from typing import List, Optional
+
+import requests
+from packaging.version import Version
+
+
+FLM_MINIMUM_VERSION = "0.9.12"
+
+
+def check_flm_version() -> Optional[str]:
+    """
+    Check if FLM is installed and return version, or None if not available.
+    """
+    try:
+        result = subprocess.run(
+            ["flm", "version"],
+            capture_output=True,
+            text=True,
+            check=True,
+            encoding="utf-8",
+            errors="replace",
+        )
+
+        # Parse version from output like "FLM v0.9.4"
+        output = result.stdout.strip()
+        if output.startswith("FLM v"):
+            version_str = output[5:]  # Remove "FLM v" prefix
+            return version_str
+        return None
+
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return None
+
+
+def refresh_environment():
+    """
+    Refresh PATH to pick up newly installed executables.
+    """
+    if os.name == "nt":  # Windows
+        # On Windows, we need to refresh the PATH from registry
+        import winreg
+
+        try:
+            with winreg.OpenKey(
+                winreg.HKEY_LOCAL_MACHINE,
+                r"SYSTEM\CurrentControlSet\Control\Session Manager\Environment",
+            ) as key:
+                path_value, _ = winreg.QueryValueEx(key, "PATH")
+                os.environ["PATH"] = path_value + ";" + os.environ.get("PATH", "")
+        except Exception as e:  # pylint: disable=broad-except
+            logging.debug("Could not refresh PATH from registry: %s", e)
+
+        # Also try to add common installation paths
+        common_paths = [
+            r"C:\Program Files\FLM",
+            r"C:\Program Files (x86)\FLM",
+            os.path.expanduser(r"~\AppData\Local\FLM"),
+        ]
+        for path in common_paths:
+            if os.path.exists(path) and path not in os.environ.get("PATH", ""):
+                os.environ["PATH"] = path + ";" + os.environ.get("PATH", "")
+
+
+def install_flm():
+    """
+    Check if FLM is installed and at minimum version.
+    If not, download and run the GUI installer, then wait for completion.
+    """
+    # Check current FLM installation
+    current_version = check_flm_version()
+
+    if current_version and Version(current_version) >= Version(FLM_MINIMUM_VERSION):
+        logging.info(
+            "FLM v%s is already installed and meets minimum version requirement (v%s)",
+            current_version,
+            FLM_MINIMUM_VERSION,
+        )
+        return
+
+    if current_version:
+        logging.info(
+            "FLM v%s is installed but below minimum version v%s. Upgrading...",
+            current_version,
+            FLM_MINIMUM_VERSION,
+        )
+    else:
+        logging.info(
+            "FLM not found. Installing FLM v%s or later...", FLM_MINIMUM_VERSION
+        )
+
+    # Download the installer
+    # pylint: disable=line-too-long
+    installer_url = "https://github.com/FastFlowLM/FastFlowLM/releases/latest/download/flm-setup.exe"
+    installer_path = os.path.join(tempfile.gettempdir(), "flm-setup.exe")
+
+    try:
+        # Remove existing installer if present
+        if os.path.exists(installer_path):
+            os.remove(installer_path)
+
+        logging.info("Downloading FLM installer...")
+        response = requests.get(installer_url, stream=True, timeout=30)
+        response.raise_for_status()
+
+        # Save installer to disk
+        with open(installer_path, "wb") as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                f.write(chunk)
+            f.flush()
+            os.fsync(f.fileno())
+
+        logging.info("Downloaded FLM installer to %s", installer_path)
+
+        # Launch the installer GUI
+        logging.warning(
+            "Launching FLM installer GUI. Please complete the installation..."
+        )
+
+        # Launch installer and wait for it to complete
+        if os.name == "nt":  # Windows
+            process = subprocess.Popen([installer_path], shell=True)
+        else:
+            process = subprocess.Popen([installer_path])
+
+        # Wait for installer to complete
+        process.wait()
+
+        if process.returncode != 0:
+            raise RuntimeError(
+                f"FLM installer failed with exit code {process.returncode}"
+            )
+
+        logging.info("FLM installer completed successfully")
+
+        # Refresh environment to pick up new PATH entries
+        refresh_environment()
+
+        # Wait a moment for system to update
+        time.sleep(2)
+
+        # Verify installation
+        max_retries = 10
+        for attempt in range(max_retries):
+            new_version = check_flm_version()
+            if new_version and Version(new_version) >= Version(FLM_MINIMUM_VERSION):
+                logging.info("FLM v%s successfully installed and verified", new_version)
+                return
+
+            if attempt < max_retries - 1:
+                logging.debug(
+                    "FLM not yet available in PATH, retrying... (attempt %d/%d)",
+                    attempt + 1,
+                    max_retries,
+                )
+                time.sleep(3)
+                refresh_environment()
+
+        # Final check failed
+        raise RuntimeError(
+            "FLM installation completed but 'flm' command is not available in PATH. "
+            "Please ensure FLM is properly installed and available in your system PATH."
+        )
+
+    except requests.RequestException as e:
+        raise RuntimeError(f"Failed to download FLM installer: {e}") from e
+    except Exception as e:
+        raise RuntimeError(f"FLM installation failed: {e}") from e
+    finally:
+        # Clean up installer file
+        if os.path.exists(installer_path):
+            try:
+                os.remove(installer_path)
+            except OSError:
+                pass  # Ignore cleanup errors
+
+
+def download_flm_model(config_checkpoint, _=None, do_not_upgrade=False) -> dict:
+    """
+    Downloads the FLM model for the given configuration.
+
+    Args:
+        config_checkpoint: name of the FLM model to install.
+        _: placeholder for `config_mmproj`, which is standard
+            for WrappedServer (see llamacpp/utils.py) .
+        do_not_upgrade: whether to re-download the model if it is already
+            available.
+    """
+
+    if do_not_upgrade:
+        command = ["flm", "pull", f"{config_checkpoint}"]
+    else:
+        command = ["flm", "pull", f"{config_checkpoint}", "--force"]
+
+    subprocess.run(command, check=True)
+
+
+def get_flm_installed_models() -> List[str]:
+    """
+    Parse FLM model list and return installed model checkpoints.
+
+    Returns:
+        List of installed FLM model checkpoints (e.g., ["llama3.2:1b", "gemma3:4b"])
+    """
+    try:
+        result = subprocess.run(
+            ["flm", "list"],
+            capture_output=True,
+            text=True,
+            check=True,
+            encoding="utf-8",
+            errors="replace",
+        )
+
+        # Check if we got valid output
+        if not result.stdout:
+            return []
+
+        installed_checkpoints = []
+
+        lines = result.stdout.strip().split("\n")
+        for line in lines:
+            line = line.strip()
+            if line.startswith("- "):
+                # Remove the leading "- " and parse the model info
+                model_info = line[2:].strip()
+
+                # Check if model is installed (✅)
+                if model_info.endswith(" ✅"):
+                    checkpoint = model_info[:-2].strip()
+                    installed_checkpoints.append(checkpoint)
+
+        return installed_checkpoints
+
+    except (subprocess.CalledProcessError, FileNotFoundError, AttributeError):
+        # FLM not installed, not available, or output parsing failed
+        return []
+
+
+def is_flm_available() -> bool:
+    """
+    Check if FLM is available and meets minimum version requirements.
+    """
+    current_version = check_flm_version()
+    return current_version is not None and Version(current_version) >= Version(
+        FLM_MINIMUM_VERSION
+    )