lemonade-sdk 8.1.9__py3-none-any.whl → 8.1.11__py3-none-any.whl

lemonade/tools/flm/__init__.py

@@ -0,0 +1 @@
+# FLM (FastFlowLM) utilities for Lemonade SDK

lemonade/tools/flm/utils.py

@@ -0,0 +1,255 @@
+"""
+FLM (FastFlowLM) utilities for installation, version checking, and model management.
+"""
+
+import os
+import logging
+import subprocess
+import tempfile
+import time
+from typing import List, Optional
+
+import requests
+from packaging.version import Version
+
+
+FLM_MINIMUM_VERSION = "0.9.10"
+
+
+def check_flm_version() -> Optional[str]:
+    """
+    Check if FLM is installed and return version, or None if not available.
+    """
+    try:
+        result = subprocess.run(
+            ["flm", "version"],
+            capture_output=True,
+            text=True,
+            check=True,
+            encoding="utf-8",
+            errors="replace",
+        )
+
+        # Parse version from output like "FLM v0.9.4"
+        output = result.stdout.strip()
+        if output.startswith("FLM v"):
+            version_str = output[5:]  # Remove "FLM v" prefix
+            return version_str
+        return None
+
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return None
+
+
+def refresh_environment():
+    """
+    Refresh PATH to pick up newly installed executables.
+    """
+    if os.name == "nt":  # Windows
+        # On Windows, we need to refresh the PATH from registry
+        import winreg
+
+        try:
+            with winreg.OpenKey(
+                winreg.HKEY_LOCAL_MACHINE,
+                r"SYSTEM\CurrentControlSet\Control\Session Manager\Environment",
+            ) as key:
+                path_value, _ = winreg.QueryValueEx(key, "PATH")
+                os.environ["PATH"] = path_value + ";" + os.environ.get("PATH", "")
+        except Exception as e:  # pylint: disable=broad-except
+            logging.debug("Could not refresh PATH from registry: %s", e)
+
+        # Also try to add common installation paths
+        common_paths = [
+            r"C:\Program Files\FLM",
+            r"C:\Program Files (x86)\FLM",
+            os.path.expanduser(r"~\AppData\Local\FLM"),
+        ]
+        for path in common_paths:
+            if os.path.exists(path) and path not in os.environ.get("PATH", ""):
+                os.environ["PATH"] = path + ";" + os.environ.get("PATH", "")
+
+
+def install_flm():
+    """
+    Check if FLM is installed and at minimum version.
+    If not, download and run the GUI installer, then wait for completion.
+    """
+    # Check current FLM installation
+    current_version = check_flm_version()
+
+    if current_version and Version(current_version) >= Version(FLM_MINIMUM_VERSION):
+        logging.info(
+            "FLM v%s is already installed and meets minimum version requirement (v%s)",
+            current_version,
+            FLM_MINIMUM_VERSION,
+        )
+        return
+
+    if current_version:
+        logging.info(
+            "FLM v%s is installed but below minimum version v%s. Upgrading...",
+            current_version,
+            FLM_MINIMUM_VERSION,
+        )
+    else:
+        logging.info(
+            "FLM not found. Installing FLM v%s or later...", FLM_MINIMUM_VERSION
+        )
+
+    # Download the installer
+    # pylint: disable=line-too-long
+    installer_url = "https://github.com/FastFlowLM/FastFlowLM/releases/latest/download/flm-setup.exe"
+    installer_path = os.path.join(tempfile.gettempdir(), "flm-setup.exe")
+
+    try:
+        # Remove existing installer if present
+        if os.path.exists(installer_path):
+            os.remove(installer_path)
+
+        logging.info("Downloading FLM installer...")
+        response = requests.get(installer_url, stream=True, timeout=30)
+        response.raise_for_status()
+
+        # Save installer to disk
+        with open(installer_path, "wb") as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                f.write(chunk)
+            f.flush()
+            os.fsync(f.fileno())
+
+        logging.info("Downloaded FLM installer to %s", installer_path)
+
+        # Launch the installer GUI
+        logging.warning(
+            "Launching FLM installer GUI. Please complete the installation..."
+        )
+
+        # Launch installer and wait for it to complete
+        if os.name == "nt":  # Windows
+            process = subprocess.Popen([installer_path], shell=True)
+        else:
+            process = subprocess.Popen([installer_path])
+
+        # Wait for installer to complete
+        process.wait()
+
+        if process.returncode != 0:
+            raise RuntimeError(
+                f"FLM installer failed with exit code {process.returncode}"
+            )
+
+        logging.info("FLM installer completed successfully")
+
+        # Refresh environment to pick up new PATH entries
+        refresh_environment()
+
+        # Wait a moment for system to update
+        time.sleep(2)
+
+        # Verify installation
+        max_retries = 10
+        for attempt in range(max_retries):
+            new_version = check_flm_version()
+            if new_version and Version(new_version) >= Version(FLM_MINIMUM_VERSION):
+                logging.info("FLM v%s successfully installed and verified", new_version)
+                return
+
+            if attempt < max_retries - 1:
+                logging.debug(
+                    "FLM not yet available in PATH, retrying... (attempt %d/%d)",
+                    attempt + 1,
+                    max_retries,
+                )
+                time.sleep(3)
+                refresh_environment()
+
+        # Final check failed
+        raise RuntimeError(
+            "FLM installation completed but 'flm' command is not available in PATH. "
+            "Please ensure FLM is properly installed and available in your system PATH."
+        )
+
+    except requests.RequestException as e:
+        raise RuntimeError(f"Failed to download FLM installer: {e}") from e
+    except Exception as e:
+        raise RuntimeError(f"FLM installation failed: {e}") from e
+    finally:
+        # Clean up installer file
+        if os.path.exists(installer_path):
+            try:
+                os.remove(installer_path)
+            except OSError:
+                pass  # Ignore cleanup errors
+
+
+def download_flm_model(config_checkpoint, _=None, do_not_upgrade=False) -> dict:
+    """
+    Downloads the FLM model for the given configuration.
+
+    Args:
+        config_checkpoint: name of the FLM model to install.
+        _: placeholder for `config_mmproj`, which is standard
+            for WrappedServer (see llamacpp/utils.py) .
+        do_not_upgrade: whether to re-download the model if it is already
+            available.
+    """
+
+    if do_not_upgrade:
+        command = ["flm", "pull", f"{config_checkpoint}"]
+    else:
+        command = ["flm", "pull", f"{config_checkpoint}", "--force"]
+
+    subprocess.run(command, check=True)
+
+
+def get_flm_installed_models() -> List[str]:
+    """
+    Parse FLM model list and return installed model checkpoints.
+
+    Returns:
+        List of installed FLM model checkpoints (e.g., ["llama3.2:1b", "gemma3:4b"])
+    """
+    try:
+        result = subprocess.run(
+            ["flm", "list"],
+            capture_output=True,
+            text=True,
+            check=True,
+            encoding="utf-8",
+            errors="replace",
+        )
+
+        # Check if we got valid output
+        if not result.stdout:
+            return []
+
+        installed_checkpoints = []
+
+        lines = result.stdout.strip().split("\n")
+        for line in lines:
+            line = line.strip()
+            if line.startswith("- "):
+                # Remove the leading "- " and parse the model info
+                model_info = line[2:].strip()
+
+                # Check if model is installed (✅)
+                if model_info.endswith(" ✅"):
+                    checkpoint = model_info[:-2].strip()
+                    installed_checkpoints.append(checkpoint)
+
+        return installed_checkpoints
+
+    except (subprocess.CalledProcessError, FileNotFoundError, AttributeError):
+        # FLM not installed, not available, or output parsing failed
+        return []
+
+
+def is_flm_available() -> bool:
+    """
+    Check if FLM is available and meets minimum version requirements.
+    """
+    current_version = check_flm_version()
+    return current_version is not None and Version(current_version) >= Version(
+        FLM_MINIMUM_VERSION
+    )

lemonade/tools/llamacpp/utils.py

@@ -14,8 +14,9 @@ from lemonade.common.system_info import get_system_info
 
 from dotenv import set_key, load_dotenv
 
-LLAMA_VERSION_VULKAN = "b6431"
-LLAMA_VERSION_ROCM = "b1057"
+LLAMA_VERSION_VULKAN = "b6510"
+LLAMA_VERSION_ROCM = "b1066"
+LLAMA_VERSION_METAL = "b6510"
 
 
 def identify_rocm_arch_from_name(device_name: str) -> str | None:
@@ -126,8 +127,12 @@ def get_llama_version(backend: str) -> str:
         return LLAMA_VERSION_ROCM
     elif backend == "vulkan":
         return LLAMA_VERSION_VULKAN
+    elif backend == "metal":
+        return LLAMA_VERSION_METAL
     else:
-        raise ValueError(f"Unsupported backend: {backend}")
+        raise ValueError(
+            f"Unsupported backend: {backend}. Supported: vulkan, rocm, metal"
+        )
 
 
 def get_llama_folder_path(backend: str):
@@ -142,10 +147,12 @@ def get_llama_exe_path(exe_name: str, backend: str):
     Get path to platform-specific llama-server executable
     """
     base_dir = get_llama_folder_path(backend)
-    if platform.system().lower() == "windows":
+    system = platform.system().lower()
+
+    if system == "windows":
         return os.path.join(base_dir, f"{exe_name}.exe")
-    else:  # Linux/Ubuntu
-        # Check if executable exists in build/bin subdirectory (Current Ubuntu structure)
+    else:  # Darwin/Linux/Ubuntu
+        # Check if executable exists in build/bin subdirectory
         build_bin_path = os.path.join(base_dir, "build", "bin", exe_name)
         if os.path.exists(build_bin_path):
             return build_bin_path
@@ -223,8 +230,24 @@ def get_binary_url_and_filename(backend: str, target_arch: str = None):
             raise NotImplementedError(
                 f"Platform {system} not supported for Vulkan llamacpp. Supported: Windows, Ubuntu Linux"
             )
+
+    elif backend == "metal":
+        # Metal support for macOS Apple Silicon from ggml-org/llama.cpp
+        repo = "ggml-org/llama.cpp"
+        version = LLAMA_VERSION_METAL
+        if system == "darwin":
+            if platform.machine().lower() in ["arm64", "aarch64"]:
+                filename = f"llama-{version}-bin-macos-arm64.zip"
+            else:
+                raise NotImplementedError(
+                    "Metal backend only supports Apple Silicon (ARM64) processors"
+                )
+        else:
+            raise NotImplementedError(
+                f"Platform {system} not supported for Metal llamacpp. Metal is only supported on macOS"
+            )
     else:
-        supported_backends = ["vulkan", "rocm"]
+        supported_backends = ["vulkan", "rocm", "metal"]
         raise NotImplementedError(
             f"Unsupported backend: {backend}. Supported backends: {supported_backends}"
         )
@@ -239,10 +262,10 @@ def validate_platform_support():
     """
     system = platform.system().lower()
 
-    if system not in ["windows", "linux"]:
+    if system not in ["windows", "linux", "darwin"]:
         raise NotImplementedError(
             f"Platform {system} not supported for llamacpp. "
-            "Supported: Windows, Ubuntu Linux"
+            "Supported: Windows, Ubuntu Linux, macOS"
         )
 
     if system == "linux":
@@ -341,6 +364,29 @@ def install_llamacpp(backend):
         if filename.endswith(".zip"):
             with zipfile.ZipFile(llama_archive_path, "r") as zip_ref:
                 zip_ref.extractall(llama_server_exe_dir)
+
+            # On Unix-like systems (macOS/Linux), make executables executable
+            if platform.system().lower() in ["darwin", "linux"]:
+                import stat
+
+                # Find and make executable files executable
+                for root, dirs, files in os.walk(llama_server_exe_dir):
+                    for file in files:
+                        file_path = os.path.join(root, file)
+                        # Make files in bin/ directories executable
+                        if "bin" in root.split(os.sep) or file in [
+                            "llama-server",
+                            "llama-simple",
+                        ]:
+                            try:
+                                current_permissions = os.stat(file_path).st_mode
+                                os.chmod(file_path, current_permissions | stat.S_IEXEC)
+                                logging.debug(f"Made {file_path} executable")
+                            except Exception as e:
+                                raise RuntimeError(
+                                    f"Failed to make {file_path} executable. This will prevent "
+                                    f"llama-server from starting. Error: {e}"
+                                )
         else:
             raise NotImplementedError(f"Unsupported archive format: {filename}")
 
@@ -510,14 +556,14 @@ def identify_gguf_models(
     The CHECKPOINT:VARIANT scheme is used to specify model files in Hugging Face repositories.
 
     The VARIANT format can be one of several types:
-    0. wildcard (*): download all files in the repo
+    0. wildcard (*): download all .gguf files in the repo
     1. Full filename: exact file to download
     2. None/empty: gets the first .gguf file in the repository (excludes mmproj files)
     3. Quantization variant: find a single file ending with the variant name (case insensitive)
     4. Folder name: downloads all .gguf files in the folder that matches the variant name (case insensitive)
 
     Examples:
-    - "ggml-org/gpt-oss-120b-GGUF:*" -> downloads all files in repo
+    - "ggml-org/gpt-oss-120b-GGUF:*" -> downloads all .gguf files in repo
     - "unsloth/Qwen3-8B-GGUF:qwen3.gguf" -> downloads "qwen3.gguf"
     - "unsloth/Qwen3-30B-A3B-GGUF" -> downloads "Qwen3-30B-A3B-GGUF.gguf"
     - "unsloth/Qwen3-8B-GGUF:Q4_1" -> downloads "Qwen3-8B-GGUF-Q4_1.gguf"
@@ -531,13 +577,14 @@ def identify_gguf_models(
 
     # (case 0) Wildcard, download everything
     if variant and variant == "*":
-        sharded_files = repo_files
+        sharded_files = [f for f in repo_files if f.endswith(".gguf")]
 
         # Sort to ensure consistent ordering
         sharded_files.sort()
 
         # Use first file as primary (this is how llamacpp handles it)
         variant_name = sharded_files[0]
+
     # (case 1) If variant ends in .gguf, use it directly
     elif variant and variant.endswith(".gguf"):
         variant_name = variant
@@ -856,7 +903,9 @@ def get_hip_devices():
     try:
         libhip = ctypes.CDLL(matching_files[0])
     except OSError:
-        raise RuntimeError(f"Could not load HIP runtime library from {path}")
+        raise RuntimeError(
+            f"Could not load HIP runtime library from {matching_files[0]}"
+        )
 
     # Setup function signatures
     hipError_t = c_int

lemonade/tools/server/flm.py

@@ -0,0 +1,137 @@
+import os
+import logging
+import subprocess
+import time
+import threading
+
+import requests
+
+from lemonade_server.pydantic_models import (
+    PullConfig,
+    ChatCompletionRequest,
+)
+
+from lemonade.tools.server.wrapped_server import WrappedServerTelemetry, WrappedServer
+from lemonade.tools.flm.utils import install_flm, download_flm_model
+
+
+class FlmTelemetry(WrappedServerTelemetry):
+    """
+    Manages telemetry data collection and display for FLM server.
+    """
+
+    def parse_telemetry_line(self, line: str):
+        """
+        Parse telemetry data from FLM server output lines.
+
+        Note: as of FLM 0.9.10, no telemetry data is provided by the server CLI.
+                This function is required to be implemented, so we leave it empty
+                as a placeholder for now.
+        """
+
+        return
+
+
+class FlmServer(WrappedServer):
+    """
+    Routes OpenAI API requests to an FLM server instance and returns the result
+    back to Lemonade Server.
+    """
+
+    def __init__(self):
+        self.flm_model_name = None
+        super().__init__(server_name="flm-server", telemetry=FlmTelemetry())
+
+    def _choose_port(self):
+        """
+        `flm serve` doesn't support port selection as of v0.9.10
+        """
+        self.port = 11434
+
+    def address(self):
+        return f"http://localhost:{self.port}/v1"
+
+    def install_server(self):
+        """
+        Check if FLM is installed and at minimum version.
+        If not, download and run the GUI installer, then wait for completion.
+        """
+        install_flm()
+
+    def download_model(
+        self, config_checkpoint, config_mmproj=None, do_not_upgrade=False
+    ) -> dict:
+        download_flm_model(config_checkpoint, config_mmproj, do_not_upgrade)
+
+    def _launch_server_subprocess(
+        self,
+        model_config: PullConfig,
+        snapshot_files: dict,
+        ctx_size: int,
+        supports_embeddings: bool = False,
+        supports_reranking: bool = False,
+    ):
+
+        self._choose_port()
+
+        # Keep track of the FLM model name so that we can use it later
+        self.flm_model_name = model_config.checkpoint
+
+        command = [
+            "flm",
+            "serve",
+            f"{self.flm_model_name}",
+            "--ctx-len",
+            str(ctx_size),
+        ]
+
+        # Set up environment with library path for Linux
+        env = os.environ.copy()
+
+        self.process = subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            bufsize=1,
+            env=env,
+        )
+
+        # Start background thread to log subprocess output
+        threading.Thread(
+            target=self._log_subprocess_output,
+            args=("FLM SERVER",),
+            daemon=True,
+        ).start()
+
+    def _wait_for_load(self):
+        """
+        FLM doesn't seem to have a health API, so we'll use the "list local models"
+        API to check if the server is up.
+        """
+        status_code = None
+        while not self.process.poll() and status_code != 200:
+            health_url = f"http://localhost:{self.port}/api/tags"
+            try:
+                health_response = requests.get(health_url)
+            except requests.exceptions.ConnectionError:
+                logging.debug(
+                    "Not able to connect to %s yet, will retry", self.server_name
+                )
+            else:
+                status_code = health_response.status_code
+                logging.debug(
+                    "Testing %s readiness (will retry until ready), result: %s",
+                    self.server_name,
+                    health_response.json(),
+                )
+            time.sleep(1)
+
+    def chat_completion(self, chat_completion_request: ChatCompletionRequest):
+        # FLM requires the correct model name to be in the request
+        # (whereas llama-server ignores the model name field in the request)
+        chat_completion_request.model = self.flm_model_name
+
+        return super().chat_completion(chat_completion_request)

lemonade/tools/server/llamacpp.py

@@ -88,9 +88,8 @@ class LlamaTelemetry(WrappedServerTelemetry):
 
 class LlamaServer(WrappedServer):
     def __init__(self, backend: str):
-        self.telemetry = LlamaTelemetry()
         self.backend = backend
-        super().__init__(server_name="llama-server", telemetry=self.telemetry)
+        super().__init__(server_name="llama-server", telemetry=LlamaTelemetry())
 
     def install_server(self, backend=None):
         """
@@ -157,13 +156,23 @@ class LlamaServer(WrappedServer):
 
         # Find a port, and save it in the telemetry object for future reference
         # by other functions
-        self.choose_port()
+        self._choose_port()
 
         # Add port and jinja to enable tool use
         base_command.extend(["--port", str(self.port), "--jinja"])
 
         # Enable context shift and avoid attention sink issues by preserving the initial tokens
-        base_command.extend(["--context-shift", "--keep", "16"])
+        # Note: --context-shift is not supported on all backends (e.g., Metal on macOS)
+        # Only add context-shift for backends that support it
+        context_shift_supported_backends = ["vulkan", "rocm"]
+        if self.backend in context_shift_supported_backends:
+            base_command.extend(["--context-shift", "--keep", "16"])
+        else:
+            # For backends that don't support context-shift (e.g., Metal), just use keep
+            base_command.extend(["--keep", "16"])
+            logging.debug(
+                f"Skipped --context-shift for backend: {self.backend} (not supported)"
+            )
 
         # Use legacy reasoning formatting, since not all apps support the new
         # reasoning_content field
@@ -192,7 +201,8 @@ class LlamaServer(WrappedServer):
             env.update(os.environ)
             logging.debug(f"Loaded environment variables from {env_file_path}")
 
-        if platform.system().lower() == "linux":
+        system = platform.system().lower()
+        if system == "linux":
             lib_dir = os.path.dirname(exe_path)  # Same directory as the executable
             current_ld_path = env.get("LD_LIBRARY_PATH", "")
             if current_ld_path:
@@ -200,6 +210,14 @@ class LlamaServer(WrappedServer):
             else:
                 env["LD_LIBRARY_PATH"] = lib_dir
             logging.debug(f"Set LD_LIBRARY_PATH to {env['LD_LIBRARY_PATH']}")
+        elif system == "darwin":
+            lib_dir = os.path.dirname(exe_path)
+            current_dyld_path = env.get("DYLD_LIBRARY_PATH", "")
+            if current_dyld_path:
+                env["DYLD_LIBRARY_PATH"] = f"{lib_dir}:{current_dyld_path}"
+            else:
+                env["DYLD_LIBRARY_PATH"] = lib_dir
+            logging.debug(f"Set DYLD_LIBRARY_PATH to {env['DYLD_LIBRARY_PATH']}")
 
         # Start subprocess with output capture
         self.process = subprocess.Popen(