lemonade-sdk 8.0.6__py3-none-any.whl → 8.1.1__py3-none-any.whl

lemonade/tools/server/llamacpp.py

@@ -1,5 +1,4 @@
 import os
-import sys
 import logging
 import time
 import subprocess
@@ -9,6 +8,7 @@ import platform
 
 import requests
 from tabulate import tabulate
+from dotenv import load_dotenv
 from fastapi import HTTPException, status
 from fastapi.responses import StreamingResponse
 
@@ -29,8 +29,6 @@ from lemonade.tools.llamacpp.utils import (
     download_gguf,
 )
 
-LLAMA_VERSION = "b5787"
-
 
 def llamacpp_address(port: int) -> str:
     """
@@ -45,45 +43,6 @@ def llamacpp_address(port: int) -> str:
     return f"http://127.0.0.1:{port}/v1"
 
 
-def get_llama_server_paths():
-    """
-    Get platform-specific paths for llama server directory and executable
-    """
-    base_dir = os.path.join(os.path.dirname(sys.executable), "llama_server")
-
-    if platform.system().lower() == "windows":
-        return base_dir, os.path.join(base_dir, "llama-server.exe")
-    else:  # Linux/Ubuntu
-        # Check if executable exists in build/bin subdirectory (Current Ubuntu structure)
-        build_bin_path = os.path.join(base_dir, "build", "bin", "llama-server")
-        if os.path.exists(build_bin_path):
-            return base_dir, build_bin_path
-        else:
-            # Fallback to root directory
-            return base_dir, os.path.join(base_dir, "llama-server")
-
-
-def get_binary_url_and_filename(version):
-    """
-    Get the appropriate binary URL and filename based on platform
-    """
-    system = platform.system().lower()
-
-    if system == "windows":
-        filename = f"llama-{version}-bin-win-vulkan-x64.zip"
-    elif system == "linux":
-        filename = f"llama-{version}-bin-ubuntu-vulkan-x64.zip"
-    else:
-        raise NotImplementedError(
-            f"Platform {system} not supported for llamacpp. Supported: Windows, Ubuntu Linux"
-        )
-
-    url = (
-        f"https://github.com/ggml-org/llama.cpp/releases/download/{version}/{filename}"
-    )
-    return url, filename
-
-
 class LlamaTelemetry:
     """
     Manages telemetry data collection and display for llama server.
@@ -125,7 +84,7 @@ class LlamaTelemetry:
             device_count = int(vulkan_match.group(1))
             if device_count > 0:
                 logging.info(
-                    f"GPU acceleration active: {device_count} Vulkan device(s) "
+                    f"GPU acceleration active: {device_count} device(s) "
                     "detected by llama-server"
                 )
             return
@@ -236,6 +195,8 @@ def _launch_llama_subprocess(
     snapshot_files: dict,
     use_gpu: bool,
     telemetry: LlamaTelemetry,
+    backend: str,
+    ctx_size: int,
     supports_embeddings: bool = False,
     supports_reranking: bool = False,
 ) -> subprocess.Popen:
@@ -246,6 +207,7 @@ def _launch_llama_subprocess(
         snapshot_files: Dictionary of model files to load
         use_gpu: Whether to use GPU acceleration
         telemetry: Telemetry object for tracking performance metrics
+        backend: Backend to use (e.g., 'vulkan', 'rocm')
         supports_embeddings: Whether the model supports embeddings
         supports_reranking: Whether the model supports reranking
 
@@ -254,10 +216,16 @@ def _launch_llama_subprocess(
     """
 
     # Get the current executable path (handles both Windows and Ubuntu structures)
-    exe_path = get_llama_server_exe_path()
+    exe_path = get_llama_server_exe_path(backend)
 
     # Build the base command
-    base_command = [exe_path, "-m", snapshot_files["variant"]]
+    base_command = [
+        exe_path,
+        "-m",
+        snapshot_files["variant"],
+        "--ctx-size",
+        str(ctx_size),
+    ]
     if "mmproj" in snapshot_files:
         base_command.extend(["--mmproj", snapshot_files["mmproj"]])
         if not use_gpu:
@@ -288,6 +256,15 @@ def _launch_llama_subprocess(
 
     # Set up environment with library path for Linux
     env = os.environ.copy()
+
+    # Load environment variables from .env file in the executable directory
+    exe_dir = os.path.dirname(exe_path)
+    env_file_path = os.path.join(exe_dir, ".env")
+    if os.path.exists(env_file_path):
+        load_dotenv(env_file_path, override=True)
+        env.update(os.environ)
+        logging.debug(f"Loaded environment variables from {env_file_path}")
+
     if platform.system().lower() == "linux":
         lib_dir = os.path.dirname(exe_path)  # Same directory as the executable
         current_ld_path = env.get("LD_LIBRARY_PATH", "")
@@ -320,18 +297,17 @@ def _launch_llama_subprocess(
     return process
 
 
-def server_load(model_config: PullConfig, telemetry: LlamaTelemetry):
+def server_load(
+    model_config: PullConfig, telemetry: LlamaTelemetry, backend: str, ctx_size: int
+):
     # Install and/or update llama.cpp if needed
     try:
-        install_llamacpp()
+        install_llamacpp(backend)
     except NotImplementedError as e:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)
         )
 
-    # Get platform-specific paths at runtime
-    llama_server_exe_path = get_llama_server_exe_path()
-
     # Download the gguf to the hugging face cache
     snapshot_files = download_gguf(model_config.checkpoint, model_config.mmproj)
     logging.debug(f"GGUF file paths: {snapshot_files}")
@@ -342,14 +318,13 @@ def server_load(model_config: PullConfig, telemetry: LlamaTelemetry):
     supports_embeddings = "embeddings" in model_info.get("labels", [])
     supports_reranking = "reranking" in model_info.get("labels", [])
 
-    # Start the llama-serve.exe process
-    logging.debug(f"Using llama_server for GGUF model: {llama_server_exe_path}")
-
     # Attempt loading on GPU first
     llama_server_process = _launch_llama_subprocess(
         snapshot_files,
         use_gpu=True,
         telemetry=telemetry,
+        backend=backend,
+        ctx_size=ctx_size,
         supports_embeddings=supports_embeddings,
         supports_reranking=supports_reranking,
     )
@@ -374,6 +349,8 @@ def server_load(model_config: PullConfig, telemetry: LlamaTelemetry):
             snapshot_files,
             use_gpu=False,
             telemetry=telemetry,
+            backend=backend,
+            ctx_size=ctx_size,
             supports_embeddings=supports_embeddings,
             supports_reranking=supports_reranking,
         )

lemonade/tools/server/serve.py

@@ -1,5 +1,4 @@
 import sys
-import argparse
 import asyncio
 import statistics
 import time
@@ -48,6 +47,11 @@ from openai.types.responses import (
 )
 
 import lemonade.api as lemonade_api
+import lemonade.tools.server.llamacpp as llamacpp
+from lemonade.tools.server.tool_calls import extract_tool_calls, get_tool_call_pattern
+from lemonade.tools.server.webapp import get_webapp_html
+from lemonade.tools.server.utils.port import lifespan
+
 from lemonade_server.model_manager import ModelManager
 from lemonade_server.pydantic_models import (
     DEFAULT_MAX_NEW_TOKENS,
@@ -60,18 +64,17 @@ from lemonade_server.pydantic_models import (
     PullConfig,
     DeleteConfig,
 )
-from lemonade.tools.management_tools import ManagementTool
-import lemonade.tools.server.llamacpp as llamacpp
-from lemonade.tools.server.tool_calls import extract_tool_calls, get_tool_call_pattern
-from lemonade.tools.server.webapp import get_webapp_html
-from lemonade.tools.server.utils.port import lifespan
 
 # Only import tray on Windows
 if platform.system() == "Windows":
+    # pylint: disable=ungrouped-imports
     from lemonade.tools.server.tray import LemonadeTray, OutputDuplicator
 
+
 DEFAULT_PORT = 8000
 DEFAULT_LOG_LEVEL = "info"
+DEFAULT_LLAMACPP_BACKEND = "vulkan"
+DEFAULT_CTX_SIZE = 4096
 
 
 class ServerModel(Model):
@@ -126,7 +129,7 @@ class StopOnEvent:
         return self.stop_event.is_set()
 
 
-class Server(ManagementTool):
+class Server:
     """
     Open a web server that apps can use to communicate with the LLM.
 
@@ -144,11 +147,25 @@ class Server(ManagementTool):
     - /api/v1/models: list all available models.
     """
 
-    unique_name = "serve"
-
-    def __init__(self):
+    def __init__(
+        self,
+        port: int = DEFAULT_PORT,
+        log_level: str = DEFAULT_LOG_LEVEL,
+        ctx_size: int = DEFAULT_CTX_SIZE,
+        tray: bool = False,
+        log_file: str = None,
+        llamacpp_backend: str = DEFAULT_LLAMACPP_BACKEND,
+    ):
         super().__init__()
 
+        # Save args as members
+        self.port = port
+        self.log_level = log_level
+        self.ctx_size = ctx_size
+        self.tray = tray
+        self.log_file = log_file
+        self.llamacpp_backend = llamacpp_backend
+
         # Initialize FastAPI app
         self.app = FastAPI(lifespan=lifespan)
 
@@ -186,9 +203,6 @@ class Server(ManagementTool):
         self.output_tokens = None
         self.decode_token_times = None
 
-        # Input truncation settings
-        self.truncate_inputs = False
-
         # Store debug logging state
         self.debug_logging_enabled = logging.getLogger().isEnabledFor(logging.DEBUG)
 
@@ -241,66 +255,18 @@ class Server(ManagementTool):
             self.app.post(f"{prefix}/reranking")(self.reranking)
             self.app.post(f"{prefix}/rerank")(self.reranking)
 
-    @staticmethod
-    def parser(add_help: bool = True) -> argparse.ArgumentParser:
-        parser = __class__.helpful_parser(
-            short_description="Launch an industry-standard LLM server",
-            add_help=add_help,
-        )
-
-        # Only add the tray option on Windows
-        if platform.system() == "Windows":
-            parser.add_argument(
-                "--tray",
-                action="store_true",
-                help="Run the server in system tray mode",
-            )
-
-        parser.add_argument(
-            "--port",
-            required=False,
-            type=int,
-            default=DEFAULT_PORT,
-            help=f"Port number to run the server on (default: {DEFAULT_PORT})",
-        )
-        parser.add_argument(
-            "--log-level",
-            required=False,
-            type=str,
-            default=DEFAULT_LOG_LEVEL,
-            choices=["critical", "error", "warning", "info", "debug", "trace"],
-            help=f"Logging level (default: {DEFAULT_LOG_LEVEL})",
-        )
-
-        parser.add_argument(
-            "--log-file",
-            required=False,
-            type=str,
-            help="Path to the log file",
-        )
-
-        return parser
-
     def _setup_server_common(
         self,
-        port: int,
-        truncate_inputs: bool = False,
-        log_level: str = DEFAULT_LOG_LEVEL,
         tray: bool = False,
-        log_file: str = None,
         threaded_mode: bool = False,
     ):
         """
         Common setup logic shared between run() and run_in_thread().
 
         Args:
-            port: Port number for the server
-            truncate_inputs: Whether to truncate inputs if they exceed max length
-            log_level: Logging level to configure
+            tray: Whether to run the server in tray mode
             threaded_mode: Whether this is being set up for threaded execution
         """
-        # Store truncation settings
-        self.truncate_inputs = truncate_inputs
 
         # Define TRACE level
         logging.TRACE = 9  # Lower than DEBUG which is 10
@@ -318,18 +284,20 @@ class Server(ManagementTool):
             logging.getLogger("uvicorn.error").setLevel(logging.WARNING)
         else:
             # Configure logging to match uvicorn's format
-            logging_level = getattr(logging, log_level.upper())
+            logging_level = getattr(logging, self.log_level.upper())
 
             # Set up file handler for logging to lemonade.log
             uvicorn_formatter = uvicorn.logging.DefaultFormatter(
                 fmt="%(levelprefix)s %(message)s",
                 use_colors=True,
             )
-            if not log_file:
-                log_file = tempfile.NamedTemporaryFile(
+            if not self.log_file:
+                self.log_file = tempfile.NamedTemporaryFile(
                     prefix="lemonade_", suffix=".log", delete=False
                 ).name
-            file_handler = logging.FileHandler(log_file, mode="a", encoding="utf-8")
+            file_handler = logging.FileHandler(
+                self.log_file, mode="a", encoding="utf-8"
+            )
             file_handler.setLevel(logging_level)
             file_handler.setFormatter(uvicorn_formatter)
 
@@ -349,12 +317,12 @@ class Server(ManagementTool):
         self.debug_logging_enabled = logging.getLogger().isEnabledFor(logging.DEBUG)
         if tray:
             # Save original stdout/stderr
-            sys.stdout = OutputDuplicator(log_file, sys.stdout)
-            sys.stderr = OutputDuplicator(log_file, sys.stderr)
+            sys.stdout = OutputDuplicator(self.log_file, sys.stdout)
+            sys.stderr = OutputDuplicator(self.log_file, sys.stderr)
 
             # Open lemonade server in tray mode
             # lambda function used for deferred instantiation and thread safety
-            LemonadeTray(log_file, port, lambda: Server()).run()
+            LemonadeTray(self.log_file, self.port, lambda: self).run()
             sys.exit(0)
 
         if self.debug_logging_enabled:
@@ -363,47 +331,26 @@ class Server(ManagementTool):
 
         # Let the app know what port it's running on, so
         # that the lifespan can access it
-        self.app.port = port
+        self.app.port = self.port
 
-    def run(
-        self,
-        # ManagementTool has a required cache_dir arg, but
-        # we always use the default cache directory
-        _=None,
-        port: int = DEFAULT_PORT,
-        log_level: str = DEFAULT_LOG_LEVEL,
-        truncate_inputs: bool = False,
-        tray: bool = False,
-        log_file: str = None,
-    ):
+    def run(self):
         # Common setup
         self._setup_server_common(
-            port=port,
-            truncate_inputs=truncate_inputs,
-            log_level=log_level,
             threaded_mode=False,
-            tray=tray,
-            log_file=log_file,
+            tray=self.tray,
         )
 
-        uvicorn.run(self.app, host="localhost", port=port, log_level=log_level)
+        uvicorn.run(
+            self.app, host="localhost", port=self.port, log_level=self.log_level
+        )
 
-    def run_in_thread(
-        self,
-        port: int = DEFAULT_PORT,
-        host: str = "localhost",
-        log_level: str = "warning",
-        truncate_inputs: bool = False,
-    ):
+    def run_in_thread(self, host: str = "localhost"):
         """
         Set up the server for running in a thread.
         Returns a uvicorn server instance that can be controlled externally.
         """
         # Common setup
         self._setup_server_common(
-            port=port,
-            truncate_inputs=truncate_inputs,
-            log_level=log_level,
             threaded_mode=True,
             tray=False,
         )
@@ -418,8 +365,8 @@ class Server(ManagementTool):
         config = Config(
             app=self.app,
             host=host,
-            port=port,
-            log_level=log_level,
+            port=self.port,
+            log_level=self.log_level,
             log_config=None,
         )
 
@@ -1099,29 +1046,21 @@ class Server(ManagementTool):
             )
             self.input_tokens = len(input_ids[0])
 
-        if (
-            self.llm_loaded.max_prompt_length
-            and self.input_tokens > self.llm_loaded.max_prompt_length
-        ):
-            if self.truncate_inputs:
-                # Truncate input ids
-                truncate_amount = self.input_tokens - self.llm_loaded.max_prompt_length
-                input_ids = input_ids[: self.llm_loaded.max_prompt_length]
-
-                # Update token count
-                self.input_tokens = len(input_ids)
-
-                # Show warning message
-                truncation_message = (
-                    f"Input exceeded {self.llm_loaded.max_prompt_length} tokens. "
-                    f"Truncated {truncate_amount} tokens."
-                )
-                logging.warning(truncation_message)
-            else:
-                raise RuntimeError(
-                    f"Prompt tokens ({self.input_tokens}) cannot be greater "
-                    f"than the model's max prompt length ({self.llm_loaded.max_prompt_length})"
-                )
+        # For non-llamacpp recipes, truncate inputs to ctx_size if needed
+        if self.llm_loaded.recipe != "llamacpp" and self.input_tokens > self.ctx_size:
+            # Truncate input ids
+            truncate_amount = self.input_tokens - self.ctx_size
+            input_ids = input_ids[: self.ctx_size]
+
+            # Update token count
+            self.input_tokens = len(input_ids)
+
+            # Show warning message
+            truncation_message = (
+                f"Input exceeded {self.ctx_size} tokens. "
+                f"Truncated {truncate_amount} tokens from the beginning."
+            )
+            logging.warning(truncation_message)
 
         # Log the input tokens early to avoid this not showing due to potential crashes
         logging.debug(f"Input Tokens: {self.input_tokens}")
@@ -1317,7 +1256,7 @@ class Server(ManagementTool):
         self.tokenizer = None
         self.model = None
 
-        default_message = f"model {model_reference} not found"
+        default_message = "see stack trace and error message below"
         if message:
             detail = message
         else:
@@ -1438,6 +1377,8 @@ class Server(ManagementTool):
                     self.llama_server_process = llamacpp.server_load(
                         model_config=config_to_use,
                         telemetry=self.llama_telemetry,
+                        backend=self.llamacpp_backend,
+                        ctx_size=self.ctx_size,
                     )
 
                 else: