lemonade-sdk 8.0.4__py3-none-any.whl → 8.0.6__py3-none-any.whl

lemonade/tools/management_tools.py

@@ -1,12 +1,17 @@
 import argparse
 import abc
+import json
 from typing import List
 import lemonade.common.filesystem as fs
 import lemonade.common.exceptions as exp
 import lemonade.common.printing as printing
 from lemonade.tools.tool import ToolParser
 from lemonade.version import __version__ as lemonade_version
-from lemonade.common.system_info import get_system_info_dict
+from lemonade.common.system_info import (
+    get_system_info_dict,
+    get_device_info_dict,
+    get_system_info,
+)
 from lemonade.common.build import output_dir
 import lemonade.cache as lemonade_cache
 
@@ -245,28 +250,69 @@ class SystemInfo(ManagementTool):
     @staticmethod
     def parser(add_help: bool = True) -> argparse.ArgumentParser:
         parser = __class__.helpful_parser(
-            short_description="Print system information",
+            short_description="Print system and device information",
             add_help=add_help,
         )
 
+        parser.add_argument(
+            "--format", choices=["table", "json"], default="table", help="Output format"
+        )
+
+        parser.add_argument(
+            "--verbose",
+            action="store_true",
+            help="Show detailed system information",
+        )
+
         return parser
 
     @staticmethod
     def pretty_print(my_dict: dict, level=0):
         for k, v in my_dict.items():
+            if k == "available" and v is True:
+                continue
+
             if isinstance(v, dict):
-                print("    " * level + f"{k}:")
-                SystemInfo.pretty_print(v, level + 1)
+                # Special handling for device availability
+                if v.get("available") is False:
+                    error_msg = v.get("error", "Not available")
+                    print("    " * level + f"{k}: {error_msg}")
+                else:
+                    print("    " * level + f"{k}:")
+                    SystemInfo.pretty_print(v, level + 1)
             elif isinstance(v, list):
                 print("    " * level + f"{k}:")
                 for item in v:
-                    print("    " * (level + 1) + f"{item}")
+                    if isinstance(item, dict):
+                        SystemInfo.pretty_print(item, level + 1)
+                        print()
+                    else:
+                        print("    " * (level + 1) + f"{item}")
             else:
                 print("    " * level + f"{k}: {v}")
 
-    def run(self, _):
+    def run(self, _, format="table", verbose=False):
+        # Get basic system info
         system_info_dict = get_system_info_dict()
-        self.pretty_print(system_info_dict)
+
+        # Always include devices
+        system_info_dict["Devices"] = get_device_info_dict()
+
+        # Filter out verbose-only information if not in verbose mode
+        if not verbose:
+            essential_keys = ["OS Version", "Processor", "Physical Memory", "Devices"]
+            system_info_dict = {
+                k: v for k, v in system_info_dict.items() if k in essential_keys
+            }
+        else:
+            # In verbose mode, add Python packages at the end
+            system_info = get_system_info()
+            system_info_dict["Python Packages"] = system_info.get_python_packages()
+
+        if format == "json":
+            print(json.dumps(system_info_dict, indent=2))
+        else:
+            self.pretty_print(system_info_dict)
 
 
 # This file was originally licensed under Apache 2.0. It has been modified.

lemonade/tools/oga/bench.py

@@ -74,12 +74,12 @@ class OgaBench(Bench):
 
         # Don't capture time for warmup
         for count in range(warmup_iterations):
-            outputs = model.generate(input_ids, max_new_tokens=output_tokens)
-            self.tokens_out_len_list.append(len(outputs[0]) - len(input_ids))
+            _ = model.generate(input_ids, max_new_tokens=output_tokens)
+            self.tokens_out_len_list.append(model.response_tokens)
             report_progress_fn((count + 1) / (warmup_iterations + iterations))
 
         for count in range(iterations):
-            outputs = model.generate(
+            _ = model.generate(
                 input_ids,
                 max_new_tokens=output_tokens,
                 min_new_tokens=output_tokens,
@@ -88,11 +88,10 @@ class OgaBench(Bench):
                 (warmup_iterations + count + 1) / (warmup_iterations + iterations)
             )
 
-            token_len = len(outputs[0]) - len(input_ids)
-            self.tokens_out_len_list.append(token_len)
+            self.tokens_out_len_list.append(model.response_tokens)
 
             # Only count an iteration if it produced enough tokens
-            if token_len >= output_tokens:
+            if model.response_tokens >= output_tokens:
                 per_iteration_time_to_first_token.append(model.time_to_first_token)
                 per_iteration_tokens_per_second.append(model.tokens_per_second)
 

lemonade/tools/oga/utils.py

@@ -99,13 +99,16 @@ class OrtGenaiModel(ModelAdapter):
     ):
         params = og.GeneratorParams(self.model)
 
+        # OGA models return a list of tokens (older versions) or 1d numpy array (newer versions)
         prompt_length = len(input_ids)
+
         max_prompt_length = self.config.get("max_prompt_length")
         if max_prompt_length and prompt_length > max_prompt_length:
             raise ValueError(
                 f"This prompt (length {prompt_length}) exceeds the model's "
                 f"maximum allowed prompt length ({max_prompt_length})."
             )
+        self.prompt_tokens = prompt_length
 
         # There is a breaking API change in OGA 0.6.0
         # Determine whether we should use the old or new APIs
@@ -206,18 +209,21 @@ class OrtGenaiModel(ModelAdapter):
                     )
                     self.tokens_per_second = 1 / avg_token_gen_latency_s
 
-            return [generator.get_sequence(0)]
+            response = generator.get_sequence(0)
+            self.response_tokens = len(response) - self.prompt_tokens
+            return [response]
         else:
             if use_oga_post_6_api:
                 generator.append_tokens(input_ids)
             tokenizer_stream = streamer.tokenizer.tokenizer.create_stream()
-
+            self.response_tokens = 0
             stop_early = False
 
             while not generator.is_done() and not stop_early:
                 if use_oga_pre_6_api:
                     generator.compute_logits()
                 generator.generate_next_token()
+                self.response_tokens += 1
 
                 new_token = generator.get_next_tokens()[0]
                 new_text = tokenizer_stream.decode(new_token)

lemonade/tools/prompt.py

@@ -161,7 +161,11 @@ class LLMPrompt(Tool):
         # If template flag is set, then wrap prompt in template
         if template:
             # Embed prompt in model's chat template
-            if tokenizer.chat_template:
+            if not hasattr(tokenizer, "prompt_template"):
+                printing.log_warning(
+                    "Templates for this model type are not yet implemented."
+                )
+            elif tokenizer.chat_template:
                 # Use the model's built-in chat template if available
                 messages_dict = [{"role": "user", "content": prompt}]
                 prompt = tokenizer.apply_chat_template(
@@ -175,25 +179,10 @@ class LLMPrompt(Tool):
                 state.save_stat(Keys.PROMPT_TEMPLATE, "Default")
 
         input_ids = tokenizer(prompt, return_tensors="pt").input_ids
-        if isinstance(input_ids, (list, str)):
-            # OGA models return a list of tokens (older versions)
-            # Our llama.cpp adapter returns a string
-            len_tokens_in = len(input_ids)
-        elif hasattr(input_ids, "shape"):
-            # HF models return a 2-D tensor
-            # OGA models with newer versions may return numpy arrays
-            if len(input_ids.shape) == 1:
-                # 1-D array from newer OGA versions
-                len_tokens_in = len(input_ids)
-            else:
-                # 2-D tensor from HF models
-                len_tokens_in = input_ids.shape[1]
-        else:
-            # Fallback: try to get length directly
-            len_tokens_in = len(input_ids)
 
         len_tokens_out = []
         response_texts = []
+        prompt_tokens = None  # will be determined in generate function
         for trial in range(n_trials):
             if n_trials > 1:
                 self.set_percent_progress(100.0 * trial / n_trials)
@@ -222,19 +211,22 @@ class LLMPrompt(Tool):
 
             response_array = response if isinstance(response, str) else response[0]
 
-            # Separate the prompt from the response
-            len_tokens_out.append(len(response_array) - len_tokens_in)
+            prompt_tokens = model.prompt_tokens
+            len_tokens_out.append(model.response_tokens)
 
-            input_token = 0
+            # Remove the input from the response
+            # (up to the point they diverge, which they should not)
+            counter = 0
+            len_input_ids = len(input_ids_array)
             while (
-                input_token < len_tokens_in
-                and input_ids_array[input_token] == response_array[input_token]
+                counter < len_input_ids
+                and input_ids_array[counter] == response_array[counter]
             ):
-                input_token += 1
+                counter += 1
 
             # Only decode the actual response (not the prompt)
             response_text = tokenizer.decode(
-                response_array[input_token:], skip_special_tokens=True
+                response_array[counter:], skip_special_tokens=True
             ).strip()
             response_texts.append(response_text)
 
@@ -259,7 +251,7 @@ class LLMPrompt(Tool):
             plt.savefig(figure_path)
             state.save_stat(Keys.RESPONSE_LENGTHS_HISTOGRAM, figure_path)
 
-        state.save_stat(Keys.PROMPT_TOKENS, len_tokens_in)
+        state.save_stat(Keys.PROMPT_TOKENS, prompt_tokens)
         state.save_stat(Keys.PROMPT, prompt)
         state.save_stat(Keys.RESPONSE_TOKENS, len_tokens_out)
         state.save_stat(Keys.RESPONSE, sanitize_text(response_texts))

lemonade/tools/report/table.py

@@ -758,15 +758,18 @@ class LemonadePerfTable(Table):
             data[key] = model_stats.get(key, "")
 
         # Create a new entry with Driver Versions and relevant Python Packages
-        sw_versions = [
-            key + ": " + value
-            for key, value in data[fs.Keys.SYSTEM_INFO]["Driver Versions"].items()
-        ]
-        sw_versions += [
-            pkg
-            for pkg in data[fs.Keys.SYSTEM_INFO]["Python Packages"]
-            if any(name in pkg for name in PYTHON_PACKAGES)
-        ]
+        sw_versions = []
+        if "Driver Versions" in data[fs.Keys.SYSTEM_INFO]:
+            sw_versions += [
+                key + ": " + value
+                for key, value in data[fs.Keys.SYSTEM_INFO]["Driver Versions"].items()
+            ]
+        if "Python Packages" in data[fs.Keys.SYSTEM_INFO]:
+            sw_versions += [
+                pkg
+                for pkg in data[fs.Keys.SYSTEM_INFO]["Python Packages"]
+                if any(name in pkg for name in PYTHON_PACKAGES)
+            ]
         if isinstance(data[Keys.RYZEN_AI_VERSION_INFO], dict):
             sw_versions += [
                 "Ryzen AI: " + value

lemonade/tools/server/llamacpp.py

@@ -1,13 +1,11 @@
-import sys
 import os
+import sys
 import logging
 import time
 import subprocess
-import zipfile
 import re
 import threading
 import platform
-import shutil
 
 import requests
 from tabulate import tabulate
@@ -18,12 +16,18 @@ from openai import OpenAI
 
 from lemonade_server.pydantic_models import (
     ChatCompletionRequest,
+    CompletionRequest,
     PullConfig,
     EmbeddingsRequest,
     RerankingRequest,
 )
 from lemonade_server.model_manager import ModelManager
 from lemonade.tools.server.utils.port import find_free_port
+from lemonade.tools.llamacpp.utils import (
+    get_llama_server_exe_path,
+    install_llamacpp,
+    download_gguf,
+)
 
 LLAMA_VERSION = "b5787"
 
@@ -80,39 +84,6 @@ def get_binary_url_and_filename(version):
     return url, filename
 
 
-def validate_platform_support():
-    """
-    Validate platform support before attempting download
-    """
-    system = platform.system().lower()
-
-    if system not in ["windows", "linux"]:
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=(
-                f"Platform {system} not supported for llamacpp. "
-                "Supported: Windows, Ubuntu Linux"
-            ),
-        )
-
-    if system == "linux":
-        # Check if we're actually on Ubuntu/compatible distro and log a warning if not
-        try:
-            with open("/etc/os-release", "r", encoding="utf-8") as f:
-                os_info = f.read().lower()
-                if "ubuntu" not in os_info and "debian" not in os_info:
-                    logging.warning(
-                        "llamacpp binaries are built for Ubuntu. "
-                        "Compatibility with other Linux distributions is not guaranteed."
-                    )
-        except (FileNotFoundError, PermissionError, OSError) as e:
-            logging.warning(
-                "Could not determine Linux distribution (%s). "
-                "llamacpp binaries are built for Ubuntu.",
-                str(e),
-            )
-
-
 class LlamaTelemetry:
     """
     Manages telemetry data collection and display for llama server.
@@ -283,7 +254,7 @@ def _launch_llama_subprocess(
     """
 
     # Get the current executable path (handles both Windows and Ubuntu structures)
-    _, exe_path = get_llama_server_paths()
+    exe_path = get_llama_server_exe_path()
 
     # Build the base command
     base_command = [exe_path, "-m", snapshot_files["variant"]]
@@ -350,68 +321,23 @@ def _launch_llama_subprocess(
 
 
 def server_load(model_config: PullConfig, telemetry: LlamaTelemetry):
-    # Validate platform support before proceeding
-    validate_platform_support()
+    # Install and/or update llama.cpp if needed
+    try:
+        install_llamacpp()
+    except NotImplementedError as e:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)
+        )
 
     # Get platform-specific paths at runtime
-    llama_server_exe_dir, llama_server_exe_path = get_llama_server_paths()
-
-    # Check whether the llamacpp install needs an upgrade
-    version_txt_path = os.path.join(llama_server_exe_dir, "version.txt")
-    if os.path.exists(version_txt_path):
-        with open(version_txt_path, "r", encoding="utf-8") as f:
-            llamacpp_installed_version = f.read()
-
-        if llamacpp_installed_version != LLAMA_VERSION:
-            # Remove the existing install, which will trigger a new install
-            # in the next code block
-            shutil.rmtree(llama_server_exe_dir)
-
-    # Download llama.cpp server if it isn't already available
-    if not os.path.exists(llama_server_exe_dir):
-        # Download llama.cpp server zip
-        llama_zip_url, filename = get_binary_url_and_filename(LLAMA_VERSION)
-        llama_zip_path = os.path.join(os.path.dirname(sys.executable), filename)
-        logging.info(f"Downloading llama.cpp server from {llama_zip_url}")
-
-        with requests.get(llama_zip_url, stream=True) as r:
-            r.raise_for_status()
-            with open(llama_zip_path, "wb") as f:
-                for chunk in r.iter_content(chunk_size=8192):
-                    f.write(chunk)
-
-        # Extract zip
-        logging.info(f"Extracting {llama_zip_path} to {llama_server_exe_dir}")
-        with zipfile.ZipFile(llama_zip_path, "r") as zip_ref:
-            zip_ref.extractall(llama_server_exe_dir)
-
-        # Make executable on Linux - need to update paths after extraction
-        if platform.system().lower() == "linux":
-            # Re-get the paths since extraction might have changed the directory structure
-            _, updated_exe_path = get_llama_server_paths()
-            if os.path.exists(updated_exe_path):
-                os.chmod(updated_exe_path, 0o755)
-                logging.info(f"Set executable permissions for {updated_exe_path}")
-            else:
-                logging.warning(
-                    f"Could not find llama-server executable at {updated_exe_path}"
-                )
-
-        # Save version.txt
-        with open(version_txt_path, "w", encoding="utf-8") as vf:
-            vf.write(LLAMA_VERSION)
-
-        # Delete zip file
-        os.remove(llama_zip_path)
-        logging.info("Cleaned up zip file")
+    llama_server_exe_path = get_llama_server_exe_path()
 
     # Download the gguf to the hugging face cache
-    model_manager = ModelManager()
-    snapshot_files = model_manager.download_gguf(model_config)
+    snapshot_files = download_gguf(model_config.checkpoint, model_config.mmproj)
     logging.debug(f"GGUF file paths: {snapshot_files}")
 
     # Check if model supports embeddings
-    supported_models = model_manager.supported_models
+    supported_models = ModelManager().supported_models
     model_info = supported_models.get(model_config.model_name, {})
     supports_embeddings = "embeddings" in model_info.get("labels", [])
     supports_reranking = "reranking" in model_info.get("labels", [])
@@ -523,6 +449,68 @@ def chat_completion(
             )
 
 
+def completion(completion_request: CompletionRequest, telemetry: LlamaTelemetry):
+    """
+    Handle text completions using the llamacpp server.
+
+    Args:
+        completion_request: The completion request containing prompt and parameters
+        telemetry: Telemetry object containing the server port
+
+    Returns:
+        Completion response from the llamacpp server
+    """
+    base_url = llamacpp_address(telemetry.port)
+    client = OpenAI(
+        base_url=base_url,
+        api_key="lemonade",
+    )
+
+    # Convert Pydantic model to dict and remove unset/null values
+    request_dict = completion_request.model_dump(exclude_unset=True, exclude_none=True)
+
+    # Check if streaming is requested
+    if completion_request.stream:
+
+        def event_stream():
+            try:
+                # Enable streaming
+                for chunk in client.completions.create(**request_dict):
+                    yield f"data: {chunk.model_dump_json()}\n\n"
+                yield "data: [DONE]\n\n"
+
+                # Show telemetry after completion
+                telemetry.show_telemetry()
+
+            except Exception as e:  # pylint: disable=broad-exception-caught
+                yield f'data: {{"error": "{str(e)}"}}\n\n'
+
+        return StreamingResponse(
+            event_stream(),
+            media_type="text/event-stream",
+            headers={
+                "Cache-Control": "no-cache",
+                "Connection": "keep-alive",
+            },
+        )
+    else:
+        # Non-streaming response
+        try:
+            # Disable streaming for non-streaming requests
+            response = client.completions.create(**request_dict)
+
+            # Show telemetry after completion
+            telemetry.show_telemetry()
+
+            return response
+
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"Completion error: {str(e)}",
+            )
+
+
 def embeddings(embeddings_request: EmbeddingsRequest, telemetry: LlamaTelemetry):
     """
     Generate embeddings using the llamacpp server.

lemonade/tools/server/serve.py

@@ -228,6 +228,7 @@ class Server(ManagementTool):
             self.app.get(f"{prefix}/health")(self.health)
             self.app.get(f"{prefix}/halt")(self.halt_generation)
             self.app.get(f"{prefix}/stats")(self.send_stats)
+            self.app.get(f"{prefix}/system-info")(self.get_system_info)
             self.app.post(f"{prefix}/completions")(self.completions)
             self.app.post(f"{prefix}/responses")(self.responses)
 
@@ -486,6 +487,9 @@ class Server(ManagementTool):
         # Load the model if it's different from the currently loaded one
         await self.load_llm(lc)
 
+        if self.llm_loaded.recipe == "llamacpp":
+            return llamacpp.completion(completion_request, self.llama_telemetry)
+
         # Check if the model supports reasoning
         reasoning_first_token = self.llm_loaded.reasoning
 
@@ -1276,6 +1280,34 @@ class Server(ManagementTool):
             ),
         }
 
+    async def get_system_info(self, request: Request):
+        """
+        Return system and device enumeration information.
+        Supports optional 'verbose' query parameter.
+        """
+        from lemonade.common.system_info import (
+            get_system_info_dict,
+            get_device_info_dict,
+            get_system_info as get_system_info_obj,
+        )
+
+        # Get verbose parameter from query string (default to False)
+        verbose = request.query_params.get("verbose", "false").lower() in ["true", "1"]
+
+        info = get_system_info_dict()
+        info["devices"] = get_device_info_dict()
+
+        # Filter out verbose-only information if not in verbose mode
+        if not verbose:
+            essential_keys = ["OS Version", "Processor", "Physical Memory", "devices"]
+            info = {k: v for k, v in info.items() if k in essential_keys}
+        else:
+            # In verbose mode, add Python packages at the end
+            system_info_obj = get_system_info_obj()
+            info["Python Packages"] = system_info_obj.get_python_packages()
+
+        return info
+
     def model_load_failure(self, model_reference: str, message: Optional[str] = None):
         """
         Clean up after a model load failure, then log it and raise