lemonade-sdk 8.0.5__tar.gz → 8.0.6__tar.gz

{lemonade_sdk-8.0.5/src/lemonade_sdk.egg-info → lemonade_sdk-8.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lemonade-sdk
-Version: 8.0.5
+Version: 8.0.6
 Summary: Lemonade SDK: Your LLM Aide for Validation and Deployment
 Author-email: lemonade@amd.com
 Requires-Python: >=3.10, <3.13
@@ -23,7 +23,7 @@ Requires-Dist: zstandard
 Requires-Dist: fastapi
 Requires-Dist: uvicorn[standard]
 Requires-Dist: openai>=1.81.0
-Requires-Dist: transformers<=4.51.3
+Requires-Dist: transformers<=4.53.2
 Requires-Dist: jinja2
 Requires-Dist: tabulate
 Requires-Dist: sentencepiece
@@ -284,7 +284,7 @@ New contributors can find beginner-friendly issues tagged with "Good First Issue
 
 ## Maintainers
 
-This project is sponsored by AMD. It is maintained by @danielholanda @jeremyfowers @ramkrishna @vgodsoe in equal measure. You can reach us by filing an [issue](https://github.com/lemonade-sdk/lemonade/issues), email [lemonade@amd.com](mailto:lemonade@amd.com), or join our [Discord](https://discord.gg/5xXzkMu8Zk).
+This project is sponsored by AMD. It is maintained by @danielholanda @jeremyfowers @ramkrishna @vgodsoe in equal measure. You can reach us by filing an [issue](https://github.com/lemonade-sdk/lemonade/issues), emailing [lemonade@amd.com](mailto:lemonade@amd.com), or joining our [Discord](https://discord.gg/5xXzkMu8Zk).
 
 ## License
 

{lemonade_sdk-8.0.5 → lemonade_sdk-8.0.6}/README.md

@@ -202,7 +202,7 @@ New contributors can find beginner-friendly issues tagged with "Good First Issue
 
 ## Maintainers
 
-This project is sponsored by AMD. It is maintained by @danielholanda @jeremyfowers @ramkrishna @vgodsoe in equal measure. You can reach us by filing an [issue](https://github.com/lemonade-sdk/lemonade/issues), email [lemonade@amd.com](mailto:lemonade@amd.com), or join our [Discord](https://discord.gg/5xXzkMu8Zk).
+This project is sponsored by AMD. It is maintained by @danielholanda @jeremyfowers @ramkrishna @vgodsoe in equal measure. You can reach us by filing an [issue](https://github.com/lemonade-sdk/lemonade/issues), emailing [lemonade@amd.com](mailto:lemonade@amd.com), or joining our [Discord](https://discord.gg/5xXzkMu8Zk).
 
 ## License
 

{lemonade_sdk-8.0.5 → lemonade_sdk-8.0.6}/setup.py

@@ -45,7 +45,7 @@ setup(
         "fastapi",
         "uvicorn[standard]",
         "openai>=1.81.0",
-        "transformers<=4.51.3",
+        "transformers<=4.53.2",
         "jinja2",
         "tabulate",
         "sentencepiece",

{lemonade_sdk-8.0.5 → lemonade_sdk-8.0.6}/src/lemonade/cache.py

@@ -34,7 +34,7 @@ def build_name(input_name):
     """
     Name the lemonade build by concatenating these two factors:
         1. Sanitize the input name (typically a model checkpoint name) by
-            replacing any `/` characters with `_`.
+            replacing any `/` characters with `_` and ':' characters with '-'.
         2. Timestamp to ensure that builds in the same cache will not
             collide in the same build directory.
 
@@ -47,6 +47,7 @@ def build_name(input_name):
     else:
         # Sanitize the input name
         input_name_sanitized = input_name.replace("/", "_")
+        input_name_sanitized = input_name_sanitized.replace(":", "-")
 
     # Get the formatted timestamp string
     timestamp = get_timestamp()
@@ -79,6 +80,7 @@ class Keys:
     MAX_MEMORY_USED_GB = "max_memory_used_GB"
     MAX_MEMORY_USED_GBYTE = "max_memory_used_gbyte"
     RYZEN_AI_VERSION_INFO = "ryzen_ai_version_info"
+    LLAMA_CLI_VERSION_INFO = "llama_cli_version_info"
 
 
 # This file was originally licensed under Apache 2.0. It has been modified.

{lemonade_sdk-8.0.5 → lemonade_sdk-8.0.6}/src/lemonade/tools/adapter.py

@@ -13,6 +13,9 @@ class ModelAdapter(abc.ABC):
         """
         self.tokens_per_second = None
         self.time_to_first_token = None
+        self.prompt_tokens = None
+        self.response_tokens = None
+
         self.type = "generic"
 
     @abc.abstractmethod
@@ -22,6 +25,9 @@ class ModelAdapter(abc.ABC):
 
         We try to keep the signature here minimal to allow for maximum compatibility
         with recipe components, which themselves may not support a lot of arguments.
+
+        The generate method should store prompt and response lengths (in tokens)
+        in the prompt_tokens and response_tokens members.
         """
 
 

{lemonade_sdk-8.0.5 → lemonade_sdk-8.0.6}/src/lemonade/tools/huggingface/utils.py

@@ -108,7 +108,9 @@ class HuggingfaceAdapter(ModelAdapter):
         with torch.no_grad(), torch.inference_mode():
             outputs = self.model.generate(input_ids=input_ids, **generation_kwargs)
 
-            return outputs
+        self.prompt_tokens = input_ids.shape[1]
+        self.response_tokens = len(outputs[0]) - self.prompt_tokens
+        return outputs
 
     def _model_call(self, input_tensor):
         """Forward pass through the model to get logits
@@ -341,12 +343,11 @@ def benchmark_huggingface_llm(
 
                 latency = end_time - start_time
 
-                token_len = outputs.shape[1] - input_ids.shape[1]
-                tokens_out_len_list.append(token_len)
+                tokens_out_len_list.append(model.response_tokens)
 
                 # Only count an iteration if it produced enough tokens
-                if token_len >= target_output_tokens:
-                    per_iteration_result.append((latency, token_len))
+                if model.response_tokens >= target_output_tokens:
+                    per_iteration_result.append((latency, model.response_tokens))
 
                 report_progress_fn(
                     (warmup_iterations + count + 1) / (warmup_iterations + iterations)

{lemonade_sdk-8.0.5 → lemonade_sdk-8.0.6}/src/lemonade/tools/llamacpp/bench.py

@@ -3,27 +3,31 @@ import statistics
 from statistics import StatisticsError
 from lemonade.state import State
 from lemonade.cache import Keys
-from lemonade.tools.llamacpp.load import LlamaCppAdapter
+from lemonade.tools.llamacpp.utils import LlamaCppAdapter
 from lemonade.tools.bench import Bench
 
 
 class LlamaCppBench(Bench):
+    """
+    Benchmark a llama.cpp model
+    """
 
-    unique_name = "llama-cpp-bench"
+    unique_name = "llamacpp-bench"
 
     def __init__(self):
         super().__init__()
 
         # Additional statistics generated by this bench tool
-        self.status_stats += [
+        self.status_stats.insert(
+            self.status_stats.index(Keys.TOKEN_GENERATION_TOKENS_PER_SECOND) + 1,
             Keys.STD_DEV_TOKENS_PER_SECOND,
-        ]
+        )
         self.std_dev_token_generation_tokens_per_second_list = []
 
     @staticmethod
     def parser(add_help: bool = True) -> argparse.ArgumentParser:
         parser = __class__.helpful_parser(
-            short_description="Benchmark a llama.cpp model",
+            short_description="Benchmark an LLM in llama.cpp",
             add_help=add_help,
         )
 
@@ -53,38 +57,20 @@ class LlamaCppBench(Bench):
                     f"{self.__class__.unique_name} requires a LlamaCppAdapter model to be "
                     "loaded first. Please run load-llama-cpp before this tool."
                 )
+        model: LlamaCppAdapter = state.model
 
-        iteration_tokens_per_second = []
-        iteration_time_to_first_token = []
+        per_iteration_tokens_per_second = []
+        per_iteration_time_to_first_token = []
 
         for iteration in range(iterations + warmup_iterations):
             try:
                 # Use the adapter's generate method which already has the timeout
                 # and error handling
-                raw_output, stderr = state.model.generate(prompt, return_raw=True)
-
-                # Parse the timing information from the output
-                ms_per_token = None
-                time_to_first_token_ms = None
-                input_tokens = None
-
-                # Look for timing in both stdout and stderr
-                for output in [raw_output, stderr]:
-                    for line in output.splitlines():
-                        if "llama_perf_context_print:        eval time =" in line:
-                            parts = line.split("(")[1].strip()
-                            parts = parts.split(",")
-                            ms_per_token = float(
-                                parts[0].split("ms per token")[0].strip()
-                            )
-                        if "llama_perf_context_print: prompt eval time =" in line:
-                            parts = line.split("=")[1].split("/")
-                            time_to_first_token_ms = float(
-                                parts[0].split("ms")[0].strip()
-                            )
-                            input_tokens = int(parts[1].split("tokens")[0].strip())
-
-                if ms_per_token is None or time_to_first_token_ms is None:
+                model.time_to_first_token = None
+                model.tokens_per_second = None
+                raw_output, stderr = model.generate(prompt, return_raw=True)
+
+                if model.time_to_first_token is None or model.tokens_per_second is None:
                     error_msg = (
                         "Could not find timing information in llama.cpp output.\n"
                     )
@@ -92,17 +78,11 @@ class LlamaCppBench(Bench):
                     error_msg += "Stderr:\n" + stderr
                     raise Exception(error_msg)
 
-                # When output_tokens is set to 1 for accuracy tests, ms_per_token tends to 0
-                # and causes a divide-by-zero error. Set tokens_per_second to 0 in such cases
-                # as performance data for generating a few tokens is not relevant.
-                tokens_per_second = 0
-                if output_tokens > 5 and ms_per_token > 0:
-                    tokens_per_second = 1000 / ms_per_token
-                time_to_first_token = time_to_first_token_ms / 1000
+                self.tokens_out_len_list.append(model.response_tokens)
 
                 if iteration > warmup_iterations - 1:
-                    iteration_tokens_per_second.append(tokens_per_second)
-                    iteration_time_to_first_token.append(time_to_first_token)
+                    per_iteration_tokens_per_second.append(model.tokens_per_second)
+                    per_iteration_time_to_first_token.append(model.time_to_first_token)
 
                 report_progress_fn((iteration + 1) / (warmup_iterations + iterations))
 
@@ -110,25 +90,25 @@ class LlamaCppBench(Bench):
                 error_msg = f"Failed to run benchmark: {str(e)}"
                 raise Exception(error_msg)
 
-        self.input_ids_len_list.append(input_tokens)
-        mean_time_to_first_token = statistics.mean(iteration_time_to_first_token)
+        self.input_ids_len_list.append(model.prompt_tokens)
+        mean_time_to_first_token = statistics.mean(per_iteration_time_to_first_token)
         self.mean_time_to_first_token_list.append(mean_time_to_first_token)
         self.prefill_tokens_per_second_list.append(
-            input_tokens / mean_time_to_first_token
+            model.prompt_tokens / mean_time_to_first_token
         )
         self.token_generation_tokens_per_second_list.append(
-            statistics.mean(iteration_tokens_per_second)
+            statistics.mean(per_iteration_tokens_per_second)
         )
         try:
             self.std_dev_time_to_first_token_list.append(
-                statistics.stdev(iteration_time_to_first_token)
+                statistics.stdev(per_iteration_time_to_first_token)
             )
         except StatisticsError:
             # Less than 2 measurements
             self.std_dev_time_to_first_token_list.append(None)
         try:
             self.std_dev_token_generation_tokens_per_second_list.append(
-                statistics.stdev(iteration_tokens_per_second)
+                statistics.stdev(per_iteration_tokens_per_second)
             )
         except StatisticsError:
             # Less than 2 measurements

lemonade_sdk-8.0.6/src/lemonade/tools/llamacpp/load.py

@@ -0,0 +1,185 @@
+import argparse
+import os
+import lemonade.common.printing as printing
+import lemonade.common.status as status
+from lemonade.state import State
+from lemonade.tools import FirstTool
+from lemonade.cache import Keys
+
+
+class LoadLlamaCpp(FirstTool):
+    unique_name = "llamacpp-load"
+
+    def __init__(self):
+        super().__init__(monitor_message="Loading llama.cpp model")
+
+        self.status_stats = [
+            Keys.DEVICE,
+        ]
+
+    @staticmethod
+    def parser(add_help: bool = True) -> argparse.ArgumentParser:
+        parser = __class__.helpful_parser(
+            short_description="Wrap llama.cpp models with an API",
+            add_help=add_help,
+        )
+
+        parser.add_argument(
+            "-d",
+            "--device",
+            choices=["cpu", "igpu"],
+            default="igpu",
+            help="Which device to load the model on to (default: igpu)",
+        )
+
+        default_threads = -1
+        parser.add_argument(
+            "--threads",
+            required=False,
+            type=int,
+            default=default_threads,
+            help=f"Number of threads to use during generation (default: {default_threads})",
+        )
+
+        context_size = 4096
+        parser.add_argument(
+            "--context-size",
+            required=False,
+            type=int,
+            default=context_size,
+            help=f"Size of the prompt context (default: {context_size}. 0 = loaded from model)",
+        )
+
+        output_tokens = 512
+        parser.add_argument(
+            "--output-tokens",
+            required=False,
+            type=int,
+            default=output_tokens,
+            help=f"Maximum number of output tokens to generate (default: {output_tokens})",
+        )
+
+        parser.add_argument(
+            "--reasoning",
+            action="store_true",
+            help="Set this flag to indicate the model is a reasoning model",
+        )
+
+        return parser
+
+    def run(
+        self,
+        state: State,
+        input: str = "",
+        device: str = "igpu",
+        context_size: int = 512,
+        threads: int = 1,
+        output_tokens: int = 512,
+        reasoning: bool = False,
+    ) -> State:
+        """
+        Load a llama.cpp model
+        """
+
+        from lemonade.common.network import is_offline
+        from lemonade.tools.llamacpp.utils import (
+            install_llamacpp,
+            get_llama_cli_exe_path,
+            get_llama_installed_version,
+            parse_checkpoint,
+            download_gguf,
+            get_local_checkpoint_path,
+            LlamaCppTokenizerAdapter,
+            LlamaCppAdapter,
+        )
+
+        # Validate and install llama.cpp, if needed
+        install_llamacpp()
+
+        # Check if input is a local folder containing a .GGUF model
+        if os.path.isdir(input):
+            # input is a local folder
+            local_model_folder = os.path.abspath(input)
+            checkpoint = "local_model"
+            state.checkpoint = checkpoint
+            state.save_stat(Keys.CHECKPOINT, checkpoint)
+            state.save_stat(Keys.LOCAL_MODEL_FOLDER, local_model_folder)
+
+            # See if there is a file ending in ".gguf" in this folder
+            dir = os.listdir(input)
+            gguf_files = [filename for filename in dir if filename.endswith(".gguf")]
+            if len(gguf_files) == 0:
+                raise ValueError(
+                    f"The folder {input} does not contain a GGUF model file."
+                )
+            model_to_use = gguf_files[0]
+            full_model_path = os.path.join(local_model_folder, model_to_use)
+
+        else:
+            # Input is a model checkpoint
+            checkpoint = input
+            state.checkpoint = checkpoint
+            state.save_stat(Keys.CHECKPOINT, checkpoint)
+
+            # Make sure that a variant is provided for the GGUF model
+            base_checkpoint, variant = parse_checkpoint(checkpoint)
+            if variant is None:
+                raise ValueError(
+                    "You are required to provide a 'variant' when "
+                    "selecting a GGUF model. The variant is provided "
+                    "as CHECKPOINT:VARIANT. For example: "
+                    "Qwen/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_0 or "
+                    "Qwen/Qwen2.5-Coder-3B-Instruct-GGUF:qwen2.5-coder-3b-instruct-q4_0.gguf"
+                )
+
+            # Auto-detect offline status
+            offline = is_offline()
+            if offline:
+                printing.log_warning(
+                    "Network connectivity to huggingface.co not detected. Running in offline mode."
+                )
+                full_model_path, model_to_use = get_local_checkpoint_path(
+                    base_checkpoint, variant
+                )
+                if not full_model_path:
+                    raise ValueError(
+                        f"Model {checkpoint} is not available locally."
+                        f"Cannot download in offline mode."
+                    )
+
+            else:
+
+                snapshot_files = download_gguf(checkpoint)
+                full_model_path = snapshot_files["variant"]
+                model_to_use = os.path.basename(full_model_path)
+
+        llama_cli_exe_path = get_llama_cli_exe_path()
+        printing.log_info(f"Using llama_cli for GGUF model: {llama_cli_exe_path}")
+
+        # Get the directory containing the executable for shared libraries
+        lib_dir = os.path.dirname(llama_cli_exe_path)
+
+        # Pass the model and inputs into state
+        state.model = LlamaCppAdapter(
+            model=full_model_path,
+            device=device,
+            output_tokens=output_tokens,
+            context_size=context_size,
+            threads=threads,
+            executable=llama_cli_exe_path,
+            reasoning=reasoning,
+            lib_dir=lib_dir,
+        )
+        state.tokenizer = LlamaCppTokenizerAdapter()
+        state.device = device
+
+        # Save initial stats
+        state.save_stat(Keys.DEVICE, device)
+        state.save_stat(Keys.LLAMA_CLI_VERSION_INFO, get_llama_installed_version())
+
+        status.add_to_state(state=state, name=input, model=model_to_use)
+        return state
+
+
+# This file was originally licensed under Apache 2.0. It has been modified.
+# Modifications Copyright (c) 2025 AMD