lemonade-sdk 8.0.5__tar.gz → 8.1.0__tar.gz

{lemonade_sdk-8.0.5/src/lemonade_sdk.egg-info → lemonade_sdk-8.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lemonade-sdk
-Version: 8.0.5
+Version: 8.1.0
 Summary: Lemonade SDK: Your LLM Aide for Validation and Deployment
 Author-email: lemonade@amd.com
 Requires-Python: >=3.10, <3.13
@@ -22,16 +22,15 @@ Requires-Dist: pytz
 Requires-Dist: zstandard
 Requires-Dist: fastapi
 Requires-Dist: uvicorn[standard]
-Requires-Dist: openai>=1.81.0
-Requires-Dist: transformers<=4.51.3
+Requires-Dist: openai<1.97.1,>=1.81.0
+Requires-Dist: transformers<=4.53.2
 Requires-Dist: jinja2
 Requires-Dist: tabulate
 Requires-Dist: sentencepiece
 Requires-Dist: huggingface-hub==0.33.0
-Provides-Extra: oga-hybrid
-Requires-Dist: onnx==1.16.1; extra == "oga-hybrid"
-Requires-Dist: numpy==1.26.4; extra == "oga-hybrid"
-Requires-Dist: protobuf>=6.30.1; extra == "oga-hybrid"
+Provides-Extra: oga-ryzenai
+Requires-Dist: onnxruntime-genai-directml-ryzenai==0.7.0.2; extra == "oga-ryzenai"
+Requires-Dist: protobuf>=6.30.1; extra == "oga-ryzenai"
 Provides-Extra: oga-cpu
 Requires-Dist: onnxruntime-genai==0.8.2; extra == "oga-cpu"
 Requires-Dist: onnxruntime>=1.22.0; extra == "oga-cpu"
@@ -43,14 +42,32 @@ Requires-Dist: pandas>=1.5.3; extra == "dev"
 Requires-Dist: matplotlib; extra == "dev"
 Requires-Dist: human-eval-windows==1.0.4; extra == "dev"
 Requires-Dist: lm-eval[api]; extra == "dev"
+Provides-Extra: oga-hybrid
+Requires-Dist: lemonade-sdk[oga-ryzenai]; extra == "oga-hybrid"
+Provides-Extra: oga-unified
+Requires-Dist: lemonade-sdk[oga-ryzenai]; extra == "oga-unified"
 Provides-Extra: oga-hybrid-minimal
-Requires-Dist: lemonade-sdk[oga-hybrid]; extra == "oga-hybrid-minimal"
+Requires-Dist: lemonade-sdk[oga-ryzenai]; extra == "oga-hybrid-minimal"
 Provides-Extra: oga-cpu-minimal
 Requires-Dist: lemonade-sdk[oga-cpu]; extra == "oga-cpu-minimal"
+Provides-Extra: oga-npu-minimal
+Requires-Dist: lemonade-sdk[oga-ryzenai]; extra == "oga-npu-minimal"
 Provides-Extra: llm
 Requires-Dist: lemonade-sdk[dev]; extra == "llm"
 Provides-Extra: llm-oga-cpu
 Requires-Dist: lemonade-sdk[dev,oga-cpu]; extra == "llm-oga-cpu"
+Provides-Extra: llm-oga-npu
+Requires-Dist: onnx==1.16.0; extra == "llm-oga-npu"
+Requires-Dist: onnxruntime==1.18.0; extra == "llm-oga-npu"
+Requires-Dist: numpy==1.26.4; extra == "llm-oga-npu"
+Requires-Dist: protobuf>=6.30.1; extra == "llm-oga-npu"
+Requires-Dist: lemonade-sdk[dev]; extra == "llm-oga-npu"
+Provides-Extra: llm-oga-hybrid
+Requires-Dist: onnx==1.16.1; extra == "llm-oga-hybrid"
+Requires-Dist: numpy==1.26.4; extra == "llm-oga-hybrid"
+Requires-Dist: protobuf>=6.30.1; extra == "llm-oga-hybrid"
+Provides-Extra: llm-oga-unified
+Requires-Dist: lemonade-sdk[dev,llm-oga-hybrid]; extra == "llm-oga-unified"
 Provides-Extra: llm-oga-igpu
 Requires-Dist: onnxruntime-genai-directml==0.6.0; extra == "llm-oga-igpu"
 Requires-Dist: onnxruntime-directml<1.22.0,>=1.19.0; extra == "llm-oga-igpu"
@@ -61,16 +78,6 @@ Requires-Dist: onnxruntime-genai-cuda==0.8.2; extra == "llm-oga-cuda"
 Requires-Dist: onnxruntime-gpu>=1.22.0; extra == "llm-oga-cuda"
 Requires-Dist: transformers<=4.51.3; extra == "llm-oga-cuda"
 Requires-Dist: lemonade-sdk[dev]; extra == "llm-oga-cuda"
-Provides-Extra: llm-oga-npu
-Requires-Dist: onnx==1.16.0; extra == "llm-oga-npu"
-Requires-Dist: onnxruntime==1.18.0; extra == "llm-oga-npu"
-Requires-Dist: numpy==1.26.4; extra == "llm-oga-npu"
-Requires-Dist: protobuf>=6.30.1; extra == "llm-oga-npu"
-Requires-Dist: lemonade-sdk[dev]; extra == "llm-oga-npu"
-Provides-Extra: llm-oga-hybrid
-Requires-Dist: lemonade-sdk[dev,oga-hybrid]; extra == "llm-oga-hybrid"
-Provides-Extra: llm-oga-unified
-Requires-Dist: lemonade-sdk[llm-oga-hybrid]; extra == "llm-oga-unified"
 Dynamic: author-email
 Dynamic: description
 Dynamic: description-content-type
@@ -174,7 +181,7 @@ lemonade-server list
 
 ## Model Library
 
-Lemonade supports both GGUF and ONNX models as detailed in the [Supported Configuration](#supported-configurations) section. A list of all built-in models is available [here](https://lemonade-server.ai/docs/server/models/).
+Lemonade supports both GGUF and ONNX models as detailed in the [Supported Configuration](#supported-configurations) section. A list of all built-in models is available [here](https://lemonade-server.ai/docs/server/server_models/).
 
 You can also import custom GGUF and ONNX models from Hugging Face by using our [Model Manager](http://localhost:8000/#model-management) (requires server to be running).
 <p align="center">
@@ -263,7 +270,7 @@ completion = client.chat.completions.create(
 print(completion.choices[0].message.content)
 ```
 
-For more detailed integration instructions, see the [Integration Guide](./server_integration.md).
+For more detailed integration instructions, see the [Integration Guide](./docs/server/server_integration.md).
 
 ## Beyond an LLM Server
 
@@ -272,6 +279,10 @@ The [Lemonade SDK](./docs/README.md) also include the following components:
 - 🐍 **[Lemonade API](./docs/lemonade_api.md)**: High-level Python API to directly integrate Lemonade LLMs into Python applications.
 - 🖥️ **[Lemonade CLI](./docs/dev_cli/README.md)**: The `lemonade` CLI lets you mix-and-match LLMs (ONNX, GGUF, SafeTensors) with prompting templates, accuracy testing, performance benchmarking, and memory profiling to characterize your models on your hardware.
 
+## FAQ
+
+To read our frequently asked questions, see our [FAQ Guide](./docs/faq.md)
+
 ## Contributing
 
 We are actively seeking collaborators from across the industry. If you would like to contribute to this project, please check out our [contribution guide](./docs/contribute.md).
@@ -284,7 +295,7 @@ New contributors can find beginner-friendly issues tagged with "Good First Issue
 
 ## Maintainers
 
-This project is sponsored by AMD. It is maintained by @danielholanda @jeremyfowers @ramkrishna @vgodsoe in equal measure. You can reach us by filing an [issue](https://github.com/lemonade-sdk/lemonade/issues), email [lemonade@amd.com](mailto:lemonade@amd.com), or join our [Discord](https://discord.gg/5xXzkMu8Zk).
+This project is sponsored by AMD. It is maintained by @danielholanda @jeremyfowers @ramkrishna @vgodsoe in equal measure. You can reach us by filing an [issue](https://github.com/lemonade-sdk/lemonade/issues), emailing [lemonade@amd.com](mailto:lemonade@amd.com), or joining our [Discord](https://discord.gg/5xXzkMu8Zk).
 
 ## License
 

{lemonade_sdk-8.0.5 → lemonade_sdk-8.1.0}/README.md

@@ -92,7 +92,7 @@ lemonade-server list
 
 ## Model Library
 
-Lemonade supports both GGUF and ONNX models as detailed in the [Supported Configuration](#supported-configurations) section. A list of all built-in models is available [here](https://lemonade-server.ai/docs/server/models/).
+Lemonade supports both GGUF and ONNX models as detailed in the [Supported Configuration](#supported-configurations) section. A list of all built-in models is available [here](https://lemonade-server.ai/docs/server/server_models/).
 
 You can also import custom GGUF and ONNX models from Hugging Face by using our [Model Manager](http://localhost:8000/#model-management) (requires server to be running).
 <p align="center">
@@ -181,7 +181,7 @@ completion = client.chat.completions.create(
 print(completion.choices[0].message.content)
 ```
 
-For more detailed integration instructions, see the [Integration Guide](./server_integration.md).
+For more detailed integration instructions, see the [Integration Guide](./docs/server/server_integration.md).
 
 ## Beyond an LLM Server
 
@@ -190,6 +190,10 @@ The [Lemonade SDK](./docs/README.md) also include the following components:
 - 🐍 **[Lemonade API](./docs/lemonade_api.md)**: High-level Python API to directly integrate Lemonade LLMs into Python applications.
 - 🖥️ **[Lemonade CLI](./docs/dev_cli/README.md)**: The `lemonade` CLI lets you mix-and-match LLMs (ONNX, GGUF, SafeTensors) with prompting templates, accuracy testing, performance benchmarking, and memory profiling to characterize your models on your hardware.
 
+## FAQ
+
+To read our frequently asked questions, see our [FAQ Guide](./docs/faq.md)
+
 ## Contributing
 
 We are actively seeking collaborators from across the industry. If you would like to contribute to this project, please check out our [contribution guide](./docs/contribute.md).
@@ -202,7 +206,7 @@ New contributors can find beginner-friendly issues tagged with "Good First Issue
 
 ## Maintainers
 
-This project is sponsored by AMD. It is maintained by @danielholanda @jeremyfowers @ramkrishna @vgodsoe in equal measure. You can reach us by filing an [issue](https://github.com/lemonade-sdk/lemonade/issues), email [lemonade@amd.com](mailto:lemonade@amd.com), or join our [Discord](https://discord.gg/5xXzkMu8Zk).
+This project is sponsored by AMD. It is maintained by @danielholanda @jeremyfowers @ramkrishna @vgodsoe in equal measure. You can reach us by filing an [issue](https://github.com/lemonade-sdk/lemonade/issues), emailing [lemonade@amd.com](mailto:lemonade@amd.com), or joining our [Discord](https://discord.gg/5xXzkMu8Zk).
 
 ## License
 

{lemonade_sdk-8.0.5 → lemonade_sdk-8.1.0}/setup.py

@@ -44,8 +44,8 @@ setup(
         "zstandard",
         "fastapi",
         "uvicorn[standard]",
-        "openai>=1.81.0",
-        "transformers<=4.51.3",
+        "openai>=1.81.0,<1.97.1",
+        "transformers<=4.53.2",
         "jinja2",
         "tabulate",
         "sentencepiece",
@@ -54,11 +54,9 @@ setup(
     extras_require={
         # The non-dev extras are meant to deploy specific backends into end-user
         # applications, without including developer-focused tools
-        "oga-hybrid": [
-            # Note: `lemonade-install --ryzenai hybrid` is necessary
-            # to complete installation
-            "onnx==1.16.1",
-            "numpy==1.26.4",
+        # Primary NPU extra using unified PyPI package
+        "oga-ryzenai": [
+            "onnxruntime-genai-directml-ryzenai==0.7.0.2",
             "protobuf>=6.30.1",
         ],
         "oga-cpu": [
@@ -81,11 +79,31 @@ setup(
             "lm-eval[api]",
         ],
         # Keep backwards compatibility for old extras names
-        "oga-hybrid-minimal": ["lemonade-sdk[oga-hybrid]"],
+        "oga-hybrid": ["lemonade-sdk[oga-ryzenai]"],
+        "oga-unified": ["lemonade-sdk[oga-ryzenai]"],
+        "oga-hybrid-minimal": ["lemonade-sdk[oga-ryzenai]"],
         "oga-cpu-minimal": ["lemonade-sdk[oga-cpu]"],
+        "oga-npu-minimal": ["lemonade-sdk[oga-ryzenai]"],
         "llm": ["lemonade-sdk[dev]"],
         "llm-oga-cpu": ["lemonade-sdk[dev,oga-cpu]"],
         # The following extras are deprecated and/or not commonly used
+        "llm-oga-npu": [
+            "onnx==1.16.0",
+            # NPU requires specific onnxruntime version for Ryzen AI compatibility
+            # This may conflict with other OGA extras that require >=1.22.0
+            "onnxruntime==1.18.0",
+            "numpy==1.26.4",
+            "protobuf>=6.30.1",
+            "lemonade-sdk[dev]",
+        ],
+        "llm-oga-hybrid": [
+            # Note: `lemonade-install --ryzenai hybrid` is necessary
+            # to complete installation for RAI 1.4.0.
+            "onnx==1.16.1",
+            "numpy==1.26.4",
+            "protobuf>=6.30.1",
+        ],
+        "llm-oga-unified": ["lemonade-sdk[dev, llm-oga-hybrid]"],
         "llm-oga-igpu": [
             "onnxruntime-genai-directml==0.6.0",
             "onnxruntime-directml>=1.19.0,<1.22.0",
@@ -98,17 +116,6 @@ setup(
             "transformers<=4.51.3",
             "lemonade-sdk[dev]",
         ],
-        "llm-oga-npu": [
-            "onnx==1.16.0",
-            # NPU requires specific onnxruntime version for Ryzen AI compatibility
-            # This may conflict with other OGA extras that require >=1.22.0
-            "onnxruntime==1.18.0",
-            "numpy==1.26.4",
-            "protobuf>=6.30.1",
-            "lemonade-sdk[dev]",
-        ],
-        "llm-oga-hybrid": ["lemonade-sdk[dev,oga-hybrid]"],
-        "llm-oga-unified": ["lemonade-sdk[llm-oga-hybrid]"],
     },
     classifiers=[],
     entry_points={

{lemonade_sdk-8.0.5 → lemonade_sdk-8.1.0}/src/lemonade/cache.py

@@ -34,7 +34,7 @@ def build_name(input_name):
     """
     Name the lemonade build by concatenating these two factors:
         1. Sanitize the input name (typically a model checkpoint name) by
-            replacing any `/` characters with `_`.
+            replacing any `/` characters with `_` and ':' characters with '-'.
         2. Timestamp to ensure that builds in the same cache will not
             collide in the same build directory.
 
@@ -47,6 +47,7 @@ def build_name(input_name):
     else:
         # Sanitize the input name
         input_name_sanitized = input_name.replace("/", "_")
+        input_name_sanitized = input_name_sanitized.replace(":", "-")
 
     # Get the formatted timestamp string
     timestamp = get_timestamp()
@@ -79,6 +80,7 @@ class Keys:
     MAX_MEMORY_USED_GB = "max_memory_used_GB"
     MAX_MEMORY_USED_GBYTE = "max_memory_used_gbyte"
     RYZEN_AI_VERSION_INFO = "ryzen_ai_version_info"
+    LLAMA_CLI_VERSION_INFO = "llama_cli_version_info"
 
 
 # This file was originally licensed under Apache 2.0. It has been modified.

{lemonade_sdk-8.0.5 → lemonade_sdk-8.1.0}/src/lemonade/common/network.py

@@ -1,7 +1,7 @@
 import os
 from typing import Optional
 import socket
-from huggingface_hub import model_info
+from huggingface_hub import model_info, snapshot_download
 
 
 def is_offline():
@@ -48,3 +48,20 @@ def get_base_model(checkpoint: str) -> Optional[str]:
     except Exception:  # pylint: disable=broad-except
         pass
     return None
+
+
+def custom_snapshot_download(repo_id, **kwargs):
+    """
+    Custom snapshot download with retry logic for Windows symlink privilege errors.
+    """
+    for attempt in range(2):
+        try:
+            return snapshot_download(repo_id=repo_id, **kwargs)
+        except OSError as e:
+            if (
+                hasattr(e, "winerror")
+                and e.winerror == 1314  # pylint: disable=no-member
+                and attempt < 1
+            ):
+                continue
+            raise

{lemonade_sdk-8.0.5 → lemonade_sdk-8.1.0}/src/lemonade/tools/adapter.py

@@ -13,6 +13,9 @@ class ModelAdapter(abc.ABC):
         """
         self.tokens_per_second = None
         self.time_to_first_token = None
+        self.prompt_tokens = None
+        self.response_tokens = None
+
         self.type = "generic"
 
     @abc.abstractmethod
@@ -22,6 +25,9 @@ class ModelAdapter(abc.ABC):
 
         We try to keep the signature here minimal to allow for maximum compatibility
         with recipe components, which themselves may not support a lot of arguments.
+
+        The generate method should store prompt and response lengths (in tokens)
+        in the prompt_tokens and response_tokens members.
         """
 
 

{lemonade_sdk-8.0.5 → lemonade_sdk-8.1.0}/src/lemonade/tools/huggingface/utils.py

@@ -108,7 +108,9 @@ class HuggingfaceAdapter(ModelAdapter):
         with torch.no_grad(), torch.inference_mode():
             outputs = self.model.generate(input_ids=input_ids, **generation_kwargs)
 
-            return outputs
+        self.prompt_tokens = input_ids.shape[1]
+        self.response_tokens = len(outputs[0]) - self.prompt_tokens
+        return outputs
 
     def _model_call(self, input_tensor):
         """Forward pass through the model to get logits
@@ -341,12 +343,11 @@ def benchmark_huggingface_llm(
 
                 latency = end_time - start_time
 
-                token_len = outputs.shape[1] - input_ids.shape[1]
-                tokens_out_len_list.append(token_len)
+                tokens_out_len_list.append(model.response_tokens)
 
                 # Only count an iteration if it produced enough tokens
-                if token_len >= target_output_tokens:
-                    per_iteration_result.append((latency, token_len))
+                if model.response_tokens >= target_output_tokens:
+                    per_iteration_result.append((latency, model.response_tokens))
 
                 report_progress_fn(
                     (warmup_iterations + count + 1) / (warmup_iterations + iterations)

{lemonade_sdk-8.0.5 → lemonade_sdk-8.1.0}/src/lemonade/tools/llamacpp/bench.py

@@ -3,27 +3,31 @@ import statistics
 from statistics import StatisticsError
 from lemonade.state import State
 from lemonade.cache import Keys
-from lemonade.tools.llamacpp.load import LlamaCppAdapter
+from lemonade.tools.llamacpp.utils import LlamaCppAdapter
 from lemonade.tools.bench import Bench
 
 
 class LlamaCppBench(Bench):
+    """
+    Benchmark a llama.cpp model
+    """
 
-    unique_name = "llama-cpp-bench"
+    unique_name = "llamacpp-bench"
 
     def __init__(self):
         super().__init__()
 
         # Additional statistics generated by this bench tool
-        self.status_stats += [
+        self.status_stats.insert(
+            self.status_stats.index(Keys.TOKEN_GENERATION_TOKENS_PER_SECOND) + 1,
             Keys.STD_DEV_TOKENS_PER_SECOND,
-        ]
+        )
         self.std_dev_token_generation_tokens_per_second_list = []
 
     @staticmethod
     def parser(add_help: bool = True) -> argparse.ArgumentParser:
         parser = __class__.helpful_parser(
-            short_description="Benchmark a llama.cpp model",
+            short_description="Benchmark an LLM in llama.cpp",
             add_help=add_help,
         )
 
@@ -53,38 +57,22 @@ class LlamaCppBench(Bench):
                     f"{self.__class__.unique_name} requires a LlamaCppAdapter model to be "
                     "loaded first. Please run load-llama-cpp before this tool."
                 )
+        model: LlamaCppAdapter = state.model
 
-        iteration_tokens_per_second = []
-        iteration_time_to_first_token = []
+        per_iteration_tokens_per_second = []
+        per_iteration_time_to_first_token = []
 
         for iteration in range(iterations + warmup_iterations):
             try:
                 # Use the adapter's generate method which already has the timeout
                 # and error handling
-                raw_output, stderr = state.model.generate(prompt, return_raw=True)
-
-                # Parse the timing information from the output
-                ms_per_token = None
-                time_to_first_token_ms = None
-                input_tokens = None
-
-                # Look for timing in both stdout and stderr
-                for output in [raw_output, stderr]:
-                    for line in output.splitlines():
-                        if "llama_perf_context_print:        eval time =" in line:
-                            parts = line.split("(")[1].strip()
-                            parts = parts.split(",")
-                            ms_per_token = float(
-                                parts[0].split("ms per token")[0].strip()
-                            )
-                        if "llama_perf_context_print: prompt eval time =" in line:
-                            parts = line.split("=")[1].split("/")
-                            time_to_first_token_ms = float(
-                                parts[0].split("ms")[0].strip()
-                            )
-                            input_tokens = int(parts[1].split("tokens")[0].strip())
-
-                if ms_per_token is None or time_to_first_token_ms is None:
+                model.time_to_first_token = None
+                model.tokens_per_second = None
+                raw_output, stderr = model.generate(
+                    prompt, max_new_tokens=output_tokens, return_raw=True
+                )
+
+                if model.time_to_first_token is None or model.tokens_per_second is None:
                     error_msg = (
                         "Could not find timing information in llama.cpp output.\n"
                     )
@@ -92,17 +80,11 @@ class LlamaCppBench(Bench):
                     error_msg += "Stderr:\n" + stderr
                     raise Exception(error_msg)
 
-                # When output_tokens is set to 1 for accuracy tests, ms_per_token tends to 0
-                # and causes a divide-by-zero error. Set tokens_per_second to 0 in such cases
-                # as performance data for generating a few tokens is not relevant.
-                tokens_per_second = 0
-                if output_tokens > 5 and ms_per_token > 0:
-                    tokens_per_second = 1000 / ms_per_token
-                time_to_first_token = time_to_first_token_ms / 1000
+                self.tokens_out_len_list.append(model.response_tokens)
 
                 if iteration > warmup_iterations - 1:
-                    iteration_tokens_per_second.append(tokens_per_second)
-                    iteration_time_to_first_token.append(time_to_first_token)
+                    per_iteration_tokens_per_second.append(model.tokens_per_second)
+                    per_iteration_time_to_first_token.append(model.time_to_first_token)
 
                 report_progress_fn((iteration + 1) / (warmup_iterations + iterations))
 
@@ -110,25 +92,25 @@ class LlamaCppBench(Bench):
                 error_msg = f"Failed to run benchmark: {str(e)}"
                 raise Exception(error_msg)
 
-        self.input_ids_len_list.append(input_tokens)
-        mean_time_to_first_token = statistics.mean(iteration_time_to_first_token)
+        self.input_ids_len_list.append(model.prompt_tokens)
+        mean_time_to_first_token = statistics.mean(per_iteration_time_to_first_token)
         self.mean_time_to_first_token_list.append(mean_time_to_first_token)
         self.prefill_tokens_per_second_list.append(
-            input_tokens / mean_time_to_first_token
+            model.prompt_tokens / mean_time_to_first_token
         )
         self.token_generation_tokens_per_second_list.append(
-            statistics.mean(iteration_tokens_per_second)
+            statistics.mean(per_iteration_tokens_per_second)
         )
         try:
             self.std_dev_time_to_first_token_list.append(
-                statistics.stdev(iteration_time_to_first_token)
+                statistics.stdev(per_iteration_time_to_first_token)
             )
         except StatisticsError:
             # Less than 2 measurements
             self.std_dev_time_to_first_token_list.append(None)
         try:
             self.std_dev_token_generation_tokens_per_second_list.append(
-                statistics.stdev(iteration_tokens_per_second)
+                statistics.stdev(per_iteration_tokens_per_second)
             )
         except StatisticsError:
             # Less than 2 measurements

lemonade_sdk-8.1.0/src/lemonade/tools/llamacpp/load.py

@@ -0,0 +1,185 @@
+import argparse
+import os
+import lemonade.common.printing as printing
+import lemonade.common.status as status
+from lemonade.state import State
+from lemonade.tools import FirstTool
+from lemonade.cache import Keys
+
+
+class LoadLlamaCpp(FirstTool):
+    unique_name = "llamacpp-load"
+
+    def __init__(self):
+        super().__init__(monitor_message="Loading llama.cpp model")
+
+        self.status_stats = [
+            Keys.DEVICE,
+        ]
+
+    @staticmethod
+    def parser(add_help: bool = True) -> argparse.ArgumentParser:
+        parser = __class__.helpful_parser(
+            short_description="Wrap llama.cpp models with an API",
+            add_help=add_help,
+        )
+
+        parser.add_argument(
+            "-d",
+            "--device",
+            choices=["cpu", "igpu"],
+            default="igpu",
+            help="Which device to load the model on to (default: igpu)",
+        )
+
+        default_threads = -1
+        parser.add_argument(
+            "--threads",
+            required=False,
+            type=int,
+            default=default_threads,
+            help=f"Number of threads to use during generation (default: {default_threads})",
+        )
+
+        context_size = 4096
+        parser.add_argument(
+            "--context-size",
+            required=False,
+            type=int,
+            default=context_size,
+            help=f"Size of the prompt context (default: {context_size}. 0 = loaded from model)",
+        )
+
+        output_tokens = 512
+        parser.add_argument(
+            "--output-tokens",
+            required=False,
+            type=int,
+            default=output_tokens,
+            help=f"Maximum number of output tokens to generate (default: {output_tokens})",
+        )
+
+        parser.add_argument(
+            "--reasoning",
+            action="store_true",
+            help="Set this flag to indicate the model is a reasoning model",
+        )
+
+        return parser
+
+    def run(
+        self,
+        state: State,
+        input: str = "",
+        device: str = "igpu",
+        context_size: int = 512,
+        threads: int = 1,
+        output_tokens: int = 512,
+        reasoning: bool = False,
+    ) -> State:
+        """
+        Load a llama.cpp model
+        """
+
+        from lemonade.common.network import is_offline
+        from lemonade.tools.llamacpp.utils import (
+            install_llamacpp,
+            get_llama_cli_exe_path,
+            get_llama_installed_version,
+            parse_checkpoint,
+            download_gguf,
+            get_local_checkpoint_path,
+            LlamaCppTokenizerAdapter,
+            LlamaCppAdapter,
+        )
+
+        # Validate and install llama.cpp, if needed
+        install_llamacpp()
+
+        # Check if input is a local folder containing a .GGUF model
+        if os.path.isdir(input):
+            # input is a local folder
+            local_model_folder = os.path.abspath(input)
+            checkpoint = "local_model"
+            state.checkpoint = checkpoint
+            state.save_stat(Keys.CHECKPOINT, checkpoint)
+            state.save_stat(Keys.LOCAL_MODEL_FOLDER, local_model_folder)
+
+            # See if there is a file ending in ".gguf" in this folder
+            dir = os.listdir(input)
+            gguf_files = [filename for filename in dir if filename.endswith(".gguf")]
+            if len(gguf_files) == 0:
+                raise ValueError(
+                    f"The folder {input} does not contain a GGUF model file."
+                )
+            model_to_use = gguf_files[0]
+            full_model_path = os.path.join(local_model_folder, model_to_use)
+
+        else:
+            # Input is a model checkpoint
+            checkpoint = input
+            state.checkpoint = checkpoint
+            state.save_stat(Keys.CHECKPOINT, checkpoint)
+
+            # Make sure that a variant is provided for the GGUF model
+            base_checkpoint, variant = parse_checkpoint(checkpoint)
+            if variant is None:
+                raise ValueError(
+                    "You are required to provide a 'variant' when "
+                    "selecting a GGUF model. The variant is provided "
+                    "as CHECKPOINT:VARIANT. For example: "
+                    "Qwen/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_0 or "
+                    "Qwen/Qwen2.5-Coder-3B-Instruct-GGUF:qwen2.5-coder-3b-instruct-q4_0.gguf"
+                )
+
+            # Auto-detect offline status
+            offline = is_offline()
+            if offline:
+                printing.log_warning(
+                    "Network connectivity to huggingface.co not detected. Running in offline mode."
+                )
+                full_model_path, model_to_use = get_local_checkpoint_path(
+                    base_checkpoint, variant
+                )
+                if not full_model_path:
+                    raise ValueError(
+                        f"Model {checkpoint} is not available locally."
+                        f"Cannot download in offline mode."
+                    )
+
+            else:
+
+                snapshot_files = download_gguf(checkpoint)
+                full_model_path = snapshot_files["variant"]
+                model_to_use = os.path.basename(full_model_path)
+
+        llama_cli_exe_path = get_llama_cli_exe_path()
+        printing.log_info(f"Using llama_cli for GGUF model: {llama_cli_exe_path}")
+
+        # Get the directory containing the executable for shared libraries
+        lib_dir = os.path.dirname(llama_cli_exe_path)
+
+        # Pass the model and inputs into state
+        state.model = LlamaCppAdapter(
+            model=full_model_path,
+            device=device,
+            output_tokens=output_tokens,
+            context_size=context_size,
+            threads=threads,
+            executable=llama_cli_exe_path,
+            reasoning=reasoning,
+            lib_dir=lib_dir,
+        )
+        state.tokenizer = LlamaCppTokenizerAdapter()
+        state.device = device
+
+        # Save initial stats
+        state.save_stat(Keys.DEVICE, device)
+        state.save_stat(Keys.LLAMA_CLI_VERSION_INFO, get_llama_installed_version())
+
+        status.add_to_state(state=state, name=input, model=model_to_use)
+        return state
+
+
+# This file was originally licensed under Apache 2.0. It has been modified.
+# Modifications Copyright (c) 2025 AMD