lemonade-sdk 8.0.2__py3-none-any.whl → 8.0.4__py3-none-any.whl

lemonade/cli.py

@@ -90,9 +90,9 @@ https://github.com/lemonade-sdk/lemonade/blob/main/docs/README.md""",
     )
 
     profiler_instances = [
-        profiler(global_args[profiler.unique_name])
+        profiler(global_args[profiler.unique_name.replace("-", "_")])
         for profiler in profilers
-        if global_args.get(profiler.unique_name, None) is not None
+        if global_args.get(profiler.unique_name.replace("-", "_"), None) is not None
     ]
 
     if len(evaluation_tools) > 0:

lemonade/profilers/profiler.py

@@ -48,7 +48,10 @@ class Profiler(abc.ABC):
         This method is called so that the profiler can create its output files.
         The state is passed so that build info can be gathered and stats can be written.
         The timestamp can be used for filename in current working directory.
-        The start times contain a list of tools and start times.
+        The start times parameter is a dict with the keys being the tools names and
+        the values being the time the tool started.  There is an initial "warmup" key
+        that has a start time before the first tool and a "cool down" key that contains the
+        time when the last tool ended.
         """
 
 

lemonade/tools/humaneval.py

@@ -24,7 +24,7 @@ class AccuracyHumaneval(Tool):
     - pass@10: Percentage of problems solved within 10 generation attempts
     - pass@100: Percentage of problems solved within 100 generation attempts
 
-    See docs/lemonade/humaneval_accuracy.md for more details
+    See docs/dev_cli/humaneval_accuracy.md for more details
     """
 
     unique_name = "accuracy-humaneval"

lemonade/tools/mmlu.py

@@ -27,7 +27,7 @@ def min_handle_none(*args: int):
 
 class AccuracyMMLU(Tool):
     """
-    See docs/lemonade/mmlu_accuracy.md for more details
+    See docs/dev_cli/mmlu_accuracy.md for more details
     """
 
     unique_name = "accuracy-mmlu"

lemonade/tools/oga/load.py

@@ -1,12 +1,6 @@
 # onnxruntime_genai is not lint-friendly yet and PyLint can't
 # find any of the class methods
 # pylint: disable=no-member
-#
-# Model builder constraints:
-#   11/10/24 Need transformers <4.45.0 OR onnxruntime-genai 0.5.0 (which must be built from source)
-#   (transformers v4.45 changes the format of the tokenizer.json file which will be supported in
-#   onnxruntime-genai 0.5)
-#
 
 import argparse
 import os
@@ -51,8 +45,8 @@ def import_error_heler(e: Exception):
     """
     raise ImportError(
         f"{e}\n Please install lemonade-sdk with "
-        "one of the llm-oga extras, for example:\n"
-        "pip install lemonade-sdk[llm-oga-cpu]\n"
+        "one of the oga extras, for example:\n"
+        "pip install lemonade-sdk[dev,oga-cpu]\n"
         "See https://lemonade_server.ai/install_options.html for details"
     )
 
@@ -64,7 +58,7 @@ class OgaLoad(FirstTool):
     Input: path to a checkpoint.
         Supported choices for cpu and igpu from HF model repository:
             LLM models on Huggingface supported by model_builder.  See documentation
-            (https://github.com/lemonade-sdk/lemonade/blob/main/docs/ort_genai_igpu.md)
+            (https://github.com/lemonade-sdk/lemonade/blob/main/docs/dev_cli/ort_genai_igpu.md)
             for supported models.
         Supported choices for npu from HF model repository:
             Models on Hugging Face that follow the "amd/**-onnx-ryzen-strix" pattern

lemonade/tools/perplexity.py

@@ -17,7 +17,7 @@ class AccuracyPerplexity(Tool):
 
     Output state produced: None
 
-    See docs/lemonade/perplexity.md for more details.
+    See docs/dev_cli/perplexity.md for more details.
     """
 
     unique_name = "accuracy-perplexity"
@@ -63,7 +63,7 @@ class AccuracyPerplexity(Tool):
             # try-except will allow a few more LLMs to work
             max_length = 2048
         # Set stride to half of the maximum input length for overlapping window processing
-        # Refer to docs/perplexity.md for more information on sliding window
+        # Refer to docs/dev_cli/perplexity.md for more information on sliding window
         stride = max_length // 2
         # Determine the total sequence length of the tokenized input
         seq_len = encodings.input_ids.size(1)

lemonade/tools/prompt.py

@@ -176,12 +176,21 @@ class LLMPrompt(Tool):
 
         input_ids = tokenizer(prompt, return_tensors="pt").input_ids
         if isinstance(input_ids, (list, str)):
-            # OGA models return a list of tokens
+            # OGA models return a list of tokens (older versions)
             # Our llama.cpp adapter returns a string
             len_tokens_in = len(input_ids)
-        else:
+        elif hasattr(input_ids, "shape"):
             # HF models return a 2-D tensor
-            len_tokens_in = input_ids.shape[1]
+            # OGA models with newer versions may return numpy arrays
+            if len(input_ids.shape) == 1:
+                # 1-D array from newer OGA versions
+                len_tokens_in = len(input_ids)
+            else:
+                # 2-D tensor from HF models
+                len_tokens_in = input_ids.shape[1]
+        else:
+            # Fallback: try to get length directly
+            len_tokens_in = len(input_ids)
 
         len_tokens_out = []
         response_texts = []
@@ -202,9 +211,15 @@ class LLMPrompt(Tool):
                 random_seed += 1
 
             # Flatten the input and response
-            input_ids_array = (
-                input_ids if isinstance(input_ids, (list, str)) else input_ids[0]
-            )
+            if isinstance(input_ids, (list, str)):
+                input_ids_array = input_ids
+            elif hasattr(input_ids, "shape") and len(input_ids.shape) == 1:
+                # 1-D array from newer OGA versions - already flat
+                input_ids_array = input_ids
+            else:
+                # 2-D tensor from HF models - take first row
+                input_ids_array = input_ids[0]
+
             response_array = response if isinstance(response, str) else response[0]
 
             # Separate the prompt from the response

lemonade/tools/quark/quark_load.py

@@ -18,7 +18,7 @@ class QuarkLoad(Tool):
     Output:
         - state of the loaded model
 
-    See docs/quark.md for more details.
+    See docs/dev_cli/quark.md for more details.
     """
 
     unique_name = "quark-load"

lemonade/tools/quark/quark_quantize.py

@@ -25,7 +25,7 @@ class QuarkQuantize(Tool):
     Output:
         - Modifies `state` with quantized and optionally exported model.
 
-    See docs/quark.md for more details.
+    See docs/dev_cli/quark.md for more details.
     """
 
     unique_name = "quark-quantize"
@@ -94,7 +94,7 @@ class QuarkQuantize(Tool):
             help="Number of samples for calibration.",
         )
 
-        # See docs/quark.md for more details.
+        # See docs/dev_cli/quark.md for more details.
         parser.add_argument(
             "--quant-scheme",
             type=str,

lemonade/tools/report/table.py

@@ -74,6 +74,7 @@ class SimpleStat(TableColumn):
         align="center",
         omit_if_lean=False,
         wrap=None,
+        stat_fn=None,
     ):
         self.column_header = column_header
         self.stat = stat
@@ -81,6 +82,7 @@ class SimpleStat(TableColumn):
         self.align = align
         self.omit_if_lean = omit_if_lean
         self.wrap = wrap or self.default_wrap
+        self.stat_fn = stat_fn
 
     def get_str(self, build_stats, lean=False):
         if lean and self.omit_if_lean:
@@ -88,6 +90,8 @@ class SimpleStat(TableColumn):
         data = build_stats.get(self.stat, None)
         if data is None:
             return ""
+        if self.stat_fn:
+            data = self.stat_fn(data)
         cell_str = "\n".join(
             [_wrap(f"{x:{self.format_str}}", self.wrap) for x in _to_list(data)]
         )
@@ -233,6 +237,47 @@ class AdditionalStat(TableColumn):
         return "\n".join(cell_entry)
 
 
+class DictListStat(TableColumn):
+    """
+    A statistic that is a list of dicts and values from a given list of keys will be
+    pulled out of each dict and placed in the cell
+    """
+
+    def __init__(
+        self,
+        column_header,
+        statistic_name,
+        key_format_list,
+        align="center",
+        omit_if_lean=False,
+        wrap=None,
+    ):
+        self.column_header = column_header
+        self.statistic_name = statistic_name
+        self.key_format_list = key_format_list
+        self.align = align
+        self.omit_if_lean = omit_if_lean
+        self.wrap = wrap or self.default_wrap
+
+    def get_str(self, build_stats, lean=False):
+        if lean and self.omit_if_lean:
+            return None
+        stat = build_stats.get(self.statistic_name, None)
+        if not stat:
+            return ""
+        cell_entry = []
+        for stat_dict in stat:
+            line = [
+                format_str.format(stat_dict[key])
+                for key, format_str in self.key_format_list
+            ]
+            cell_entry.append(" ".join(line))
+        return "\n".join(cell_entry)
+
+    def get_keys(self):
+        return [self.statistic_name]
+
+
 ################################################################################
 # ABSTRACT BASE CLASS FOR DEFINING A TABLE
 ################################################################################
@@ -350,6 +395,28 @@ class Table(ABC):
                     headers.append(column.column_header)
                     col_align += (column.align,)
 
+        # Stat column headers
+        stat_columns = self.table_descriptor.get("stat_columns", [])
+        stat_columns_include = []
+        for column in stat_columns:
+            # Check to see that at least one build has data for the column
+            keep_column = False
+            if not (self.lean and column.omit_if_lean):
+                keys = column.get_keys()
+                for build_stats in self.all_stats:
+                    found = [(key in build_stats) for key in keys]
+                    if any(found):
+                        keep_column = True
+                        headers.append(column.column_header)
+                        col_align += (column.align,)
+                        break
+            stat_columns_include.append(keep_column)
+        stat_columns = [
+            column
+            for column, include in zip(stat_columns, stat_columns_include)
+            if include
+        ]
+
         # Final headers
         last_columns = self.table_descriptor.get("last_columns", [])
         for column in last_columns:
@@ -386,6 +453,12 @@ class Table(ABC):
                     if entry_str is not None:
                         row.append(entry_str)
 
+            # Per stat columns
+            for entry in stat_columns:
+                entry_str = entry.get_str(build_stats, self.lean)
+                if entry_str is not None:
+                    row.append(entry_str)
+
             # Final columns
             for entry in last_columns:
                 entry_str = entry.get_str(build_stats, self.lean)
@@ -514,6 +587,12 @@ class LemonadePerfTable(Table):
                     Keys.STD_DEV_TOKENS_PER_SECOND,
                     ".2f",
                 ),
+                SimpleStat(
+                    _wrap("Total Generated Tokens", 9),
+                    Keys.RESPONSE_TOKENS,
+                    "d",
+                    stat_fn=sum,
+                ),
                 SimpleStat(
                     _wrap("Memory Used (GB)", 8), Keys.MAX_MEMORY_USED_GBYTE, ".3f"
                 ),
@@ -537,6 +616,7 @@ class LemonadePerfTable(Table):
                 )
             ],
         },
+        "stat_columns": [],
         "last_columns": [
             SimpleStat(
                 "System Info",

lemonade/tools/server/llamacpp.py

@@ -16,11 +16,29 @@ from fastapi.responses import StreamingResponse
 
 from openai import OpenAI
 
-from lemonade_server.pydantic_models import ChatCompletionRequest, PullConfig
+from lemonade_server.pydantic_models import (
+    ChatCompletionRequest,
+    PullConfig,
+    EmbeddingsRequest,
+    RerankingRequest,
+)
 from lemonade_server.model_manager import ModelManager
 from lemonade.tools.server.utils.port import find_free_port
 
-LLAMA_VERSION = "b5699"
+LLAMA_VERSION = "b5787"
+
+
+def llamacpp_address(port: int) -> str:
+    """
+    Generate the base URL for the llamacpp server.
+
+    Args:
+        port: The port number the llamacpp server is running on
+
+    Returns:
+        The base URL for the llamacpp server
+    """
+    return f"http://127.0.0.1:{port}/v1"
 
 
 def get_llama_server_paths():
@@ -210,15 +228,20 @@ def _log_subprocess_output(
     """
 
     if process.stdout:
-        for line in iter(process.stdout.readline, ""):
-            if line:
-                line_stripped = line.strip()
-                logging.debug("%s: %s", prefix, line_stripped)
+        try:
+            for line in iter(process.stdout.readline, ""):
+                if line:
+                    line_stripped = line.strip()
+                    logging.debug("%s: %s", prefix, line_stripped)
 
-                telemetry.parse_telemetry_line(line_stripped)
+                    telemetry.parse_telemetry_line(line_stripped)
 
-            if process.poll() is not None:
-                break
+                if process.poll() is not None:
+                    break
+        except UnicodeDecodeError as e:
+            logging.debug("Unicode decode error reading subprocess output: %s", str(e))
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            logging.error("Unexpected error reading subprocess output: %s", str(e))
 
 
 def _wait_for_load(llama_server_process: subprocess.Popen, port: int):
@@ -239,10 +262,24 @@ def _wait_for_load(llama_server_process: subprocess.Popen, port: int):
 
 
 def _launch_llama_subprocess(
-    snapshot_files: dict, use_gpu: bool, telemetry: LlamaTelemetry
+    snapshot_files: dict,
+    use_gpu: bool,
+    telemetry: LlamaTelemetry,
+    supports_embeddings: bool = False,
+    supports_reranking: bool = False,
 ) -> subprocess.Popen:
     """
-    Launch llama server subprocess with GPU or CPU configuration
+    Launch llama server subprocess with appropriate configuration.
+
+    Args:
+        snapshot_files: Dictionary of model files to load
+        use_gpu: Whether to use GPU acceleration
+        telemetry: Telemetry object for tracking performance metrics
+        supports_embeddings: Whether the model supports embeddings
+        supports_reranking: Whether the model supports reranking
+
+    Returns:
+        Subprocess handle for the llama server
     """
 
     # Get the current executable path (handles both Windows and Ubuntu structures)
@@ -266,6 +303,14 @@ def _launch_llama_subprocess(
     # reasoning_content field
     base_command.extend(["--reasoning-format", "none"])
 
+    # Add embeddings support if the model supports it
+    if supports_embeddings:
+        base_command.append("--embeddings")
+
+    # Add reranking support if the model supports it
+    if supports_reranking:
+        base_command.append("--reranking")
+
     # Configure GPU layers: 99 for GPU, 0 for CPU-only
     ngl_value = "99" if use_gpu else "0"
     command = base_command + ["-ngl", ngl_value]
@@ -287,6 +332,8 @@ def _launch_llama_subprocess(
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
         text=True,
+        encoding="utf-8",
+        errors="replace",
         bufsize=1,
         env=env,
     )
@@ -303,7 +350,6 @@ def _launch_llama_subprocess(
 
 
 def server_load(model_config: PullConfig, telemetry: LlamaTelemetry):
-
     # Validate platform support before proceeding
     validate_platform_support()
 
@@ -360,15 +406,26 @@ def server_load(model_config: PullConfig, telemetry: LlamaTelemetry):
         logging.info("Cleaned up zip file")
 
     # Download the gguf to the hugging face cache
-    snapshot_files = ModelManager().download_gguf(model_config)
+    model_manager = ModelManager()
+    snapshot_files = model_manager.download_gguf(model_config)
     logging.debug(f"GGUF file paths: {snapshot_files}")
 
+    # Check if model supports embeddings
+    supported_models = model_manager.supported_models
+    model_info = supported_models.get(model_config.model_name, {})
+    supports_embeddings = "embeddings" in model_info.get("labels", [])
+    supports_reranking = "reranking" in model_info.get("labels", [])
+
     # Start the llama-serve.exe process
     logging.debug(f"Using llama_server for GGUF model: {llama_server_exe_path}")
 
     # Attempt loading on GPU first
     llama_server_process = _launch_llama_subprocess(
-        snapshot_files, use_gpu=True, telemetry=telemetry
+        snapshot_files,
+        use_gpu=True,
+        telemetry=telemetry,
+        supports_embeddings=supports_embeddings,
+        supports_reranking=supports_reranking,
     )
 
     # Check the /health endpoint until GPU server is ready
@@ -383,8 +440,16 @@ def server_load(model_config: PullConfig, telemetry: LlamaTelemetry):
             f"Loading {model_config.model_name} on GPU didn't work, re-attempting on CPU"
         )
 
+        if os.environ.get("LEMONADE_LLAMACPP_NO_FALLBACK"):
+            # Used for testing, when the test should fail if GPU didn't work
+            raise Exception("llamacpp GPU loading failed")
+
         llama_server_process = _launch_llama_subprocess(
-            snapshot_files, use_gpu=False, telemetry=telemetry
+            snapshot_files,
+            use_gpu=False,
+            telemetry=telemetry,
+            supports_embeddings=supports_embeddings,
+            supports_reranking=supports_reranking,
         )
 
         # Check the /health endpoint until CPU server is ready
@@ -405,7 +470,7 @@ def server_load(model_config: PullConfig, telemetry: LlamaTelemetry):
 def chat_completion(
     chat_completion_request: ChatCompletionRequest, telemetry: LlamaTelemetry
 ):
-    base_url = f"http://127.0.0.1:{telemetry.port}/v1"
+    base_url = llamacpp_address(telemetry.port)
     client = OpenAI(
         base_url=base_url,
         api_key="lemonade",
@@ -456,3 +521,70 @@ def chat_completion(
                 status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                 detail=f"Chat completion error: {str(e)}",
             )
+
+
+def embeddings(embeddings_request: EmbeddingsRequest, telemetry: LlamaTelemetry):
+    """
+    Generate embeddings using the llamacpp server.
+
+    Args:
+        embeddings_request: The embeddings request containing input text/tokens
+        telemetry: Telemetry object containing the server port
+
+    Returns:
+        Embeddings response from the llamacpp server
+    """
+    base_url = llamacpp_address(telemetry.port)
+    client = OpenAI(
+        base_url=base_url,
+        api_key="lemonade",
+    )
+
+    # Convert Pydantic model to dict and remove unset/null values
+    request_dict = embeddings_request.model_dump(exclude_unset=True, exclude_none=True)
+
+    try:
+        # Call the embeddings endpoint
+        response = client.embeddings.create(**request_dict)
+        return response
+
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Embeddings error: {str(e)}",
+        )
+
+
+def reranking(reranking_request: RerankingRequest, telemetry: LlamaTelemetry):
+    """
+    Rerank documents based on their relevance to a query using the llamacpp server.
+
+    Args:
+        reranking_request: The reranking request containing query and documents
+        telemetry: Telemetry object containing the server port
+
+    Returns:
+        Reranking response from the llamacpp server containing ranked documents and scores
+    """
+    base_url = llamacpp_address(telemetry.port)
+
+    try:
+        # Convert Pydantic model to dict and exclude unset/null values
+        request_dict = reranking_request.model_dump(
+            exclude_unset=True, exclude_none=True
+        )
+
+        # Call the reranking endpoint directly since it's not supported by the OpenAI API
+        response = requests.post(
+            f"{base_url}/rerank",
+            json=request_dict,
+        )
+        response.raise_for_status()
+        return response.json()
+
+    except Exception as e:
+        logging.error("Error during reranking: %s", str(e))
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Reranking error: {str(e)}",
+        ) from e

lemonade/tools/server/serve.py

@@ -54,6 +54,8 @@ from lemonade_server.pydantic_models import (
     LoadConfig,
     CompletionRequest,
     ChatCompletionRequest,
+    EmbeddingsRequest,
+    RerankingRequest,
     ResponsesRequest,
     PullConfig,
     DeleteConfig,
@@ -231,8 +233,13 @@ class Server(ManagementTool):
 
             # OpenAI-compatible routes
             self.app.post(f"{prefix}/chat/completions")(self.chat_completions)
+            self.app.post(f"{prefix}/embeddings")(self.embeddings)
             self.app.get(f"{prefix}/models")(self.models)
 
+            # JinaAI routes (jina.ai/reranker/)
+            self.app.post(f"{prefix}/reranking")(self.reranking)
+            self.app.post(f"{prefix}/rerank")(self.reranking)
+
     @staticmethod
     def parser(add_help: bool = True) -> argparse.ArgumentParser:
         parser = __class__.helpful_parser(
@@ -796,6 +803,72 @@ class Server(ManagementTool):
                 created=int(time.time()),
             )
 
+    async def embeddings(self, embeddings_request: EmbeddingsRequest):
+        """
+        Generate embeddings for the provided input.
+        """
+        # Initialize load config from embeddings request
+        lc = LoadConfig(model_name=embeddings_request.model)
+
+        # Load the model if it's different from the currently loaded one
+        await self.load_llm(lc)
+
+        if self.llm_loaded.recipe == "llamacpp":
+            try:
+                return llamacpp.embeddings(embeddings_request, self.llama_telemetry)
+            except Exception as e:  # pylint: disable=broad-exception-caught
+                # Check if model has embeddings label
+                model_info = ModelManager().supported_models.get(
+                    self.llm_loaded.model_name, {}
+                )
+                if "embeddings" not in model_info.get("labels", []):
+                    raise HTTPException(
+                        status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                        detail="You tried to generate embeddings for a model that is "
+                        "not labeled as an embeddings model. Please use another model "
+                        "or re-register the current model with the 'embeddings' label.",
+                    ) from e
+                else:
+                    raise e
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=f"Embeddings not supported for recipe: {self.llm_loaded.recipe}",
+            )
+
+    async def reranking(self, reranking_request: RerankingRequest):
+        """
+        Rerank documents based on their relevance to a query using the llamacpp server.
+        """
+        # Initialize load config from reranking request
+        lc = LoadConfig(model_name=reranking_request.model)
+
+        # Load the model if it's different from the currently loaded one
+        await self.load_llm(lc)
+
+        if self.llm_loaded.recipe == "llamacpp":
+            try:
+                return llamacpp.reranking(reranking_request, self.llama_telemetry)
+            except Exception as e:  # pylint: disable=broad-exception-caught
+                # Check if model has reranking label
+                model_info = ModelManager().supported_models.get(
+                    self.llm_loaded.model_name, {}
+                )
+                if "reranking" not in model_info.get("labels", []):
+                    raise HTTPException(
+                        status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                        detail="You tried to use reranking for a model that is "
+                        "not labeled as a reranking model. Please use another model "
+                        "or re-register the current model with the 'reranking' label.",
+                    ) from e
+                else:
+                    raise e
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=f"Reranking not supported for recipe: {self.llm_loaded.recipe}",
+            )
+
     def apply_chat_template(
         self, messages: list[dict], tools: list[dict] | None = None
     ):