lemonade-sdk 7.0.1__tar.gz → 7.0.3__tar.gz

{lemonade_sdk-7.0.1 → lemonade_sdk-7.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lemonade-sdk
-Version: 7.0.1
+Version: 7.0.3
 Summary: Lemonade SDK: Your LLM Aide for Validation and Deployment
 Author-email: lemonade@amd.com
 Requires-Python: >=3.10, <3.12

{lemonade_sdk-7.0.1 → lemonade_sdk-7.0.3}/setup.py

@@ -107,7 +107,7 @@ setup(
     include_package_data=True,
     package_data={
         "lemonade_server": ["server_models.json"],
-        "lemonade": ["tools/server/static/styles.css"],
+        "lemonade": ["tools/server/static/*"],
     },
 )
 

{lemonade_sdk-7.0.1 → lemonade_sdk-7.0.3}/src/lemonade/cli.py

@@ -19,6 +19,7 @@ import lemonade.cache as cache
 from lemonade.tools.mmlu import AccuracyMMLU
 from lemonade.tools.humaneval import AccuracyHumaneval
 from lemonade.tools.perplexity import AccuracyPerplexity
+from lemonade.tools.accuracy import LMEvalHarness
 from lemonade.tools.prompt import LLMPrompt
 from lemonade.tools.quark.quark_load import QuarkLoad
 from lemonade.tools.quark.quark_quantize import QuarkQuantize
@@ -36,6 +37,7 @@ def main():
         AccuracyMMLU,
         AccuracyHumaneval,
         AccuracyPerplexity,
+        LMEvalHarness,
         LLMPrompt,
         HuggingfaceBench,
         OgaBench,

lemonade_sdk-7.0.3/src/lemonade/tools/accuracy.py

@@ -0,0 +1,335 @@
+import argparse
+import json
+import os
+import socket
+import subprocess
+import sys
+import time
+from typing import Optional
+
+import requests
+
+from lemonade.state import State
+from lemonade.tools import Tool
+import lemonade.common.printing as printing
+import lemonade.common.build as build
+
+from lemonade.tools.server.thread_utils import ServerRunner
+
+
+def is_port_in_use(port, host="localhost"):
+    """
+    Check if a port is in use
+    """
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        return s.connect_ex((host, port)) == 0
+
+
+class LMEvalHarness(Tool):
+    """
+    Tool for evaluating LLMs using lm-eval-harness on industry standard benchmarks
+    like MMLU, GSM8k, and more. See docs/lemonade/lm_eval.md for more details.
+    """
+
+    unique_name = "lm-eval-harness"
+
+    def __init__(self):
+        super().__init__(
+            monitor_message="Evaluate model accuracy using ElutherAI's lm-eval-harness"
+        )
+        self.status_stats = []
+        self.server_runner = None
+
+    @staticmethod
+    def parser(add_help: bool = True) -> argparse.ArgumentParser:
+        parser = __class__.helpful_parser(
+            short_description="Evaluate model using lm-eval-harness",
+            add_help=add_help,
+        )
+
+        parser.add_argument(
+            "--task",
+            type=str,
+            required=True,
+            help="Task(s) to evaluate on (e.g., gsm8k, mmlu)",
+        )
+
+        parser.add_argument(
+            "--server-port", type=int, default=8000, help="Port to use for the server"
+        )
+
+        parser.add_argument(
+            "--num-fewshot",
+            type=int,
+            default=0,
+            help="Number of examples in few-shot prompts",
+        )
+
+        parser.add_argument(
+            "--limit",
+            type=int,
+            default=None,
+            help="Limit the number of examples per task",
+        )
+
+        parser.add_argument(
+            "--log-samples",
+            action="store_true",
+            help="Log samples for each task to log file",
+        )
+
+        parser.add_argument(
+            "--output-path",
+            type=str,
+            default=None,
+            help="Path to save evaluation results",
+        )
+
+        return parser
+
+    def _process_results(self, results_dir, state):
+        """Process evaluation results and save to state stats"""
+        if not os.path.exists(results_dir) or not os.path.isdir(results_dir):
+            printing.log_warning(f"Results directory not found at {results_dir}")
+            return
+
+        model_dirs = [
+            d
+            for d in os.listdir(results_dir)
+            if os.path.isdir(os.path.join(results_dir, d))
+        ]
+
+        if not model_dirs:
+            printing.log_warning(f"No model directories found in {results_dir}")
+            return
+
+        model_dir = os.path.join(results_dir, model_dirs[0])
+        printing.log_info(f"Found model directory: {model_dir}")
+
+        # Find the results JSON file with timestamp
+        results_files = [
+            f
+            for f in os.listdir(model_dir)
+            if f.startswith("results_") and f.endswith(".json")
+        ]
+
+        if not results_files:
+            printing.log_warning(f"No results files found in {model_dir}")
+            return
+
+        # Sort by timestamp
+        results_files.sort(reverse=True)
+        results_file_path = os.path.join(model_dir, results_files[0])
+        printing.log_info(f"Processing results from {results_file_path}")
+
+        # Read and process results
+        try:
+            with open(results_file_path, "r", encoding="utf-8") as f:
+                results = json.load(f)
+
+            # Extract and display metrics
+            if "results" in results:
+                for task_name, metrics in results["results"].items():
+                    printing.log_info(f"Results for {task_name}:")
+
+                    for metric, value in metrics.items():
+                        if isinstance(value, (int, float)) and not metric.startswith(
+                            "alias"
+                        ):
+                            # Format metric name for stats
+                            clean_metric = metric.replace(",", "_")
+                            stat_name = f"lm_eval_{task_name}_{clean_metric}"
+
+                            # Save to state stats as percentage
+                            state.save_stat(stat_name, float(value) * 100)
+                            state.save_stat(f"{stat_name}_units", "%")
+                            self.status_stats.append(stat_name)
+
+                            printing.log_info(
+                                f"  {metric}: {value:.4f} ({value*100:.2f}%)"
+                            )
+
+                # Save summary metrics if available
+                avg_metrics = {}
+                if "higher_is_better" in results:
+                    for metric_type in results["higher_is_better"].values():
+                        for metric in metric_type.keys():
+                            if metric not in avg_metrics:
+                                avg_metrics[metric] = []
+
+                for task_metrics in results["results"].values():
+                    for metric, value in task_metrics.items():
+                        if isinstance(value, (int, float)) and not metric.startswith(
+                            "alias"
+                        ):
+                            base_metric = metric.split(",")[0]
+                            if base_metric in avg_metrics:
+                                avg_metrics[base_metric].append(value)
+
+                # Calculate and save averages
+                for metric, values in avg_metrics.items():
+                    if values:
+                        avg_value = sum(values) / len(values)
+                        stat_name = f"lm_eval_average_{metric}"
+                        state.save_stat(stat_name, float(avg_value) * 100)
+                        state.save_stat(f"{stat_name}_units", "%")
+                        self.status_stats.append(stat_name)
+                        printing.log_info(
+                            f"Average {metric}: {avg_value:.4f} ({avg_value*100:.2f}%)"
+                        )
+
+        except (IOError, json.JSONDecodeError) as e:
+            printing.log_error(f"Error processing results: {e}")
+
+    def run(
+        self,
+        state: State,
+        task: str,
+        server_port: int = 8000,
+        server_host: str = "localhost",
+        num_fewshot: int = 0,
+        limit: Optional[int] = None,
+        log_samples: bool = False,
+        output_path: Optional[str] = None,
+    ) -> State:
+
+        model = state.model
+        tokenizer = state.tokenizer
+
+        if model is None or tokenizer is None:
+            raise ValueError(
+                "Model and tokenizer must be loaded in state before running lm-eval-harness"
+            )
+
+        # Set up output path
+        if output_path is None:
+            output_path = os.path.join(
+                build.output_dir(state.cache_dir, state.build_name), "lm_eval_results"
+            )
+
+        os.makedirs(output_path, exist_ok=True)
+
+        # Check if port is already in use
+        if is_port_in_use(server_port, server_host):
+            error_msg = (
+                f"Port {server_port} is already in use. "
+                "Please close all applications using this port and try again."
+            )
+            printing.log_error(error_msg)
+            raise RuntimeError(error_msg)
+
+        # Retroactively determine recipe based on model type to select correct iterator
+        # The model is already loaded in server, so we only need recipe for iterator selection
+        checkpoint = getattr(state, "checkpoint", "unknown")
+        if "OrtGenaiModel" in str(type(model)):
+            recipe = "oga-"
+        else:
+            recipe = "unknown"
+
+        # Start the server thread
+        self.server_runner = ServerRunner(
+            model=model,
+            tokenizer=tokenizer,
+            checkpoint=checkpoint,
+            recipe=recipe,
+            host=server_host,
+            port=server_port,
+        )
+        self.server_runner.start()
+
+        # Wait for server initialization
+        printing.log_info("Waiting for server initialization...")
+
+        # Wait for server to start and be responsive
+        server_url = f"http://{server_host}:{server_port}"
+        max_retries = 30
+        retry_delay = 1
+
+        printing.log_info(f"Checking if server is available at {server_url}...")
+        for i in range(max_retries):
+            try:
+                response = requests.get(f"{server_url}/api/v0/health", timeout=2)
+                if response.status_code == 200:
+                    printing.log_info(f"Server is ready after {i+1} attempts")
+                    break
+            except requests.exceptions.RequestException:
+                if i < max_retries - 1:
+                    time.sleep(retry_delay)
+                else:
+                    printing.log_error(
+                        f"Server did not start after {max_retries} attempts"
+                    )
+                    raise RuntimeError("Failed to start the server")
+
+        # Build API URL
+        results_file = os.path.join(output_path, f"{task}_results")
+
+        printing.log_info(f"Running lm-eval-harness on {task}...")
+
+        # Build lm-eval-harness command
+        cmd = [
+            "lm_eval",
+            "--model",
+            "local-completions",
+            "--tasks",
+            task,
+            "--model_args",
+            (
+                f"model={checkpoint},"
+                f"base_url={server_url}/api/v0/completions,"
+                f"num_concurrent=1,"
+                f"max_retries=5,"
+                f"retry_timeout=10,"
+                f"tokenized_requests=False"
+            ),
+            "--num_fewshot",
+            str(num_fewshot),
+            "--output_path",
+            results_file,
+        ]
+
+        if limit is not None:
+            cmd.extend(["--limit", str(limit)])
+
+        if log_samples:
+            cmd.extend(["--log_samples"])
+
+        try:
+            # On Windows, set UTF-8 mode to handle Unicode output
+            env = os.environ.copy()
+            if sys.platform == "win32":
+                env["PYTHONIOENCODING"] = "utf-8"
+
+            # Execute lm-eval-harness command
+            result = subprocess.run(
+                cmd, check=True, text=True, capture_output=True, env=env
+            )
+
+            # Log relevant output and skip any parts that might cause encoding issues
+            try:
+                printing.log_info(result.stdout)
+            except UnicodeEncodeError:
+                printing.log_info(
+                    "Results obtained successfully but couldn't display due to encoding issues"
+                )
+
+            # Process results from the correct location
+            results_dir = os.path.join(output_path, f"{task}_results")
+            self._process_results(results_dir, state)
+
+        except subprocess.CalledProcessError as e:
+            printing.log_error(f"Error running lm-eval-harness: {e}")
+            printing.log_error(f"stderr: {e.stderr}")
+        except (IOError, ValueError, requests.RequestException) as e:
+            printing.log_error(f"Error: {e}")
+        finally:
+            # Shut down server
+            if self.server_runner and self.server_runner.is_alive():
+                printing.log_info("Shutting down server runner...")
+                self.server_runner.shutdown()
+
+            # Make sure we don't have any lingering references to state's model/tokenizer
+            # that could prevent garbage collection
+            self.server_runner = None
+
+        return state

{lemonade_sdk-7.0.1 → lemonade_sdk-7.0.3}/src/lemonade/tools/huggingface_load.py

@@ -326,6 +326,7 @@ class HuggingfaceAdapter(ModelAdapter):
     def generate(
         self,
         input_ids,
+        random_seed=1,
         **kwargs,
     ):
 
@@ -346,6 +347,11 @@ class HuggingfaceAdapter(ModelAdapter):
             **kwargs,
         }
 
+        if random_seed is None:
+            torch.random.seed()
+        else:
+            torch.random.manual_seed(random_seed)
+
         with torch.no_grad(), torch.inference_mode():
             outputs = self.model.generate(input_ids=input_ids, **generation_kwargs)
 

{lemonade_sdk-7.0.1 → lemonade_sdk-7.0.3}/src/lemonade/tools/ort_genai/oga.py

@@ -139,6 +139,7 @@ class OrtGenaiModel(ModelAdapter):
         pad_token_id=None,
         stopping_criteria=None,
         max_length=None,
+        random_seed=1,
     ):
         params = og.GeneratorParams(self.model)
 
@@ -179,6 +180,9 @@ class OrtGenaiModel(ModelAdapter):
         if use_oga_pre_6_api:
             params.input_ids = input_ids
 
+        if random_seed is None:
+            random_seed = -1  # In og.Generator, -1 = seed with random device
+
         if self.config and "search" in self.config:
             search_config = self.config["search"]
             params.set_search_options(
@@ -196,10 +200,7 @@ class OrtGenaiModel(ModelAdapter):
                 past_present_share_buffer=search_config.get(
                     "past_present_share_buffer", True
                 ),
-                # Make sure that results do not vary across laptops
-                # by default, random_seed=-1 causes different laptops to give
-                # different results
-                random_seed=1,
+                random_seed=random_seed,
                 # Not currently supported by OGA
                 # diversity_penalty=search_config.get('diversity_penalty', 0.0),
                 # no_repeat_ngram_size=search_config.get('no_repeat_ngram_size', 0),
@@ -212,6 +213,7 @@ class OrtGenaiModel(ModelAdapter):
                 temperature=temperature,
                 max_length=max_length_to_use,
                 min_length=min_length,
+                random_seed=random_seed,
             )
         params.try_graph_capture_with_max_batch_size(1)
 

{lemonade_sdk-7.0.1 → lemonade_sdk-7.0.3}/src/lemonade/tools/prompt.py

@@ -15,6 +15,7 @@ DEFAULT_GENERATE_PARAMS = {
     "temperature": 0.7,
 }
 
+DEFAULT_RANDOM_SEED = 1
 DEFAULT_MAX_NEW_TOKENS = 512
 DEFAULT_N_TRIALS = 1
 
@@ -108,6 +109,19 @@ class LLMPrompt(Tool):
             f"(useful for testing, default is {DEFAULT_N_TRIALS})",
         )
 
+        parser.add_argument(
+            "--random-seed",
+            "-r",
+            default=str(DEFAULT_RANDOM_SEED),
+            help="Positive integer seed for random number generator used in "
+            "sampling tokens "
+            f"(default is {DEFAULT_RANDOM_SEED}). If the number of trials is "
+            "greater than one, then the seed is incremented by one for each "
+            "trial. Set to `None` for random, non-repeatable results.  This "
+            "random seed behavior only applies to models loaded with "
+            "`oga-load` or `huggingface-load`.",
+        )
+
         return parser
 
     def parse(self, state: State, args, known_only=True) -> argparse.Namespace:
@@ -123,6 +137,11 @@ class LLMPrompt(Tool):
             with open(parsed_args.prompt, "r", encoding="utf-8") as f:
                 parsed_args.prompt = f.read()
 
+        if parsed_args.random_seed == "None":
+            parsed_args.random_seed = None
+        else:
+            parsed_args.random_seed = int(parsed_args.random_seed)
+
         return parsed_args
 
     def run(
@@ -132,6 +151,7 @@ class LLMPrompt(Tool):
         max_new_tokens: int = DEFAULT_MAX_NEW_TOKENS,
         n_trials: int = DEFAULT_N_TRIALS,
         template: bool = False,
+        random_seed: int = DEFAULT_RANDOM_SEED,
     ) -> State:
 
         model: ModelAdapter = state.model
@@ -170,9 +190,16 @@ class LLMPrompt(Tool):
 
             # Get the response from the LLM, which may include the prompt in it
             response = model.generate(
-                input_ids, max_new_tokens=max_new_tokens, **DEFAULT_GENERATE_PARAMS
+                input_ids,
+                max_new_tokens=max_new_tokens,
+                random_seed=random_seed,
+                **DEFAULT_GENERATE_PARAMS,
             )
 
+            # Increment random seed if not none
+            if random_seed is not None:
+                random_seed += 1
+
             # Flatten the input and response
             input_ids_array = (
                 input_ids if isinstance(input_ids, (list, str)) else input_ids[0]

lemonade_sdk-7.0.3/src/lemonade/tools/server/instructions.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+import json
+from fastapi.responses import HTMLResponse
+from lemonade_server.model_manager import ModelManager
+
+
+def get_instructions_html(port=8000):
+    """
+    Show instructions on how to use the server.
+    """
+    # Load server models from JSON
+    server_models_path = (
+        Path(__file__).parent.parent.parent.parent
+        / "lemonade_server"
+        / "server_models.json"
+    )
+    with open(server_models_path, "r", encoding="utf-8") as f:
+        server_models = json.load(f)
+
+    # Use shared filter function from model_manager.py
+    filtered_models = ModelManager().filter_models_by_backend(server_models)
+
+    # Pass filtered server_models to JS
+    server_models_js = (
+        f"<script>window.SERVER_MODELS = {json.dumps(filtered_models)};</script>"
+    )
+
+    # Load HTML template
+    template_path = Path(__file__).parent / "static" / "instructions.html"
+    with open(template_path, "r", encoding="utf-8") as f:
+        html_template = f.read()
+
+    # Replace template variables
+    html_content = html_template.replace("{{SERVER_PORT}}", str(port))
+    html_content = html_content.replace("{{SERVER_MODELS_JS}}", server_models_js)
+
+    return HTMLResponse(content=html_content)

{lemonade_sdk-7.0.1 → lemonade_sdk-7.0.3}/src/lemonade/tools/server/llamacpp.py

@@ -14,11 +14,11 @@ from fastapi.responses import StreamingResponse
 
 from openai import OpenAI
 
+from lemonade_server.pydantic_models import ChatCompletionRequest
 from lemonade_server.model_manager import ModelManager
-from lemonade.tools.server.pydantic_models import ChatCompletionRequest
+from lemonade.tools.server.port_utils import find_free_port
 
 LLAMA_VERSION = "b5543"
-LLAMA_SERVER_PORT = "8081"
 
 LLAMA_SERVER_EXE_DIR = os.path.join(
     os.path.dirname(sys.executable),
@@ -43,6 +43,23 @@ class LlamaTelemetry:
         self.tokens_per_second = None
         self.prompt_eval_time = None
         self.eval_time = None
+        self.port = None
+
+    def choose_port(self):
+        """
+        Users probably don't care what port we start llama-server on, so let's
+        search for an empty port
+        """
+
+        self.port = find_free_port()
+
+        if self.port is None:
+            msg = "Failed to find an empty port to start llama-server on"
+            logging.error(msg)
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=msg,
+            )
 
     def parse_telemetry_line(self, line: str):
         """
@@ -128,10 +145,12 @@ def _log_subprocess_output(
                 break
 
 
-def _wait_for_load(llama_server_process: subprocess.Popen, fail_message: str):
+def _wait_for_load(
+    llama_server_process: subprocess.Popen, port: int, fail_message: str
+):
     status_code = None
     while not llama_server_process.poll() and status_code != 200:
-        health_url = f"http://localhost:{LLAMA_SERVER_PORT}/health"
+        health_url = f"http://localhost:{port}/health"
         try:
             health_response = requests.get(health_url)
         except requests.exceptions.ConnectionError:
@@ -146,19 +165,25 @@ def _wait_for_load(llama_server_process: subprocess.Popen, fail_message: str):
 
 
 def _launch_llama_subprocess(
-    model_path: str, use_gpu: bool, telemetry: LlamaTelemetry
+    snapshot_files: dict, use_gpu: bool, telemetry: LlamaTelemetry
 ) -> subprocess.Popen:
     """
     Launch llama server subprocess with GPU or CPU configuration
     """
 
-    base_command = [
-        LLAMA_SERVER_EXE_PATH,
-        "-m",
-        model_path,
-        "--port",
-        LLAMA_SERVER_PORT,
-    ]
+    # Build the base command
+    base_command = [LLAMA_SERVER_EXE_PATH, "-m", snapshot_files["variant"]]
+    if "mmproj" in snapshot_files:
+        base_command.extend(["--mmproj", snapshot_files["mmproj"]])
+        if not use_gpu:
+            base_command.extend(["--no-mmproj-offload"])
+
+    # Find a port, and save it in the telemetry object for future reference
+    # by other functions
+    telemetry.choose_port()
+
+    # Add port and jinja to enable tool use
+    base_command.extend(["--port", str(telemetry.port), "--jinja"])
 
     # Configure GPU layers: 99 for GPU, 0 for CPU-only
     ngl_value = "99" if use_gpu else "0"
@@ -180,7 +205,7 @@ def _launch_llama_subprocess(
     return process
 
 
-def server_load(checkpoint: str, model_reference: str, telemetry: LlamaTelemetry):
+def server_load(model_config: dict, model_reference: str, telemetry: LlamaTelemetry):
     # Download llama.cpp server if it isn't already available
     if not os.path.exists(LLAMA_SERVER_EXE_DIR):
         # Download llama.cpp server zip
@@ -212,33 +237,34 @@ def server_load(checkpoint: str, model_reference: str, telemetry: LlamaTelemetry
         logging.info("Cleaned up zip file")
 
     # Download the gguf to the hugging face cache
-    snapshot_path = ModelManager().download_gguf(checkpoint)
-    model_path = os.path.join(snapshot_path, os.listdir(snapshot_path)[0])
-    logging.debug(f"GGUF file path: {model_path}")
+    snapshot_files = ModelManager().download_gguf(model_config)
+    logging.debug(f"GGUF file paths: {snapshot_files}")
 
     # Start the llama-serve.exe process
     logging.debug(f"Using llama_server for GGUF model: {LLAMA_SERVER_EXE_PATH}")
 
     # Attempt loading on GPU first
     llama_server_process = _launch_llama_subprocess(
-        model_path, use_gpu=True, telemetry=telemetry
+        snapshot_files, use_gpu=True, telemetry=telemetry
     )
 
     # Check the /health endpoint until GPU server is ready
     _wait_for_load(
         llama_server_process,
+        telemetry.port,
         f"Loading {model_reference} on GPU didn't work, re-attempting on CPU",
     )
 
     # If loading on GPU failed, try loading on CPU
     if llama_server_process.poll():
         llama_server_process = _launch_llama_subprocess(
-            model_path, use_gpu=False, telemetry=telemetry
+            snapshot_files, use_gpu=False, telemetry=telemetry
         )
 
         # Check the /health endpoint until CPU server is ready
         _wait_for_load(
             llama_server_process,
+            telemetry.port,
             f"Loading {model_reference} on CPU didn't work",
         )
 
@@ -254,7 +280,7 @@ def server_load(checkpoint: str, model_reference: str, telemetry: LlamaTelemetry
 def chat_completion(
     chat_completion_request: ChatCompletionRequest, telemetry: LlamaTelemetry
 ):
-    base_url = f"http://127.0.0.1:{LLAMA_SERVER_PORT}/v1"
+    base_url = f"http://127.0.0.1:{telemetry.port}/v1"
     client = OpenAI(
         base_url=base_url,
         api_key="lemonade",