lemonade-sdk 7.0.0__py3-none-any.whl

lemonade/tools/humaneval.py

@@ -0,0 +1,258 @@
+import argparse
+import os
+import csv
+from typing import Dict, Optional, Any
+import requests
+from human_eval.data import write_jsonl, read_problems
+from human_eval.evaluation import evaluate_functional_correctness
+
+from lemonade.state import State
+from lemonade.tools import Tool
+import lemonade.common.printing as printing
+import lemonade.common.build as build
+
+
+class AccuracyHumaneval(Tool):
+    """
+    HumanEval accuracy measurement tool.
+
+    This tool evaluates language models on the HumanEval dataset, which consists of
+    Python programming problems. It measures the model's ability to:
+    1. Generate functionally correct code completions
+    2. Pass unit tests for each programming problem
+
+    Metrics:
+    - pass@1: Percentage of problems solved with 1 generation attempt
+    - pass@10: Percentage of problems solved within 10 generation attempts
+    - pass@100: Percentage of problems solved within 100 generation attempts
+
+    See docs/lemonade/humaneval_accuracy.md for more details
+    """
+
+    unique_name = "accuracy-humaneval"
+    DATASET = "https://github.com/openai/human-eval/blob/master/data/HumanEval.jsonl.gz?raw=true"
+    TOTAL_PROBLEMS = 164  # Total number of problems in the HumanEval dataset
+
+    def __init__(self):
+        super().__init__(monitor_message="Measuring accuracy with HumanEval")
+        self.status_stats = []
+        # Enable code evaluation for HumanEval
+        os.environ["HF_ALLOW_CODE_EVAL"] = "1"
+
+    @staticmethod
+    def parser(add_help: bool = True) -> argparse.ArgumentParser:
+        parser = __class__.helpful_parser(
+            short_description="Measure coding accuracy with HumanEval",
+            add_help=add_help,
+        )
+        parser.add_argument(
+            "--k-samples",
+            type=int,
+            default=1,
+            help="Number of completions to generate per prompt for pass@k calculation"
+            " (default: %(default)s)",
+        )
+        parser.add_argument(
+            "--first-n-samples",
+            type=int,
+            default=AccuracyHumaneval.TOTAL_PROBLEMS,
+            help=f"Evaluate only the first N problems from the dataset (default: "
+            f"%(default)s, evaluates all {AccuracyHumaneval.TOTAL_PROBLEMS} problems)",
+        )
+        parser.add_argument(
+            "--timeout",
+            type=float,
+            default=30.0,
+            help="Timeout in seconds for each test case (default: %(default)s)",
+        )
+        parser.add_argument(
+            "--data-dir",
+            type=str,
+            default=None,
+            help="Custom directory for dataset storage (default: %(default)s, "
+            "uses <lemonade_cache_dir>/data/humaneval)",
+        )
+        return parser
+
+    def run(
+        self,
+        state: State,
+        data_dir: Optional[str] = None,
+        k_samples: int = 1,
+        first_n_samples: Optional[int] = TOTAL_PROBLEMS,
+        timeout: float = 30.0,
+    ) -> State:
+        """
+        Run HumanEval evaluation on the model.
+
+        Args:
+            state: Current state containing model and tokenizer
+            data_dir: Optional custom directory for dataset storage
+            k_samples: Number of completions to generate per prompt for pass@k calculation
+            first_n_samples: Number of first N problems to evaluate
+            timeout: Timeout in seconds for each test case
+
+        Returns:
+            Updated state with evaluation results
+        """
+        # Validate required state components
+        if not hasattr(state, "model") or not hasattr(state, "tokenizer"):
+            raise ValueError("State must contain both 'model' and 'tokenizer'")
+
+        # Setup directories
+        data_dir_to_use = data_dir or os.path.join(state.cache_dir, "data", "humaneval")
+        data_path = os.path.join(data_dir_to_use, "HumanEval.jsonl.gz")
+        model_results_dir = os.path.join(
+            build.output_dir(state.cache_dir, state.build_name), "humaneval"
+        )
+        os.makedirs(model_results_dir, exist_ok=True)
+
+        # Download dataset if needed
+        self._download_dataset(data_path)
+
+        # Run evaluation
+        results = self._evaluate_model(
+            state.model,
+            state.tokenizer,
+            data_path,
+            k_samples,
+            timeout,
+            model_results_dir,
+            first_n_samples,
+        )
+
+        # Save metrics
+        self._save_metrics(state, results)
+
+        return state
+
+    def _download_dataset(self, output_path: str) -> None:
+        """Download HumanEval dataset if not already present."""
+        if os.path.exists(output_path):
+            printing.log_info(f"Dataset already exists at: {output_path}")
+            return
+
+        os.makedirs(os.path.dirname(output_path), exist_ok=True)
+        response = requests.get(self.DATASET, stream=True)
+
+        if response.status_code == 200:
+            with open(output_path, "wb") as file:
+                for chunk in response.iter_content(chunk_size=8192):
+                    file.write(chunk)
+            printing.log_info(f"Dataset downloaded successfully to: {output_path}")
+        else:
+            raise RuntimeError(
+                f"Failed to download dataset. Status code: {response.status_code}"
+            )
+
+    def _evaluate_model(
+        self,
+        model: Any,
+        tokenizer: Any,
+        data_path: str,
+        k_samples: int,
+        timeout: float,
+        results_dir: str,
+        first_n_samples: Optional[int] = TOTAL_PROBLEMS,
+    ) -> Dict[str, float]:
+        """
+        Evaluate model on HumanEval dataset.
+
+        Args:
+            model: The language model to evaluate
+            tokenizer: The tokenizer for the model
+            data_path: Path to the HumanEval dataset
+            k_samples: Number of completions per prompt for pass@k calculation
+            timeout: Test case timeout in seconds
+            results_dir: Directory to save results
+            first_n_samples: Number of first N problems to evaluate
+
+        Returns:
+            Dictionary containing evaluation metrics
+        """
+        dataset = read_problems(data_path)
+
+        # Limit to first N problems
+        dataset_keys = list(dataset.keys())[:first_n_samples]
+        ignore_incomplete = True
+
+        samples = []
+
+        # Update Tool progress monitor
+        self.set_percent_progress(0.0)
+        questions_completed = 0
+        number_of_questions = first_n_samples * k_samples
+
+        # Save completions and expected answers
+        csv_path = os.path.join(results_dir, "evaluation_results.csv")
+        with open(
+            csv_path, mode="w", newline="", encoding="utf-8", errors="replace"
+        ) as file:
+            writer = csv.writer(file)
+            writer.writerow(["Prompt", "Completion", "Expected Answer"])
+
+            for task_id in dataset_keys:
+                try:
+                    for _ in range(k_samples):
+                        prompt = dataset[task_id]["prompt"]
+                        expected = dataset[task_id]["canonical_solution"]
+
+                        # Generate completion
+                        input_ids = tokenizer(prompt, return_tensors="pt").input_ids
+                        completion = model.generate(
+                            input_ids,
+                            max_new_tokens=512,
+                            do_sample=False,
+                        )
+                        completion_text = tokenizer.decode(
+                            completion[0], skip_special_tokens=True
+                        )
+
+                        # Save results
+                        samples.append(
+                            {"task_id": task_id, "completion": completion_text}
+                        )
+                        writer.writerow([prompt, completion_text, expected])
+
+                        # Update progress monitor after completing all samples for a question
+                        questions_completed = questions_completed + 1
+                        percent_completed = (
+                            questions_completed / number_of_questions * 100
+                        )
+                        self.set_percent_progress(percent_completed)
+
+                # pylint: disable=W0718
+                except Exception as e:
+                    printing.log_info(f"Error processing task {task_id}: {str(e)}")
+                    continue
+
+        # Save predictions and evaluate
+        pred_path = os.path.join(results_dir, "humaneval_predictions.jsonl")
+        write_jsonl(pred_path, samples)
+        printing.log_info(f"Results saved in: {results_dir}")
+
+        # Run functional correctness evaluation
+        k_values = [k_samples]
+        results = evaluate_functional_correctness(
+            pred_path,
+            k_values,
+            n_workers=1,
+            timeout=timeout,
+            problem_file=data_path,
+            ignore_incomplete=ignore_incomplete,
+        )
+        return results
+
+    def _save_metrics(self, state: State, results: Dict[str, float]) -> None:
+        """Save evaluation metrics to state."""
+        for metric, value in results.items():
+            metric_name = f"humaneval_{metric}"
+            state.save_stat(
+                metric_name, float(value) * 100 if value is not None else None
+            )
+            state.save_stat(f"{metric_name}_units", "%")
+            self.status_stats.append(metric_name)
+
+
+# This file was originally licensed under Apache 2.0. It has been modified.
+# Modifications Copyright (c) 2025 AMD

lemonade/tools/llamacpp.py

@@ -0,0 +1,261 @@
+import argparse
+import os
+from typing import Optional
+import subprocess
+from lemonade.state import State
+import lemonade.common.status as status
+from lemonade.tools import FirstTool
+from lemonade.tools.adapter import PassthroughTokenizer, ModelAdapter
+from lemonade.cache import Keys
+from lemonade.tools.huggingface_load import get_base_model
+
+
+class LlamaCppAdapter(ModelAdapter):
+    def __init__(self, model, output_tokens, context_size, threads, executable):
+        super().__init__()
+
+        self.model = os.path.normpath(model)
+        self.output_tokens = output_tokens
+        self.context_size = context_size
+        self.threads = threads
+        self.executable = os.path.normpath(executable)
+
+    def generate(
+        self,
+        input_ids: str,
+        max_new_tokens: Optional[int] = None,
+        temperature: float = 0.8,
+        top_p: float = 0.95,
+        top_k: int = 40,
+        return_raw: bool = False,
+        **kwargs,  # pylint: disable=unused-argument
+    ):
+        """
+        Pass a text prompt into the llamacpp inference CLI.
+
+        The input_ids arg here should receive the original text that
+        would normally be encoded by a tokenizer.
+
+        Args:
+            input_ids: The input text prompt
+            max_new_tokens: Maximum number of tokens to generate
+            temperature: Temperature for sampling (0.0 = greedy)
+            top_p: Top-p sampling threshold
+            top_k: Top-k sampling threshold
+            return_raw: If True, returns the complete raw output including timing info
+            **kwargs: Additional arguments (ignored)
+
+        Returns:
+            List containing a single string with the generated text, or raw output if
+            return_raw=True
+        """
+
+        prompt = input_ids
+        n_predict = max_new_tokens if max_new_tokens is not None else self.output_tokens
+
+        cmd = [
+            self.executable,
+            "-m",
+            self.model,
+            "--ctx-size",
+            str(self.context_size),
+            "-n",
+            str(n_predict),
+            "-t",
+            str(self.threads),
+            "-p",
+            prompt,
+            "--temp",
+            str(temperature),
+            "--top-p",
+            str(top_p),
+            "--top-k",
+            str(top_k),
+            "-e",
+            "-no-cnv",
+        ]
+
+        cmd = [str(m) for m in cmd]
+
+        try:
+            process = subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                universal_newlines=True,
+                encoding="utf-8",
+                errors="replace",
+            )
+
+            raw_output, stderr = process.communicate(timeout=600)
+            if process.returncode != 0:
+                error_msg = f"llama.cpp failed with return code {process.returncode}.\n"
+                error_msg += f"Command: {' '.join(cmd)}\n"
+                error_msg += f"Error output:\n{stderr}\n"
+                error_msg += f"Standard output:\n{raw_output}"
+                raise Exception(error_msg)
+
+            if raw_output is None:
+                raise Exception("No output received from llama.cpp process")
+
+            # Parse timing information
+            for line in raw_output.splitlines():
+                if "llama_perf_context_print:        eval time =" in line:
+                    parts = line.split("(")[1].strip()
+                    parts = parts.split(",")
+                    ms_per_token = float(parts[0].split("ms per token")[0].strip())
+                    self.tokens_per_second = (
+                        1000 / ms_per_token if ms_per_token > 0 else 0
+                    )
+                if "llama_perf_context_print: prompt eval time =" in line:
+                    parts = line.split("=")[1].split("/")[0]
+                    time_to_first_token_ms = float(parts.split("ms")[0].strip())
+                    self.time_to_first_token = time_to_first_token_ms / 1000
+
+            if return_raw:
+                return [raw_output, stderr]
+
+            # Find where the prompt ends and the generated text begins
+            prompt_found = False
+            output_text = ""
+            prompt_first_line = prompt.split("\n")[0]
+            for line in raw_output.splitlines():
+                if prompt_first_line in line:
+                    prompt_found = True
+                if prompt_found:
+                    line = line.replace("</s> [end of text]", "")
+                    output_text = output_text + line
+
+            if not prompt_found:
+                raise Exception(
+                    f"Could not find prompt '{prompt_first_line}' in llama.cpp output. "
+                    "This usually means the model failed to process the prompt correctly.\n"
+                    f"Raw output:\n{raw_output}\n"
+                    f"Stderr:\n{stderr}"
+                )
+
+            # Return list containing the generated text
+            return [output_text]
+
+        except Exception as e:
+            error_msg = f"Failed to run llama.cpp command: {str(e)}\n"
+            error_msg += f"Command: {' '.join(cmd)}"
+            raise Exception(error_msg)
+
+
+class LoadLlamaCpp(FirstTool):
+    unique_name = "load-llama-cpp"
+
+    def __init__(self):
+        super().__init__(monitor_message="Loading llama.cpp model")
+
+    @staticmethod
+    def parser(add_help: bool = True) -> argparse.ArgumentParser:
+        parser = __class__.helpful_parser(
+            short_description="Wrap llama.cpp models with an API",
+            add_help=add_help,
+        )
+
+        parser.add_argument(
+            "--executable",
+            required=True,
+            type=str,
+            help="Path to the llama.cpp executable (e.g., llama-cli or llama-cli.exe)",
+        )
+
+        default_threads = 1
+        parser.add_argument(
+            "--threads",
+            required=False,
+            type=int,
+            default=default_threads,
+            help=f"Number of threads to use for generation (default: {default_threads})",
+        )
+
+        context_size = 512
+        parser.add_argument(
+            "--context-size",
+            required=False,
+            type=int,
+            default=context_size,
+            help=f"Context size of the prompt (default: {context_size})",
+        )
+
+        output_tokens = 512
+        parser.add_argument(
+            "--output-tokens",
+            required=False,
+            type=int,
+            default=output_tokens,
+            help=f"Maximum number of output tokens the LLM should make (default: {output_tokens})",
+        )
+
+        parser.add_argument(
+            "--model-binary",
+            required=True,
+            type=str,
+            help="Path to a .gguf model file",
+        )
+
+        return parser
+
+    def run(
+        self,
+        state: State,
+        input: str = "",
+        context_size: int = 512,
+        threads: int = 1,
+        output_tokens: int = 512,
+        model_binary: Optional[str] = None,
+        executable: str = None,
+    ) -> State:
+        """
+        Load a llama.cpp model
+        """
+
+        if executable is None:
+            raise Exception(f"{self.__class__.unique_name} requires an executable path")
+
+        # Convert paths to platform-specific format
+        executable = os.path.normpath(executable)
+
+        if model_binary:
+            model_to_use = os.path.normpath(model_binary)
+        else:
+            model_binary = input
+            model_to_use = os.path.normpath(model_binary) if model_binary else None
+
+            if not model_binary:
+                model_to_use = state.get(Keys.MODEL)
+
+        if model_to_use is None:
+            raise Exception(
+                f"{self.__class__.unique_name} requires the preceding tool to pass a "
+                "Llamacpp model, "
+                "or for the user to supply a model with `--model-binary`"
+            )
+
+        state.model = LlamaCppAdapter(
+            model=model_to_use,
+            output_tokens=output_tokens,
+            context_size=context_size,
+            threads=threads,
+            executable=executable,
+        )
+        state.tokenizer = PassthroughTokenizer()
+
+        # Save stats about the model
+        state.save_stat(Keys.CHECKPOINT, model_to_use)
+
+        # Get base model information if this is a converted HF model
+        base_model = get_base_model(input)
+        if base_model is not None:
+            state.save_stat("base_model", base_model)
+
+        status.add_to_state(state=state, name=input, model=model_to_use)
+
+        return state
+
+
+# This file was originally licensed under Apache 2.0. It has been modified.
+# Modifications Copyright (c) 2025 AMD

lemonade/tools/llamacpp_bench.py

@@ -0,0 +1,154 @@
+import argparse
+import statistics
+from statistics import StatisticsError
+from lemonade.state import State
+from lemonade.cache import Keys
+from lemonade.tools.llamacpp import LlamaCppAdapter
+from lemonade.tools.bench import Bench
+
+
+class LlamaCppBench(Bench):
+
+    unique_name = "llama-cpp-bench"
+
+    def __init__(self):
+        super().__init__()
+
+        # Additional statistics generated by this bench tool
+        self.status_stats += [
+            Keys.STD_DEV_TOKENS_PER_SECOND,
+        ]
+        self.std_dev_token_generation_tokens_per_second_list = []
+
+    @staticmethod
+    def parser(add_help: bool = True) -> argparse.ArgumentParser:
+        parser = __class__.helpful_parser(
+            short_description="Benchmark a llama.cpp model",
+            add_help=add_help,
+        )
+
+        parser = Bench.parser(parser)
+
+        return parser
+
+    def run_prompt(
+        self,
+        state: State,
+        report_progress_fn,
+        prompt: str,
+        iterations: int,
+        warmup_iterations: int,
+        output_tokens: int,
+    ) -> State:
+        """
+        Benchmark llama.cpp model that was loaded by LoadLlamaCpp.
+        """
+
+        if self.first_run_prompt:
+
+            if not hasattr(state, "model") or not isinstance(
+                state.model, LlamaCppAdapter
+            ):
+                raise Exception(
+                    f"{self.__class__.unique_name} requires a LlamaCppAdapter model to be "
+                    "loaded first. Please run load-llama-cpp before this tool."
+                )
+
+        iteration_tokens_per_second = []
+        iteration_time_to_first_token = []
+
+        for iteration in range(iterations + warmup_iterations):
+            try:
+                # Use the adapter's generate method which already has the timeout
+                # and error handling
+                raw_output, stderr = state.model.generate(prompt, return_raw=True)
+
+                # Parse the timing information from the output
+                ms_per_token = None
+                time_to_first_token_ms = None
+                input_tokens = None
+
+                # Look for timing in both stdout and stderr
+                for output in [raw_output, stderr]:
+                    for line in output.splitlines():
+                        if "llama_perf_context_print:        eval time =" in line:
+                            parts = line.split("(")[1].strip()
+                            parts = parts.split(",")
+                            ms_per_token = float(
+                                parts[0].split("ms per token")[0].strip()
+                            )
+                        if "llama_perf_context_print: prompt eval time =" in line:
+                            parts = line.split("=")[1].split("/")
+                            time_to_first_token_ms = float(
+                                parts[0].split("ms")[0].strip()
+                            )
+                            input_tokens = int(parts[1].split("tokens")[0].strip())
+
+                if ms_per_token is None or time_to_first_token_ms is None:
+                    error_msg = (
+                        "Could not find timing information in llama.cpp output.\n"
+                    )
+                    error_msg += "Raw output:\n" + raw_output + "\n"
+                    error_msg += "Stderr:\n" + stderr
+                    raise Exception(error_msg)
+
+                # When output_tokens is set to 1 for accuracy tests, ms_per_token tends to 0
+                # and causes a divide-by-zero error. Set tokens_per_second to 0 in such cases
+                # as performance data for generating a few tokens is not relevant.
+                tokens_per_second = 0
+                if output_tokens > 5 and ms_per_token > 0:
+                    tokens_per_second = 1000 / ms_per_token
+                time_to_first_token = time_to_first_token_ms / 1000
+
+                if iteration > warmup_iterations - 1:
+                    iteration_tokens_per_second.append(tokens_per_second)
+                    iteration_time_to_first_token.append(time_to_first_token)
+
+                report_progress_fn((iteration + 1) / (warmup_iterations + iterations))
+
+            except Exception as e:
+                error_msg = f"Failed to run benchmark: {str(e)}"
+                raise Exception(error_msg)
+
+        self.input_ids_len_list.append(input_tokens)
+        mean_time_to_first_token = statistics.mean(iteration_time_to_first_token)
+        self.mean_time_to_first_token_list.append(mean_time_to_first_token)
+        self.prefill_tokens_per_second_list.append(
+            input_tokens / mean_time_to_first_token
+        )
+        self.token_generation_tokens_per_second_list.append(
+            statistics.mean(iteration_tokens_per_second)
+        )
+        try:
+            self.std_dev_time_to_first_token_list.append(
+                statistics.stdev(iteration_time_to_first_token)
+            )
+        except StatisticsError:
+            # Less than 2 measurements
+            self.std_dev_time_to_first_token_list.append(None)
+        try:
+            self.std_dev_token_generation_tokens_per_second_list.append(
+                statistics.stdev(iteration_tokens_per_second)
+            )
+        except StatisticsError:
+            # Less than 2 measurements
+            self.std_dev_token_generation_tokens_per_second_list.append(None)
+
+    def save_stats(self, state):
+        super().save_stats(state)
+
+        # Save additional statistics
+        if not all(
+            element is None
+            for element in self.std_dev_token_generation_tokens_per_second_list
+        ):
+            state.save_stat(
+                Keys.STD_DEV_TOKENS_PER_SECOND,
+                self.get_item_or_list(
+                    self.std_dev_token_generation_tokens_per_second_list
+                ),
+            )
+
+
+# This file was originally licensed under Apache 2.0. It has been modified.
+# Modifications Copyright (c) 2025 AMD