lemonade-sdk 7.0.0__py3-none-any.whl

lemonade/common/cli_helpers.py

@@ -0,0 +1,139 @@
+import argparse
+import sys
+from typing import List, Dict, Tuple, Any
+from lemonade.tools import Tool, FirstTool
+import lemonade.common.printing as printing
+from lemonade.tools.management_tools import ManagementTool
+
+
+class CustomArgumentParser(argparse.ArgumentParser):
+
+    def error(self, message):
+        self.print_usage()
+        printing.log_error(message)
+        self.exit(2)
+
+
+def _tool_list_help(tools: List[Tool], subclass, exclude=None) -> str:
+    help = ""
+
+    for tool_class in tools:
+        if exclude and issubclass(tool_class, exclude):
+            continue
+        if issubclass(tool_class, subclass):
+            help = (
+                help
+                + f" * {tool_class.unique_name}: {tool_class.parser().short_description}\n"
+            )
+
+    return help
+
+
+def parse_tools(
+    parser: argparse.ArgumentParser, supported_tools: List[Tool], cli_name="lemonade"
+) -> Tuple[Dict[str, Any], Dict[Tool, List[str]], List[str]]:
+    """
+    Add the help for parsing tools and their args to an ArgumentParser.
+
+    Then, perform the task of parsing a full CLI command including
+    teasing apart the global arguments and separate tool invocations.
+    """
+
+    tool_parsers = {tool.unique_name: tool.parser() for tool in supported_tools}
+    tool_classes = {tool.unique_name: tool for tool in supported_tools}
+
+    # Sort tools into categories and format for the help menu
+    first_tool_choices = _tool_list_help(supported_tools, FirstTool)
+    eval_tool_choices = _tool_list_help(supported_tools, Tool, exclude=FirstTool)
+    mgmt_tool_choices = _tool_list_help(supported_tools, ManagementTool)
+
+    tools_action = parser.add_argument(
+        "tools",
+        metavar="tool --tool-args [tool --tool-args...]",
+        nargs="?",
+        help=f"""\
+Run `{cli_name} TOOL -h` to learn more about each tool.
+
+Tools that can start a sequence:
+{first_tool_choices}
+Tools that go into a sequence:
+{eval_tool_choices}
+Management tools:
+{mgmt_tool_choices}""",
+        choices=tool_parsers.keys(),
+    )
+
+    # run as if "-h" was passed if no parameters are passed
+    if len(sys.argv) == 1:
+        sys.argv.append("-h")
+
+    # Break sys.argv into categories based on which tools were invoked
+    # Arguments that are passed prior to invoking a tool are categorized as
+    # global arguments that should be used to initialize the state.
+    current_tool = "globals"
+    tools_invoked = {current_tool: []}
+    cmd = sys.argv[1:]
+    while len(cmd):
+        if cmd[0] in tool_parsers.keys():
+            # Make sure each tool was only called once
+            if cmd[0] in tools_invoked.keys():
+                parser.error(
+                    "A single call to lemonade can only invoke each tool once, "
+                    f"however this call invokes tool {cmd[0]} multiple times."
+                )
+            current_tool = cmd.pop(0)
+            tools_invoked[current_tool] = []
+        else:
+            tools_invoked[current_tool].append(cmd.pop(0))
+
+    # Trick argparse into thinking tools was not a positional argument
+    # this helps to avoid an error where an incorrect arg/value pair
+    # can be misinterpreted as the tools positional argument
+    tools_action.option_strings = ["--tools"]
+
+    # Do one pass of parsing to figure out if -h was used
+    global_args = vars(parser.parse_args(tools_invoked["globals"]))
+
+    # Remove "tools" from global args because it was just there
+    # as a placeholder
+    global_args.pop("tools")
+
+    # Remove globals from the list since its already been parsed
+    tools_invoked.pop("globals")
+    evaluation_tools = []
+    management_tools = []
+    for cmd, argv in tools_invoked.items():
+        tool_parsers[cmd].parse_args(argv)
+
+        # Keep track of whether the tools are ManagementTool or not,
+        # since ManagementTools are mutually exclusive with evaluation
+        # tools
+        if issubclass(tool_classes[cmd], ManagementTool):
+            management_tools.append(cmd)
+        else:
+            evaluation_tools.append(cmd)
+
+    if len(management_tools) > 0 and len(evaluation_tools) > 0:
+        parser.error(
+            "This call to lemonade invoked both management and "
+            "evaluation tools, however each call to lemonade "
+            "is only allowed to invoke one or the other. "
+            f"Management tools: {management_tools};"
+            f"Evaluation tools: {evaluation_tools}."
+        )
+
+    if len(management_tools) == 0 and len(evaluation_tools) == 0:
+        parser.error(
+            "Calls to lemonade are required to call at least "
+            "one tool or management tool."
+        )
+
+    # Convert tool names into Tool instances
+    tool_instances = {tool_classes[cmd](): argv for cmd, argv in tools_invoked.items()}
+    evaluation_tools = [tool_classes[cmd] for cmd in evaluation_tools]
+
+    return global_args, tool_instances, evaluation_tools
+
+
+# This file was originally licensed under Apache 2.0. It has been modified.
+# Modifications Copyright (c) 2025 AMD

lemonade/common/exceptions.py

@@ -0,0 +1,98 @@
+import lemonade.common.printing as printing
+
+
+class Error(Exception):
+    """
+    Indicates something has gone wrong while running the tools
+    """
+
+    def __init__(self, msg):
+        super().__init__(msg)
+        printing.log_error(msg)
+
+
+class CacheError(Error):
+    """
+    Indicates ambiguous behavior from when a build already exists in the cache,
+    but the model, inputs, or args have changed thereby invalidating
+    the cached copy of the model.
+    """
+
+
+class EnvError(Error):
+    """
+    Indicates to the user that the required tools are not
+    available on their PATH.
+    """
+
+
+class ArgError(Error):
+    """
+    Indicates to the user that they provided invalid arguments
+    """
+
+
+class ToolError(Exception):
+    """
+    Let the user know that something went wrong while
+    running a tool.
+
+    Note: not overloading __init__() so that the
+    attempt to print to stdout isn't captured into
+    the Tool's log file.
+    """
+
+
+class StateError(Exception):
+    """
+    Raised when something goes wrong with State
+    """
+
+
+class IntakeError(Exception):
+    """
+    Let the user know that something went wrong during the
+    initial intake process of analyzing a model.
+    """
+
+
+class IOError(Error):
+    """
+    Indicates to the user that an input/output operation failed,
+    such trying to open a file.
+    """
+
+
+class ModelArgError(Error):
+    """
+    Indicates to the user that values provided to a Model instance method
+    were not allowed.
+    """
+
+
+class ModelRuntimeError(Error):
+    """
+    Indicates to the user that attempting to invoke a Model instance failed.
+    """
+
+
+class BenchmarkException(Exception):
+    """
+    Indicates a failure during benchmarking
+    """
+
+
+class HardwareError(Error):
+    """
+    Indicates that the hardware used is faulty or unavailable.
+    """
+
+
+class SkipBuild(Exception):
+    """
+    Indicates that an exception is deliberately being raised to skip a build
+    """
+
+
+# This file was originally licensed under Apache 2.0. It has been modified.
+# Modifications Copyright (c) 2025 AMD

lemonade/common/filesystem.py

@@ -0,0 +1,368 @@
+import os
+import shutil
+import glob
+import pathlib
+from typing import Dict, List, Optional
+import yaml
+import lemonade.common.printing as printing
+import lemonade.common.build as build
+import lemonade.common.exceptions as exp
+
+CACHE_MARKER = ".lemonadecache"
+BUILD_MARKER = ".lemonadebuild"
+
+
+def rmdir(folder, excludes: Optional[List[str]] = None):
+    """
+    Remove the contents of a directory from the filesystem.
+    If `<name>` is in `excludes`, the directory itself and the file named <name>
+    are kept. Otherwise, the entire directory is removed.
+    """
+
+    # Use an empty list by default
+    if excludes:
+        excludes_to_use = excludes
+    else:
+        excludes_to_use = []
+
+    if os.path.isdir(folder):
+        for filename in os.listdir(folder):
+            file_path = os.path.join(folder, filename)
+            if file_path not in excludes_to_use:
+                if os.path.isfile(file_path) or os.path.islink(file_path):
+                    os.unlink(file_path)
+                elif os.path.isdir(file_path):
+                    shutil.rmtree(file_path)
+
+        if excludes is None:
+            shutil.rmtree(folder)
+
+        return True
+
+    else:
+        return False
+
+
+def get_all(path, exclude_path=False, file_type=build.state_file_name, recursive=True):
+    if recursive:
+        files = [
+            os.path.join(dp, f)
+            for dp, dn, filenames in os.walk(path)
+            for f in filenames
+            if file_type in f
+        ]
+    else:
+        files = []
+        dp, _, filenames = os.walk(path)
+        for f in filenames:
+            if file_type in f:
+                files.append(os.path.join(dp, f))
+
+    if exclude_path:
+        files = [os.path.basename(f) for f in files]
+
+    return files
+
+
+def clean_file_name(script_path: str) -> str:
+    """
+    Trim the ".py" / ".onnx" if present.
+
+    If its a state.yaml file, trim the "_state.yaml"
+    """
+
+    if script_path.endswith("_" + build.state_file_name):
+        return pathlib.Path(script_path).stem.replace(
+            "_" + os.path.splitext(build.state_file_name)[0], ""
+        )
+    else:
+        return pathlib.Path(script_path).stem
+
+
+class CacheError(exp.Error):
+    """
+    Raise this exception when the cache is being accessed incorrectly
+    """
+
+
+def _load_yaml(file) -> Dict:
+    if os.path.isfile(file):
+        with open(file, "r", encoding="utf8") as stream:
+            return yaml.load(stream, Loader=yaml.FullLoader)
+    else:
+        return {}
+
+
+def save_yaml(dict: Dict, file):
+    with open(file, "w", encoding="utf8") as outfile:
+        yaml.dump(dict, outfile)
+
+
+def print_yaml_file(file_path, description):
+    if os.path.exists(file_path):
+        with open(file_path, "r", encoding="utf-8") as file:
+            printing.log_info(f"The {description} for {file_path} are:")
+            print(file.read())
+    else:
+        raise CacheError(
+            f"No {description} found at {file_path}. "
+            "Try running `lemonade cache --list` to see the builds in your build cache."
+        )
+
+
+def make_cache_dir(cache_dir: str):
+    """
+    Create the build and cache directories, and put hidden files in them
+    to mark them as such.
+    """
+
+    os.makedirs(cache_dir, exist_ok=True)
+
+    # File that indicates that the directory is a cache directory
+    cache_file_path = os.path.join(cache_dir, CACHE_MARKER)
+    open(cache_file_path, mode="w", encoding="utf").close()
+
+
+def make_build_dir(cache_dir: str, build_name: str):
+    """
+    Create the build and cache directories, and put hidden files in them
+    to mark them as such.
+    """
+    make_cache_dir(cache_dir)
+
+    build_dir = build.output_dir(cache_dir, build_name)
+    os.makedirs(build_dir, exist_ok=True)
+
+    # File that indicates that the directory is a build directory
+    build_file_path = os.path.join(build_dir, BUILD_MARKER)
+    open(build_file_path, mode="w", encoding="utf").close()
+
+
+def check_cache_dir(cache_dir: str):
+    cache_file_path = os.path.join(cache_dir, CACHE_MARKER)
+    if not os.path.isfile(cache_file_path):
+        raise CacheError(
+            f"{cache_dir} is not a cache directory generated by Lemonade. "
+            "You can only clean, delete and generate reports for directories that "
+            "have been generated by Lemonade. Set a different --cache-dir before "
+            "trying again."
+        )
+
+
+def is_build_dir(cache_dir: str, build_name: str):
+    build_dir = build.output_dir(cache_dir, build_name)
+    build_file_path = os.path.join(build_dir, BUILD_MARKER)
+    return os.path.isfile(build_file_path)
+
+
+def clean_output_dir(cache_dir: str, build_name: str) -> None:
+    """
+    Delete all elements of the output directory that are not human readable
+    """
+    output_dir = build.output_dir(cache_dir, build_name)
+    if os.path.isdir(output_dir) and is_build_dir(cache_dir, build_name):
+        output_dir = os.path.expanduser(output_dir)
+    else:
+        raise CacheError(f"No build found at {output_dir}")
+
+    # Remove files that do not have an allowed extension
+    allowed_extensions = (".txt", ".out", ".yaml", ".json", ".png")
+    all_paths = glob.glob(f"{output_dir}/**/*", recursive=True)
+    for path in all_paths:
+        if os.path.isfile(path) and not path.endswith(allowed_extensions):
+            os.remove(path)
+
+    # Remove all empty folders
+    for path in all_paths:
+        if os.path.isdir(path):
+            if len(os.listdir(path)) == 0:
+                shutil.rmtree(path)
+
+
+def get_available_builds(cache_dir):
+    """
+    Get all of the build directories within the build cache
+    located at `cache_dir`
+    """
+
+    check_cache_dir(cache_dir)
+
+    builds = [
+        pathlib.PurePath(build_name).name
+        for build_name in os.listdir(os.path.abspath(build.builds_dir(cache_dir)))
+        if os.path.isdir(build.output_dir(cache_dir, build_name))
+        and is_build_dir(cache_dir, build_name)
+    ]
+    builds.sort()
+
+    return builds
+
+
+class Keys:
+    # Number of parameters in the model
+    PARAMETERS = "parameters"
+    # List of all build tools in the Sequence
+    SELECTED_SEQUENCE_OF_TOOLS = "selected_sequence_of_tools"
+    # MeasuredPerformance data for a benchmarked workload
+    PERFORMANCE = "performance"
+    # Runtime used for the benchmark
+    RUNTIME = "runtime"
+    # Type of device used for the benchmark (e.g., "x86")
+    DEVICE_TYPE = "device_type"
+    # Specific device used for the benchmark
+    DEVICE = "device"
+    # Name of the model
+    MODEL_NAME = "model_name"
+    # Number of iterations used in benchmarking
+    ITERATIONS = "iterations"
+    # System information to keep track of DUT
+    SYSTEM_INFO = "system_info"
+    # Indicates status of the most recent build tool run: FunctionStatus
+    BUILD_STATUS = "build_status"
+    # Prefix for reporting the execution duration of a tool
+    # In the report this will look like tool_duration:TOOL_NAME
+    TOOL_DURATION = "tool_duration"
+    # Prefix for reporting the peak working memory in the build through this tool
+    # In the report this will look like tool_memory:TOOL_NAME
+    TOOL_MEMORY = "tool_memory"
+    # Prefix for reporting the execution status of a tool
+    # In the report this will look like tool_status:TOOL_NAME
+    TOOL_STATUS = "tool_status"
+    # Records the date and time of the evaluation after analysis but before
+    # build and benchmark
+    TIMESTAMP = "timestamp"
+    # Records the logfile of any failed tool/benchmark
+    ERROR_LOG = "error_log"
+    # Name of the build in the cache
+    BUILD_NAME = "build_name"
+    # Sequence of tools used for this build, along with their args
+    SEQUENCE_INFO = "sequence_info"
+    # Version of Lemonade used for the build
+    LEMONADE_VERSION = "lemonade_version"
+    # Unique ID for this build
+    UID = "uid"
+    # Directory where the lemonade build cache is stored
+    CACHE_DIR = "cache_dir"
+    # Example inputs to the model
+    INPUTS = "inputs"
+    # Path to the file containing the memory usage plot
+    MEMORY_USAGE_PLOT = "memory_usage_plot"
+    # Average of all tested MMLU subject scores
+    AVERAGE_MMLU_ACCURACY = "average_mmlu_accuracy"
+
+
+def _clean_logfile(logfile_lines: List[str]) -> List[str]:
+    """
+    Remove the whitespace and empty lines from an array of logfile lines
+    """
+    return "\n".join([line.rstrip() for line in logfile_lines if line.rstrip()])
+
+
+def stats_file(cache_dir: str, build_name: str):
+    """
+    Returns the expected location of the lemonade stats file
+    """
+    dir = build.output_dir(cache_dir, build_name)
+    return os.path.join(dir, "lemonade_stats.yaml")
+
+
+class Stats:
+    def __init__(self, cache_dir: str, build_name: str):
+        self.file = stats_file(cache_dir, build_name)
+
+        os.makedirs(os.path.dirname(self.file), exist_ok=True)
+        if not os.path.exists(self.file):
+            # Start an empty stats file
+            save_yaml({}, self.file)
+
+    @property
+    def stats(self):
+        return _load_yaml(self.file)
+
+    def _set_key(self, dict, keys: List["str"], value):
+        """
+        Recursive approach to safely setting a key within any level of hierarchy
+        in a dictionary. If a parent key of the desired key does not exist, create
+        it and set it with an empty dictionary before proceeding.
+
+        The end result is: dict[keys[0]][keys[1]]...[keys[-1]] = value
+        """
+        if len(keys) == 1:
+            dict[keys[0]] = value
+
+        else:
+            if keys[0] not in dict.keys():
+                dict[keys[0]] = {}
+
+            self._set_key(dict[keys[0]], keys[1:], value)
+
+    def save_stat(self, key: str, value):
+        """
+        Save statistics to an yaml file in the build directory
+        """
+
+        stats_dict = self.stats
+
+        self._set_key(stats_dict, [key], value)
+
+        save_yaml(stats_dict, self.file)
+
+    def save_sub_stat(self, parent_key: str, key: str, value):
+        stats_dict = self.stats
+
+        self._set_key(stats_dict, [parent_key, key], value)
+
+        save_yaml(stats_dict, self.file)
+
+    def save_eval_error_log(self, logfile_path):
+        if logfile_path is None:
+            # Avoid an error in the situation where we crashed before
+            # initializing the tool (in which case it has no logfile path yet)
+            return
+        if os.path.exists(logfile_path):
+            with open(logfile_path, "r", encoding="utf-8") as f:
+                full_log = f.readlines()
+
+                # Log files can be quite large, so we will just record the beginning
+                # and ending lines. Users can always open the log file if they
+                # want to see the full log.
+                start_cutoff = 5
+                end_cutoff = -30
+                max_full_length = start_cutoff + abs(end_cutoff)
+
+                if len(full_log) > max_full_length:
+                    log_start = _clean_logfile(full_log[:start_cutoff])
+                    log_end = _clean_logfile(full_log[end_cutoff:])
+                    truncation_notice = (
+                        "NOTICE: This copy of the log has been truncated to the first "
+                        f"{start_cutoff} and last {abs(end_cutoff)} lines "
+                        f"to save space. Please see {logfile_path} "
+                        "to see the full log.\n"
+                    )
+
+                    stats_log = log_start + truncation_notice + log_end
+                else:
+                    stats_log = _clean_logfile(full_log)
+
+                self.save_stat(Keys.ERROR_LOG, stats_log)
+
+
+def expand_inputs(input_paths: List[str]) -> List[str]:
+    """
+    Convert regular expressions in input paths
+    into full file/dir paths (e.g., [*.py] -> [a.py, b.py] )
+
+    This makes up for Windows not resolving wildcards on the command line
+    """
+    input_paths_expanded = sum(
+        [glob.glob(f) for f in input_paths if "::" not in f], []
+    ) + [f for f in input_paths if "::" in f]
+
+    if not input_paths_expanded:
+        raise exp.ArgError("No files that match your inputs could be found.")
+
+    return input_paths_expanded
+
+
+# This file was originally licensed under Apache 2.0. It has been modified.
+# Modifications Copyright (c) 2025 AMD

lemonade/common/labels.py

@@ -0,0 +1,61 @@
+from typing import Dict, List
+import turnkeyml.common.printing as printing
+
+
+def to_dict(label_list: List[str]) -> Dict[str, List[str]]:
+    """
+    Convert label list into a dictionary of labels
+    """
+    label_dict = {}
+    for item in label_list:
+        try:
+            label_key, label_value = item.split("::")
+            label_value = label_value.split(",")
+            label_dict[label_key] = label_value
+        except ValueError:
+            printing.log_warning(
+                (
+                    f"Malformed label {item} found. "
+                    "Each label must have the format key::value1,value2,... "
+                )
+            )
+    return label_dict
+
+
+def load_from_file(file_path: str) -> Dict[str, List[str]]:
+    """
+    This function extracts labels from a Python file.
+    Labels must be in the first line of a Python file and start with "# labels: "
+    Each label must have the format "key::value1,value2,..."
+
+    Example:
+        "# labels: author::google test_group::daily,monthly"
+    """
+    # Open file
+    with open(file_path, encoding="utf-8") as f:
+        first_line = f.readline()
+
+    # Return label dict
+    if "# labels:" in first_line:
+        label_list = first_line.replace("\n", "").split(" ")[2:]
+        return to_dict(label_list)
+    else:
+        return {}
+
+
+def is_subset(label_dict_a: Dict[str, List[str]], label_dict_b: Dict[str, List[str]]):
+    """
+    This function returns True if label_dict_a is a subset of label_dict_b.
+    More specifically, we return True if:
+        * All keys of label_dict_a are also keys of label_dict_b AND,
+        * All values of label_dict_a[key] are values of label_dict_b[key]
+    """
+    for key in label_dict_a:
+        # Skip benchmarking if the label_dict_a key is not a key of label_dict_b
+        if key not in label_dict_b:
+            return False
+        # A label key may point to multiple label values
+        # Skip if not all values of label_dict_a[key] are in label_dict_b[key]
+        elif not all(elem in label_dict_a[key] for elem in label_dict_b[key]):
+            return False
+    return True