mlonmcu 0.3.0.dev0__3-none-any.whl

mlonmcu/__init__.py

@@ -0,0 +1,26 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Top-level package for ML on MCU."""
+
+__author__ = """TUM Department of Electrical and Computer Engineering - Chair of Electronic Design Automation"""
+__email__ = "philipp.van-kempen@tum.de"
+
+from mlonmcu.version import __version__
+
+__all__ = ["__version__"]

mlonmcu/artifact.py

@@ -0,0 +1,172 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Artifacts defintions internally used to refer to intermediate results."""
+
+from enum import Enum
+from pathlib import Path
+
+from mlonmcu.setup import utils
+
+# TODO: offer pack/unpack/flatten methods for mlf
+# TODO: implement restore methods
+# TODO: decide if inheritance based scheme would fit better
+# TODO: add artifact flags and lookup utility to find best match
+
+
+class ArtifactFormat(Enum):  # TODO: ArtifactType, ArtifactKind?
+    """Enumeration of artifact types."""
+
+    UNKNOWN = 0
+    SOURCE = 1
+    TEXT = 2
+    MLF = 3
+    MODEL = 4
+    IMAGE = 5
+    DATA = 6
+    NUMPY = 7
+    PARAMS = 8
+    JSON = 9  # ?
+    PATH = 10  # NOT A DIRECTORY?
+    RAW = 11
+    BIN = 11
+    SHARED_OBJECT = 12  # Here: the parent tar archive
+
+
+def lookup_artifacts(artifacts, name=None, fmt=None, flags=None, first_only=False):
+    """Utility to get a matching artifact for a given set of properties."""
+    matches = []
+    # Warning: if neither name, fmt nor flags is provided, the first artifact (first_only=True)
+    # or all (first_only=False) are returned
+    for artifact in artifacts:
+        valid = True
+        if name is not None and artifact.name != name:
+            valid = False
+        if fmt is not None and artifact.fmt != fmt:
+            valid = False
+        if flags is not None and not all(
+            flag in artifact.flags for flag in flags
+        ):  # A valid artifact may have more flags than specified
+            valid = False
+        if valid:
+            matches.append(artifact)
+    if len(matches) > 0 and first_only:
+        matches = [matches[0]]
+    return matches
+
+
+class Artifact:
+    """Artifact type."""
+
+    def __init__(
+        self,
+        name,
+        content=None,
+        path=None,
+        data=None,
+        raw=None,
+        fmt=ArtifactFormat.UNKNOWN,
+        flags=None,
+        archive=False,
+        optional=False,
+    ):
+        # TODO: Allow to store filenames as well as raw data
+        self.name = name
+        # TODO: too many attributes...
+        self.content = content
+        self.path = path
+        self.data = data
+        self.raw = raw
+        self.fmt = fmt
+        self.flags = flags if flags is not None else {}
+        self.archive = archive
+        self.optional = optional
+        self.validate()
+
+    def __repr__(self):
+        return f"Artifact({self.name}, fmt={self.fmt}, flags={self.flags})"
+
+    @property
+    def exported(self):
+        """Returns true if the artifact was writtem to disk."""
+        return bool(self.path is not None)
+
+    def validate(self):
+        """Checker for artifact attributes for the given format."""
+        if self.fmt in [ArtifactFormat.TEXT, ArtifactFormat.SOURCE]:
+            assert self.content is not None
+        elif self.fmt in [ArtifactFormat.RAW, ArtifactFormat.BIN]:
+            assert self.raw is not None
+        elif self.fmt in [ArtifactFormat.MLF, ArtifactFormat.SHARED_OBJECT]:
+            assert self.raw is not None
+        elif self.fmt in [ArtifactFormat.PATH]:
+            assert self.path is not None
+        else:
+            raise NotImplementedError
+
+    def export(self, dest, extract=False):
+        """Export the artifact to a given path (file or directory) and update its path.
+
+        Arguments
+        ---------
+        dest : str
+            Path of the destination.
+        extract : bool
+            If archive: extract to destination.
+
+        """
+        if dest.is_dir():
+            filename = Path(dest) / self.name
+        else:
+            filename = Path(dest)
+        if self.fmt in [ArtifactFormat.TEXT, ArtifactFormat.SOURCE]:
+            assert not extract, "extract option is only available for ArtifactFormat.MLF"
+            with open(filename, "w", encoding="utf-8") as handle:
+                handle.write(self.content)
+        elif self.fmt in [ArtifactFormat.RAW, ArtifactFormat.BIN]:
+            assert not extract, "extract option is only available for ArtifactFormat.MLF"
+            with open(filename, "wb") as handle:
+                handle.write(self.raw)
+        elif self.fmt in [ArtifactFormat.MLF, ArtifactFormat.SHARED_OBJECT]:
+            with open(filename, "wb") as handle:
+                handle.write(self.raw)
+            if extract:
+                utils.extract(filename, dest)
+                # os.remove(filename)
+        elif self.fmt in [ArtifactFormat.PATH]:
+            assert not extract, "extract option is only available for ArtifactFormat.MLF"
+            utils.copy(self.path, filename)
+        else:
+            raise NotImplementedError
+        self.path = filename if self.path is None else self.path
+
+    def print_summary(self):
+        """Utility to print information about an artifact to the cmdline."""
+        print("Format:", self.fmt)
+        print("Optional: ", self.optional)
+        if self.fmt in [ArtifactFormat.TEXT, ArtifactFormat.SOURCE]:
+            print("Content:")
+            print(self.content)
+        elif self.fmt in [ArtifactFormat.RAW, ArtifactFormat.BIN]:
+            print(f"Data Size: {len(self.raw)}B")
+        elif self.fmt in [ArtifactFormat.MLF, ArtifactFormat.SHARED_OBJECT]:
+            print(f"Archive Size: {len(self.raw)}B")
+        elif self.fmt in [ArtifactFormat.PATH]:
+            print(f"File Location: {self.path}")
+        else:
+            raise NotImplementedError

mlonmcu/cli/__init__.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Submodule handling the command line interface for mlonmcu."""

mlonmcu/cli/build.py

@@ -0,0 +1,119 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Command line subcommand for the build process."""
+
+from mlonmcu.flow import get_available_backend_names
+from mlonmcu.cli.common import kickoff_runs
+from mlonmcu.cli.load import handle as handle_load, add_load_options
+from mlonmcu.context.context import MlonMcuContext
+from mlonmcu.session.run import RunStage
+from mlonmcu.platform.lookup import get_platforms_targets, get_platforms_backends
+from .helper.parse import (
+    extract_backend_names,
+    extract_target_names,
+    extract_platform_names,
+    extract_config_and_feature_names,
+)
+
+
+def add_build_options(parser):
+    # TODO: rename to build_group
+    add_load_options(parser)
+    build_parser = parser.add_argument_group("build options")
+    build_parser.add_argument(
+        "-b",
+        "--backend",
+        type=str,
+        action="append",
+        choices=get_available_backend_names(),
+        help="Backends to use (default: %(default)s)",
+    )
+
+
+def get_parser(subparsers, parent=None):
+    """ "Define and return a subparser for the build subcommand."""
+    parser = subparsers.add_parser(
+        "build",
+        description="Build model using the ML on MCU flow.",
+        parents=[parent] if parent else [],
+        add_help=(parent is None),
+    )
+    parser.set_defaults(flow_func=handle)
+    add_build_options(parser)
+    return parser
+
+
+def _handle(args, context, require_target=False):
+    handle_load(args, ctx=context)
+    backends = extract_backend_names(args, context=context)
+    targets = extract_target_names(args, context=context if require_target else None)
+    platforms = extract_platform_names(args, context=context)
+
+    new_config, _, _, _ = extract_config_and_feature_names(args, context=context)
+    platform_backends = get_platforms_backends(context, config=new_config)  # This will be slow?
+    platform_targets = get_platforms_targets(context, config=new_config)  # This will be slow?
+
+    assert len(context.sessions) > 0
+    session = context.sessions[-1]
+    new_runs = []
+    for run in session.runs:
+        for target_name in targets:
+            for backend_name in backends:
+                new_run = run.copy()
+                if backend_name is not None:
+                    platform_name = None
+                    for platform in platforms:
+                        candidates = platform_backends[platform]
+                        if backend_name in candidates:
+                            try:
+                                platform_name = platform
+                                new_run.add_platform_by_name(platform_name, context=context)
+                                break
+                            except AssertionError:  # TODO: replace with incompatble error
+                                platform_name = None
+                                continue
+                    # assert (
+                    #     platform_name is not None
+                    # ), f"Unable to find a suitable platform for the backend '{target_name}'"
+                    # assert platform_name is not None
+
+                if target_name is not None:
+                    platform_name = None
+                    for platform in platforms:
+                        candidates = platform_targets[platform]
+                        if target_name in candidates:
+                            platform_name = platform
+                    assert (
+                        platform_name is not None
+                    ), f"Unable to find a suitable platform for the target '{target_name}'"
+                    new_run.add_platform_by_name(platform_name, context=context)
+                    new_run.add_target_by_name(target_name, context=context)
+                new_run.add_backend_by_name(backend_name, context=context)
+                new_runs.append(new_run)
+
+    session.runs = new_runs
+
+
+def handle(args, ctx=None, require_target=False):
+    if ctx:
+        _handle(args, ctx, require_target=require_target)
+    else:
+        with MlonMcuContext(path=args.home, deps_lock="read") as context:
+            _handle(args, context, require_target=require_target)
+            kickoff_runs(args, RunStage.BUILD, context)

mlonmcu/cli/cleanup.py

@@ -0,0 +1,75 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Command line subcommand for cleaning up the current environment."""
+from mlonmcu.context.context import MlonMcuContext
+from mlonmcu.setup import setup
+
+from mlonmcu.cli.common import (
+    add_common_options,
+    add_context_options,
+)
+
+
+def get_parser(subparsers):
+    """ "Define and return a subparser for the cleanup subcommand."""
+    parser = subparsers.add_parser("cleanup", description="Cleanup ML on MCU environment.")
+    parser.set_defaults(func=handle)
+    add_common_options(parser)
+    add_context_options(parser)
+    parser.add_argument(
+        "-f",
+        "--force",
+        default=False,
+        action="store_true",
+        help="Do not ask before removing disk contents (DANGEROUS)",
+    )
+    parser.add_argument(
+        "-k",
+        "--keep",
+        metavar="KEEP",
+        type=int,
+        default=10,
+        help="Remove everything except the latest KEEP sessions (default: %(default)s)",
+    )
+    parser.add_argument(
+        "--deps",
+        default=False,
+        action="store_true",
+        help="Also delete all dependencies from the environment.",
+    )
+    parser.add_argument(
+        "--cache",
+        default=False,
+        action="store_true",
+        help="Clear the environments dependency cache.",
+    )
+    return parser
+
+
+def handle(args):
+    with MlonMcuContext(path=args.home, deps_lock="write") as context:
+        interactive = not args.force
+        keep = args.keep
+        context.cleanup_sessions(keep=keep, interactive=interactive)
+        installer = setup.Setup(context=context)
+        if args.deps:
+            # This will also remove the cache file
+            installer.clean_dependencies(interactive=interactive)
+        elif args.cache:
+            installer.clean_cache(interactive=interactive)

mlonmcu/cli/common.py

@@ -0,0 +1,227 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+import sys
+import multiprocessing
+import logging
+
+from mlonmcu.platform import get_platforms
+from mlonmcu.session.postprocess import SUPPORTED_POSTPROCESSES
+from mlonmcu.feature.features import get_available_feature_names
+from mlonmcu.logging import get_logger, set_log_level
+from .helper.parse import extract_config
+
+logger = get_logger()
+
+
+def handle_logging_flags(args):
+    if hasattr(args, "verbose") and hasattr(args, "quiet"):
+        if args.verbose and args.quiet:
+            raise RuntimeError("--verbose and --quiet can not be used at the same time")
+        elif args.verbose:
+            set_log_level(logging.DEBUG)
+        elif args.quiet:
+            set_log_level(logging.WARNING)
+        else:
+            set_log_level(logging.INFO)
+
+
+def add_flow_options(parser):
+    flow_parser = parser.add_argument_group("flow options")
+    flow_parser.add_argument(  # TODO: move to compile.py?
+        "-t",
+        "--target",
+        type=str,
+        metavar="TARGET",
+        # choices=SUPPORTED_TARGETS.keys(),
+        action="append",
+        # default=None,
+        # nargs=1,
+        help="The target device/architecture (choices: See --list-targets)",
+    )
+    flow_parser.add_argument(  # TODO: move to compile.py?
+        "--list-targets",
+        action="store_true",
+        help="List the supported targets in the environment",
+    )
+    flow_parser.add_argument(
+        # "-p",
+        "--platform",
+        type=str,
+        metavar="PLATFORM",
+        choices=get_platforms().keys(),
+        default=None,
+        action="append",
+        nargs=1,
+        help="Explicitly choose the platforms to use (choices: %(choices)s)",
+    )
+    flow_parser.add_argument(
+        # "-p",
+        "--postprocess",
+        type=str,
+        metavar="POSTPROCESS",
+        choices=SUPPORTED_POSTPROCESSES.keys(),
+        action="append",
+        help="Choose the postprocesses to apply (choices: %(choices)s)",
+    )
+    flow_parser.add_argument(
+        "-f",
+        "--feature",
+        type=str,
+        metavar="FEATURE",
+        # nargs=1,
+        action="append",
+        choices=get_available_feature_names(),
+        help="Enabled features for target/framework/backend (choices: %(choices)s)",
+    )
+    flow_parser.add_argument(
+        "--feature-gen",
+        dest="feature_gen",
+        type=str,
+        metavar="FEATURES",
+        nargs="+",
+        action="append",
+        help="Generator statement for features.",
+    )
+    flow_parser.add_argument(
+        "-c",
+        "--config",
+        metavar="KEY=VALUE",
+        nargs="+",
+        action="append",
+        help=(
+            "Set a number of key-value pairs "
+            "(do not put spaces before or after the = sign). "
+            "If a value contains spaces, you should define "
+            "it with double quotes: "
+            'foo="this is a sentence". Note that '
+            "values are always treated as strings."
+        ),
+    )
+    flow_parser.add_argument(
+        "--config-gen",
+        dest="config_gen",
+        metavar="KEY=VALUE",
+        nargs="+",
+        action="append",
+        help="Generator statement for configs.",
+    )
+    flow_parser.add_argument(
+        "--parallel",
+        metavar="THREADS",
+        nargs="?",
+        type=int,
+        const=multiprocessing.cpu_count(),
+        default=1,
+        help="Use multiple threads to process runs in parallel (%(const)s if specified, else %(default)s)",
+    )
+    flow_parser.add_argument(
+        "-p",
+        "--progress",
+        action="store_true",
+        help="Display progress bar (default: %(default)s)",
+    )
+    flow_parser.add_argument(
+        "--resume",
+        action="store_true",
+        help="Try to resume the latest session (default: %(default)s)",
+    )
+    flow_parser.add_argument(  # TODO: move to compile.py?
+        "-l",
+        "--label",
+        type=str,
+        metavar="LABEL",
+        default="",
+        help="Label for the session (default: %(default)s)",
+    )
+
+
+def add_common_options(parser):
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Print detailed messages for easier debugging (default: %(default)s)",
+    )
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        action="store_true",
+        help="Reduce number of logging statements to a minimum (default: %(default)s)",
+    )
+    parser.add_argument(
+        "--docker",
+        action="store_true",
+        help="Execute run.py inside docker container (default: %(default)s)",
+    )
+
+
+def add_context_options(parser, with_home=True):
+    common = parser.add_argument_group("context options")
+    if with_home:
+        home = os.getenv("MLONMCU_HOME", None)
+        common.add_argument(
+            "-H",
+            "--home",
+            "--hint",
+            type=str,
+            default=home,
+            help="The path to the mlonmcu environment (overwriting $MLONMCU_HOME environment variable)",
+        )
+
+
+def add_model_options(parser):
+    parser.add_argument(
+        "models",
+        metavar="model",
+        type=str,
+        nargs="+",
+        default=None,
+        help="Model to process",
+    )
+
+
+def kickoff_runs(args, until, context):
+    assert len(context.sessions) > 0
+    session = context.sessions[-1]
+    # session.label = args.label
+    config = extract_config(args)
+    # TODO: move into context/session
+    per_stage = True
+    print_report = True
+    if "runs_per_stage" in config:
+        per_stage = bool(config["runs_per_stage"])
+    elif "runs_per_stage" in context.environment.vars:
+        per_stage = bool(context.environment.vars["runs_per_stage"])
+    if "print_report" in config:
+        print_report = bool(config["print_report"])
+    elif "print_report" in context.environment.vars:
+        print_report = bool(context.environment.vars["print_report"])
+    success = session.process_runs(
+        until=until,
+        per_stage=per_stage,
+        print_report=print_report,
+        num_workers=args.parallel,
+        progress=args.progress,
+        context=context,
+        export=True,
+    )
+    if not success:
+        logger.error("At least one error occured!")
+        sys.exit(1)