mlonmcu 0.3.0.dev0__3-none-any.whl

mlonmcu/cli/load.py

@@ -0,0 +1,87 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Command line subcommand for the load stage."""
+
+from mlonmcu.cli.common import (
+    add_common_options,
+    add_context_options,
+    add_model_options,
+    add_flow_options,
+    kickoff_runs,
+)
+
+from .helper.parse import extract_config_and_feature_names, extract_frontend_names, extract_postprocess_names
+from mlonmcu.context.context import MlonMcuContext
+from mlonmcu.models import SUPPORTED_FRONTENDS
+from mlonmcu.models.lookup import apply_modelgroups
+from mlonmcu.session.run import RunStage
+
+
+def add_load_options(parser):
+    add_common_options(parser)
+    add_context_options(parser)
+    add_flow_options(parser)
+    add_model_options(parser)
+    load_parser = parser.add_argument_group("load options")
+    load_parser.add_argument(
+        "--frontend",
+        type=str,
+        metavar="FRONTEND",
+        choices=SUPPORTED_FRONTENDS.keys(),
+        default=None,
+        nargs=1,
+        help="Explicitly choose the frontends to use (choices: %(choices)s)",
+    )
+
+
+def get_parser(subparsers):
+    """ "Define and return a subparser for the load subcommand."""
+    parser = subparsers.add_parser("load", description="Load model using the ML on MCU flow.")
+    parser.set_defaults(flow_func=handle)
+    add_load_options(parser)
+    return parser
+
+
+def _handle(args, context):
+    config = context.environment.vars
+    new_config, features, gen_config, gen_features = extract_config_and_feature_names(args, context=context)
+    config.update(new_config)
+    frontends = extract_frontend_names(args, context=context)
+    postprocesses = extract_postprocess_names(args, context=context)
+    session = context.get_session(label=args.label, resume=args.resume, config=config)
+    models = apply_modelgroups(args.models, context=context)
+    for model in models:
+        for f in gen_features:
+            for c in gen_config:
+                all_config = {**config, **c}
+                run = session.create_run(config=all_config)
+                all_features = list(set(features + f))
+                run.add_features_by_name(all_features, context=context)  # TODO do this before load.py?
+                run.add_frontends_by_name(frontends, context=context)
+                run.add_model_by_name(model, context=context)
+                run.add_postprocesses_by_name(postprocesses, context=context)  # TODO do this before load.py?
+
+
+def handle(args, ctx=None):
+    if ctx:
+        _handle(args, ctx)
+    else:
+        with MlonMcuContext(path=args.home, deps_lock="read") as context:
+            _handle(args, context)
+            kickoff_runs(args, RunStage.LOAD, context)

mlonmcu/cli/main.py

@@ -0,0 +1,116 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Console script for mlonmcu."""
+
+import os
+import argparse
+import sys
+import subprocess
+import platform
+
+
+from mlonmcu.logging import get_logger
+import mlonmcu.cli.init as init
+import mlonmcu.cli.setup as setup
+import mlonmcu.cli.flow as flow
+import mlonmcu.cli.cleanup as cleanup
+import mlonmcu.cli.export as export
+import mlonmcu.cli.env as env
+import mlonmcu.cli.models as models
+from .common import handle_logging_flags, add_common_options
+from ..version import __version__
+
+logger = get_logger()
+
+
+def handle_docker(args):
+    if args.docker:
+        home = os.environ.get("MLONMCU_HOME")
+        assert home is not None, (
+            "To use the --docker functionality, please export the MLONMCU_HOME environment variable"
+            + " to a directory which should be mounted by the container"
+        )
+        exec_args = sys.argv[1:]
+        exec_args.remove("--docker")
+        docker = subprocess.Popen(
+            [
+                "docker-compose",
+                "-f",
+                "docker/docker-compose.yml",
+                "run",
+                "-e",
+                f"MLONMCU_HOME={home}",
+                "--rm",
+                "mlonmcu",
+                "python3",
+                "-m",
+                "mlonmcu.cli.main",
+                *exec_args,
+            ],
+            env={"MLONMCU_HOME": home},
+        )
+        stdout, stderr = docker.communicate()
+        exit_code = docker.wait()
+        if exit_code > 0:
+            logger.warning(f"Docker compose process completed with exit code: {exit_code}")
+        sys.exit(exit_code)
+
+    if platform.system() in ["Darwin", "Windows"]:
+        raise RuntimeError(
+            "Only Linux is supported at the Moment. If you have Docker installed, you may want to"
+            + " try running this script using the `--docker` flag."
+        )
+
+
+# def main(args):
+def main(args=None):
+    """Console script for mlonmcu."""
+    parser = argparse.ArgumentParser(
+        description="ML on MCU Flow",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    # parser.add_argument('_', nargs='*')
+    parser.add_argument("-V", "--version", action="version", version="mlonmcu " + __version__)
+    add_common_options(parser)
+    subparsers = parser.add_subparsers(dest="subcommand")  # this line changed
+    init.get_parser(subparsers)
+    setup.get_parser(subparsers)
+    flow.get_parser(subparsers)
+    cleanup.get_parser(subparsers)
+    export.get_parser(subparsers)
+    env.get_parser(subparsers)
+    models.get_parser(subparsers)
+    if args:
+        args = parser.parse_args(args)
+    else:
+        args = parser.parse_args()
+    handle_logging_flags(args)
+    handle_docker(args)
+
+    if hasattr(args, "func"):
+        args.func(args)
+    else:
+        print("Invalid subcommand for `mlonmcu`!")
+        parser.print_help(sys.stderr)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(args=sys.argv[1:]))  # pragma: no cover

mlonmcu/cli/models.py

@@ -0,0 +1,48 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Command line subcommand for managing models."""
+
+from mlonmcu.cli.common import add_common_options, add_context_options
+from mlonmcu.context.context import MlonMcuContext
+import mlonmcu.models
+
+
+def add_models_options(parser):
+    models_parser = parser.add_argument_group("models options")
+    models_parser.add_argument(
+        "--detailed",
+        default=False,
+        action="store_true",
+        help="Display more information (default: %(default)s)",
+    )
+
+
+def get_parser(subparsers):
+    """ "Define and return a subparser for the models subcommand."""
+    parser = subparsers.add_parser("models", description="Manage ML on MCU models.")
+    parser.set_defaults(func=handle)
+    add_common_options(parser)
+    add_context_options(parser)
+    add_models_options(parser)
+    return parser
+
+
+def handle(args):
+    with MlonMcuContext(path=args.home, deps_lock="read") as context:
+        mlonmcu.models.print_summary(context, detailed=args.detailed)

mlonmcu/cli/run.py

@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Command line subcommand for the run process."""
+
+from mlonmcu.cli.common import kickoff_runs
+from mlonmcu.cli.compile import (
+    handle as handle_compile,
+    add_compile_options,
+)
+from mlonmcu.context.context import MlonMcuContext
+from mlonmcu.session.run import RunStage
+
+
+def add_run_options(parser):
+    add_compile_options(parser)
+    _ = parser.add_argument_group("run options")
+
+
+def get_parser(subparsers):
+    """ "Define and return a subparser for the run subcommand."""
+    parser = subparsers.add_parser(
+        "run",
+        description="Run model using ML on MCU flow. This is meant to reproduce the behavior"
+        + " of the original `run.py` script in older versions of mlonmcu.",
+    )
+    parser.set_defaults(flow_func=handle)
+    add_run_options(parser)
+    return parser
+
+
+def check_args(context, args):
+    # print("CHECK ARGS")
+    pass
+
+
+def handle(args):
+    with MlonMcuContext(path=args.home, deps_lock="read") as context:
+        check_args(context, args)
+        handle_compile(args, context)
+        kickoff_runs(args, RunStage.RUN, context)

mlonmcu/cli/setup.py

@@ -0,0 +1,136 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Command line subcommand for installing a mlonmcu environment."""
+from mlonmcu.context.context import MlonMcuContext
+from mlonmcu.setup import setup
+
+from mlonmcu.cli.common import (
+    add_common_options,
+    add_context_options,
+)
+
+# from .helper.parse import extract_config_and_init_features
+from .helper.parse import extract_config_and_feature_names
+
+
+def add_setup_options(parser):
+    setup_parser = parser.add_argument_group("setup options")
+    setup_parser.add_argument(
+        "-f",
+        "--feature",
+        type=str,
+        metavar="FEATURE",
+        # nargs=1,
+        action="append",
+        choices=[],  # FIXME: only setup features?
+        help="Enabled features for setup (choices: %(choices)s)",
+    )
+    setup_parser.add_argument(
+        "-c",
+        "--config",
+        metavar="KEY=VALUE",
+        nargs="+",
+        action="append",
+        help=(
+            "Set a number of key-value pairs "
+            "(do not put spaces before or after the = sign). "
+            "If a value contains spaces, you should define "
+            "it with double quotes: "
+            'foo="this is a sentence". Note that '
+            "values are always treated as strings."
+        ),
+    )
+    setup_parser.add_argument(
+        "-p",
+        "--progress",
+        action="store_true",
+        help="Display progress bar (default: %(default)s)",
+    )
+    setup_parser.add_argument(
+        "-r",
+        "--rebuild",
+        action="store_true",
+        help="Trigger a rebuild/refresh of already installed dependencies (default: %(default)s)",
+    )
+    setup_parser.add_argument(
+        "-l",
+        "--list",
+        action="store_true",
+        help="Only print a list of the tasks to be processed and quit (default: %(default)s)",
+    )
+    setup_parser.add_argument(
+        "-g",
+        "--generate-requirements",
+        action="store_true",
+        help="Generate requirements_addition.txt listing the dependent python packages.",
+    )
+    setup_parser.add_argument(
+        "--task",
+        type=str,
+        nargs=1,
+        default=None,
+        help="Invoke a single task manually by name (default: %(default)s)",
+    )
+    setup_parser.add_argument(
+        "--visualize",
+        type=str,
+        nargs=1,
+        help="Export a .dot file for the dependency graph.",
+    )
+
+
+def get_parser(subparsers):
+    """ "Define and return a subparser for the setup subcommand."""
+    parser = subparsers.add_parser("setup", description="Setup ML on MCU dependencies.")
+    parser.set_defaults(func=handle)
+    add_common_options(parser)
+    add_context_options(parser)
+    add_setup_options(parser)
+    return parser
+
+
+def handle(args):
+    with MlonMcuContext(path=args.home, deps_lock="write") as context:
+        # config, features = extract_config_and_init_features(args)
+        config, _, _, _ = extract_config_and_feature_names(args)
+        # installer = setup.Setup(features=features, config=config, context=context)
+        installer = setup.Setup(config=config, context=context)
+        if args.visualize:
+            installer.visualize(args.visualize[0], ordered=False)
+        elif args.list:
+            order = installer.get_dependency_order()
+            print("The following tasks have been selected:")
+            print("\n".join(["- " + key for key in order]))
+        elif args.task:
+            installer.invoke_single_task(
+                args.task[0],
+                progress=args.progress,
+                rebuild=args.rebuild,
+            )
+        elif args.generate_requirements:
+            installer.generate_requirements()
+            print(
+                "Next you should run `python -m pip install -r $MLONMCU_HOME/requirements_addition.txt` inside your"
+                " virtual environment."
+            )
+        else:
+            installer.install_dependencies(
+                progress=args.progress,
+                rebuild=args.rebuild,
+            )

mlonmcu/cli/tune.py

@@ -0,0 +1,48 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Command line subcommand for the tune stage."""
+
+from mlonmcu.cli.common import kickoff_runs
+from mlonmcu.cli.build import add_build_options, handle as handle_build
+from mlonmcu.context.context import MlonMcuContext
+from mlonmcu.session.run import RunStage
+
+
+def get_parser(subparsers, parent=None):
+    """ "Define and return a subparser for the tune subcommand."""
+    parser = subparsers.add_parser(
+        "tune",
+        description="Tune model using the ML on MCU flow.",
+        parents=[parent] if parent else [],
+        add_help=(parent is None),
+    )
+    parser.set_defaults(flow_func=handle)
+    add_build_options(parser)
+    return parser
+
+
+def handle(args, ctx=None):
+    if ctx:
+        handle_build(args, ctx, require_target=True)
+    else:
+        # args.features.append("autotune")  # TODO: enable autotuning automatically?
+        # args.features.append("autotuned")  # ?
+        with MlonMcuContext(path=args.home, deps_lock="read") as context:
+            handle_build(args, context, require_target=True)
+            kickoff_runs(args, RunStage.TUNE, context)