mlonmcu 0.3.0.dev0__3-none-any.whl

mlonmcu/cli/compile.py

@@ -0,0 +1,87 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Command line subcommand for the run process."""
+
+from mlonmcu.cli.common import kickoff_runs
+from mlonmcu.cli.build import (
+    handle as handle_build,
+    add_build_options,
+)
+from mlonmcu.context.context import MlonMcuContext
+from mlonmcu.session.run import RunStage
+from mlonmcu.platform.lookup import get_platforms_targets
+from .helper.parse import extract_target_names, extract_platform_names, extract_config_and_feature_names
+
+
+def add_compile_options(parser):
+    add_build_options(parser)
+    # compile_parser = parser.add_argument_group("compile options")
+
+
+def get_parser(subparsers):
+    """ "Define and return a subparser for the compile subcommand."""
+    parser = subparsers.add_parser("compile", description="Compile model using ML on MCU flow.")
+    parser.set_defaults(flow_func=handle)
+    add_compile_options(parser)
+    return parser
+
+
+def _handle(args, context):
+    handle_build(args, ctx=context)
+    targets = extract_target_names(args, context=context)  # This will eventually be ignored below
+    platforms = extract_platform_names(args, context=context)
+
+    new_config, _, _, _ = extract_config_and_feature_names(args, context=context)
+    platform_targets = get_platforms_targets(context, config=new_config)  # This will slow?
+
+    assert len(context.sessions) > 0  # TODO: automatically request session if no active one is available
+    session = context.sessions[-1]
+    new_runs = []
+    for run in session.runs:
+        if run.target is None:
+            # assert run.compile_platform is None
+            targets_ = targets
+        else:
+            targets_ = [None]
+        for target_name in targets_:
+            new_run = run.copy()
+            if target_name is not None:
+                platform_name = None
+                for platform in platforms:
+                    candidates = platform_targets[platform]
+                    if target_name in candidates:
+                        platform_name = platform
+                new_run.add_platform_by_name(platform_name, context=context)
+                new_run.add_target_by_name(target_name, context=context)
+            new_runs.append(new_run)
+    session.runs = new_runs
+
+
+def check_args(context, args):
+    # print("CHECK ARGS")
+    pass
+
+
+def handle(args, ctx=None):
+    if ctx:
+        _handle(args, ctx)
+    else:
+        with MlonMcuContext(path=args.home, deps_lock="read") as context:
+            _handle(args, context)
+            kickoff_runs(args, RunStage.COMPILE, context)

mlonmcu/cli/env.py

@@ -0,0 +1,68 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Command line subcommand for manaing environments."""
+
+import configparser
+
+from mlonmcu.environment.config import get_environments_file
+
+
+def get_parser(subparsers):
+    """ "Define and return a subparser for the env subcommand."""
+    parser = subparsers.add_parser("env", description="List ML on MCU environments.")
+    parser.set_defaults(func=handle)
+    return parser
+
+
+class EnvironmentHint:
+    def __init__(self, name, path, created_at=None):
+        self.name = name
+        self.path = path
+        self.created_at = created_at
+
+
+def lookup_user_environments(file):
+    config = configparser.ConfigParser()
+    config.read(file)
+    hints = []
+    for env_name in config.sections():
+        env = config[env_name]
+        assert "path" in env
+        path = env["path"]
+        created_at = None
+        if "created" in env:
+            created_at = env["created"]
+        hint = EnvironmentHint(env_name, path, created_at=created_at)
+        hints.append(hint)
+    return hints
+
+
+def handle(args):
+    envs_file = get_environments_file()
+    print(f"Looking for user environments config file: {envs_file}")
+    envs = lookup_user_environments(envs_file)
+    count = len(envs)
+    if not envs or count == 0:
+        print("No mlonmcu environments were found for the current user. You can create a new one using `mlonmcu init`")
+    else:
+        print(f"Found {count} mlonmcu environment(s):")
+        for env in envs:
+            print(f"    - {env.name} ({env.path})")
+
+        print("Point the environment variable MLONMCU_HOME to a environment directory to get started")

mlonmcu/cli/export.py

@@ -0,0 +1,93 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Command line subcommand for exporting session and runs."""
+from mlonmcu.context.context import MlonMcuContext
+
+from mlonmcu.cli.common import (
+    add_common_options,
+    add_context_options,
+)
+
+
+def add_export_options(parser):
+    export_parser = parser.add_argument_group("export options")
+    export_parser.add_argument(
+        "destination",
+        nargs="?",
+        default="exported.zip",
+        help="Path to the output directory or archive (default: %(default)s)",
+    )
+    export_parser.add_argument(
+        "-f",
+        "--force",
+        default=False,
+        action="store_true",
+        help="Overwrite files if the destination already exists (DANGEROUS)",
+    )
+    export_parser.add_argument(
+        "-l",
+        "--list",
+        default=False,
+        action="store_true",
+        help="Print a summary of all available sessions and runs",
+    )
+    export_parser.add_argument(
+        "-s",
+        "--session",
+        metavar="SESSION",
+        type=int,
+        nargs="?",
+        default=None,
+        const=-1,
+        action="append",
+        help="Which session(s) should be exported (default: latest session id)",
+    )
+    export_parser.add_argument(
+        "-r",
+        "--run",
+        metavar="RUN",
+        type=int,
+        nargs="?",
+        default=None,
+        const=-1,
+        action="append",
+        help="Which run(s) should be exported (default: all runs of the selected session/latest run id)",
+    )
+
+
+def get_parser(subparsers):
+    """ "Define and return a subparser for the cleanup subcommand."""
+    parser = subparsers.add_parser("export", description="Export session/run artifacts to a directory/archive.")
+    parser.set_defaults(func=handle)
+    add_common_options(parser)
+    add_context_options(parser)
+    add_export_options(parser)
+    return parser
+
+
+def handle(args):
+    with MlonMcuContext(path=args.home, deps_lock="read") as context:
+        if args.list:
+            context.print_summary(sessions=True, runs=True, labels=True)
+            return 0
+        dest = args.destination
+        interactive = not args.force
+        sids = args.session
+        rids = args.run
+        context.export(dest, session_ids=sids, run_ids=rids, interactive=interactive)

mlonmcu/cli/flow.py

@@ -0,0 +1,67 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Command line subcommand for invoking the mlonmcu flow."""
+
+import sys
+
+from mlonmcu.context.context import MlonMcuContext
+import mlonmcu.platform.lookup
+import mlonmcu.cli.load as load
+import mlonmcu.cli.tune as tune
+import mlonmcu.cli.build as build
+import mlonmcu.cli.compile as compile_  # compile is a builtin name
+import mlonmcu.cli.run as run
+from mlonmcu.cli.common import add_flow_options, add_common_options, add_context_options
+
+
+def get_parser(subparsers, parent=None):
+    """ "Define and return a subparser for the flow subcommand."""
+    parser = subparsers.add_parser(
+        "flow",
+        description="Invoke ML on MCU flow",
+        parents=[parent] if parent else [],
+        add_help=(parent is None),
+    )
+    parser.set_defaults(func=handle)
+    add_common_options(parser)
+    add_context_options(parser)
+    add_flow_options(parser)
+    subparsers = parser.add_subparsers(dest="subcommand2")  # this line changed
+    load.get_parser(subparsers)
+    tune.get_parser(subparsers)
+    build.get_parser(subparsers)
+    compile_.get_parser(subparsers)
+    run.get_parser(subparsers)
+
+
+def handle_list_targets(args):
+    with MlonMcuContext(path=args.home, deps_lock="read") as context:
+        mlonmcu.platform.lookup.print_summary(context=context)
+
+
+def handle(args):
+    """Callback function which will be called to process the flow subcommand"""
+    if args.list_targets:
+        handle_list_targets(args)
+    else:
+        if hasattr(args, "flow_func"):
+            args.flow_func(args)
+        else:
+            print("Invalid command. Check 'mlonmcu flow --help' for the available subcommands!")
+            sys.exit(1)

mlonmcu/cli/helper/__init__.py

@@ -0,0 +1,18 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

mlonmcu/cli/helper/filter.py

@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+def filter_arg(arg):
+    """TODO"""
+    if not arg:
+        return []
+    return list(filter(None, arg.split(",")))

mlonmcu/cli/helper/parse.py

@@ -0,0 +1,159 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+def parse_var(s):
+    """
+    Parse a key, value pair, separated by '='
+    That's the reverse of ShellArgs.
+
+    On the command line (argparse) a declaration will typically look like:
+        foo=hello
+    or
+        foo="hello world"
+    """
+    items = s.split("=")
+    key = items[0].strip()  # we remove blanks around keys, as is logical
+    if len(key) == 0 or len(items) <= 1 or "=" not in s:
+        raise RuntimeError(f"The argument {s} is not a key-value pair")
+    if len(items) > 1:
+        # rejoin the rest:
+        value = "=".join(items[1:])
+    return (key, value)
+
+
+def parse_vars(items):  # TODO: this needs to be used in other subcommands as well?
+    """
+    Parse a series of key-value pairs and return a dictionary
+    """
+    d = {}
+
+    if items:
+        for item in items:
+            if len(item) > 0:
+                key, value = parse_var(item)
+                d[key] = value
+    return d
+
+
+def extract_feature_names(args):
+    if args.feature:
+        features = args.feature
+    else:
+        features = []
+    if hasattr(args, "feature_gen") and args.feature_gen:
+        gen = []
+        for x in args.feature_gen:
+            if "_" in x:
+                assert len(x) == 1
+                gen.append([])
+            else:
+                gen.append(x)
+    else:
+        gen = [[]]
+    return features, gen
+
+
+def extract_config(args):
+    if args.config:
+        configs = sum(args.config, [])
+        configs = parse_vars(configs)
+    else:
+        configs = {}
+    if hasattr(args, "config_gen") and args.config_gen:
+        gen = []
+        for x in args.config_gen:
+            if "_" in x:
+                assert len(x) == 1
+                gen.append({})
+            else:
+                c = parse_vars(x)
+                gen.append(c)
+    else:
+        gen = [{}]
+
+    return configs, gen
+
+
+def extract_config_and_feature_names(args, context=None):
+    # TODO: get features from context?
+    feature_names, feature_gen = extract_feature_names(args)
+    config, config_gen = extract_config(args)
+    return config, feature_names, config_gen, feature_gen
+
+
+def extract_frontend_names(args, context=None):
+    frontend_names = args.frontend
+    names = []
+    if isinstance(frontend_names, list) and len(frontend_names) > 0:
+        names = frontend_names
+    elif isinstance(frontend_names, str):
+        names = [frontend_names]
+    else:
+        # No need to specify a default, because we just use the provided order in the environment.yml
+        assert frontend_names is None, "TODO"
+        assert context is not None, "Need context to resolve default frontends"
+        all_frontend_names = context.environment.lookup_frontend_configs(names_only=True)
+        names.extend(all_frontend_names)
+    return names
+
+
+def extract_postprocess_names(args, context=None):
+    return list(dict.fromkeys(args.postprocess)) if args.postprocess is not None else []
+
+
+def extract_backend_names(args, context=None):
+    if isinstance(args.backend, list) and len(args.backend) > 0:
+        backends = args.backend
+    elif isinstance(args.backend, str):
+        backends = [args.backend]
+    else:
+        assert args.backend is None, "TODO"
+        assert context is not None
+        frameworks = context.environment.get_default_frameworks()
+        backends = []
+        for framework in frameworks:
+            framework_backends = context.environment.get_default_backends(framework)
+            backends.extend(framework_backends)
+    return backends
+
+
+def extract_target_names(args, context=None):
+    if isinstance(args.target, list) and len(args.target) > 0:
+        targets = args.target
+    elif isinstance(args.target, str):
+        targets = [args.target]
+    else:
+        assert args.target is None, "TODO"
+        # assert context is not None
+        if context is None:
+            return [None]
+        targets = context.environment.get_default_targets()
+    return targets
+
+
+def extract_platform_names(args, context=None):
+    if args.platform:
+        platforms = [x[0] for x in args.platform]
+    else:
+        assert args.platform is None
+        if context is None:
+            return [None]
+        platforms = context.environment.lookup_platform_configs(names_only=True)
+    return platforms

mlonmcu/cli/init.py

@@ -0,0 +1,130 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Command line subcommand for initializing a mlonmcu environment."""
+
+import os
+
+from mlonmcu.environment.templates import get_template_names
+from mlonmcu.environment.config import get_environments_dir, DEFAULTS
+from mlonmcu.environment.init import initialize_environment
+from .helper.parse import extract_config
+
+
+def add_init_options(parser):
+    init_parser = parser.add_argument_group("init options")
+    init_parser.add_argument(
+        "-n",
+        "--name",
+        metavar="NAME",
+        nargs=1,
+        type=str,
+        default="",
+        help="Environment name (default: %(default)s)",
+    )
+    init_parser.add_argument(
+        "-t",
+        "--template",
+        metavar="TEMPLATE",
+        nargs=1,
+        type=str,
+        # choices=get_template_names(),
+        default=DEFAULTS["template"],
+        help="Environment template name or path (default: %(default)s, available: "
+        + ", ".join(get_template_names())
+        + ")",
+    )
+    init_parser.add_argument(
+        "DIR",
+        nargs="?",
+        type=str,
+        default=get_environments_dir(),
+        help="Environment directory (default: " + os.path.join(get_environments_dir(), "{NAME}") + ")",
+    )
+    init_parser.add_argument(
+        "--non-interactive",
+        dest="non_interactive",
+        action="store_true",
+        help="Do not ask questions interactively",
+    )
+    init_parser.add_argument(
+        "--venv",
+        default=None,
+        action="store_true",
+        help="Create virtual python environment",
+    )
+    init_parser.add_argument(
+        "--clone-models",
+        dest="clone_models",
+        default=None,
+        action="store_true",
+        help="Clone models directory into environment",
+    )
+    init_parser.add_argument(
+        "--register",
+        default=None,
+        action="store_true",
+        help="Add environment to the list of environment for the current user",
+    )
+    init_parser.add_argument(
+        "--allow-exists",
+        dest="allow_exists",
+        default=None,
+        action="store_true",
+        help="Allow overwriting an existing environment directory",
+    )
+    init_parser.add_argument(
+        "-c",
+        "--config",
+        metavar="KEY=VALUE",
+        nargs="+",
+        action="append",
+        help=(
+            "Set a number of key-value pairs "
+            "(do not put spaces before or after the = sign). "
+            "If a value contains spaces, you should define "
+            "it with double quotes: "
+            'foo="this is a sentence". Note that '
+            "values are always treated as strings."
+        ),
+    )
+
+
+def get_parser(subparsers):
+    """ "Define and return a subparser for the init subcommand."""
+    parser = subparsers.add_parser("init", description="Initialize ML on MCU environment.")
+    parser.set_defaults(func=handle)
+    add_init_options(parser)
+    return parser
+
+
+def handle(args):
+    """Callback function which will be called to process the init subcommand"""
+    name = args.name[0] if isinstance(args.name, list) else args.name
+    config, _ = extract_config(args)
+    initialize_environment(
+        args.DIR,
+        name,
+        create_venv=args.venv,
+        interactive=not args.non_interactive,
+        clone_models=args.clone_models,
+        register=args.register,
+        template=args.template[0] if isinstance(args.template, list) else args.template,
+        allow_exists=args.allow_exists,
+        config=config,
+    )