mlonmcu 0.3.0.dev0__3-none-any.whl

mlonmcu/environment/list.py

@@ -0,0 +1,81 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+import configparser
+from datetime import datetime as dt
+
+# TODO: EnvironmentList/Factory/Registry? also via code?
+from .config import get_environments_file
+
+
+def validate_name(name):
+    # TODO: regex for valid names without spaces etc
+    return True
+
+
+def get_environments_map():
+    result = {}
+    environments_file = get_environments_file()
+    if os.path.isfile(environments_file):
+        parser = configparser.ConfigParser()
+        parser.read(environments_file)
+        for env_name in parser.sections():
+            result[env_name] = parser[env_name]
+    return result
+
+
+def get_environment_names():
+    envs_dict = get_environments_map()
+    return envs_dict.keys()
+
+
+def get_alternative_name(name, names):
+    current = name
+    i = -1
+    while current in names:
+        i = i + 1
+        if i == 0:
+            current = current + "_" + str(i)
+        else:
+            temp = current.split("_")
+            current = "_".join(temp[:-1]) + "_" + str(i)
+    return current
+
+
+def register_environment(name, path, overwrite=False):
+    validate_name(name)
+    if not os.path.isabs(path):
+        raise RuntimeError("Not an absolute path")
+
+    environments_file = get_environments_file()
+    if not os.path.isfile(environments_file):
+        raise RuntimeError("Environments file does not yet exist")
+
+    # with open(environments_file, "a") as envs_file:
+    #    envs_file.write(name + "=" + path)
+    config = configparser.ConfigParser()
+    config.read(environments_file)
+    now = dt.now()
+    created = now.strftime("%Y%d%mT%H%M%S")
+    if name in config.sections() and not overwrite:
+        raise RuntimeError(f"Environment with name {name} does already exist")
+    config[name] = {"path": path, "created": created}
+    environments_file = get_environments_file()
+    with open(environments_file, "w") as handle:
+        config.write(handle)

mlonmcu/environment/loader.py

@@ -0,0 +1,250 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import yaml
+import pathlib
+import logging
+
+from .config import (
+    DefaultsConfig,
+    PathConfig,
+    RepoConfig,
+    FrameworkConfig,
+    FrameworkFeatureConfig,
+    BackendConfig,
+    BackendFeatureConfig,
+    TargetConfig,
+    TargetFeatureConfig,
+    PlatformConfig,
+    PlatformFeatureConfig,
+    FrontendConfig,
+    FrontendFeatureConfig,
+)
+
+# def load_environment_from_file(filename):
+#     """Utility to initialize a mlonmcu environment from a YAML file."""
+#     if isinstance(filename, str):
+#         filename = pathlib.Path(filename)
+#     with open(filename, encoding="utf-8") as yaml_file:
+#         data = yaml.safe_load(yaml_file)
+#         if data:
+#             if "home" in data:
+#                 print(data["home"], filename.parent)
+#                 assert os.path.realpath(data["home"]) == os.path.realpath(filename.parent)
+#             else:
+#                 data["home"] = filename.parent
+#             env = Environment(data)
+#             return env
+#         raise RuntimeError(f"Error opening environment file: {filename}")
+#     return None
+
+
+def load_environment_from_file(filename, base):
+    """Utility to initialize a mlonmcu environment from a YAML file."""
+    if isinstance(filename, str):
+        filename = pathlib.Path(filename)
+    with open(filename, encoding="utf-8") as yaml_file:
+        loaded = yaml.safe_load(yaml_file)
+        if not loaded:
+            raise RuntimeError("Invalid YAML contents")
+        if "home" in loaded:
+            home = loaded["home"]
+        else:
+            home = None
+        if "logging" in loaded:
+            if "level" in loaded["logging"]:
+                log_level = logging.getLevelName(loaded["logging"]["level"])
+            else:
+                log_level = None
+            if "to_file" in loaded["logging"]:
+                log_to_file = bool(loaded["logging"]["to_file"])
+            else:
+                log_to_file = None
+            if "rotate" in loaded["logging"]:
+                log_rotate = bool(loaded["logging"]["rotate"])
+            else:
+                log_rotate = None
+        else:
+            log_level = None
+            log_to_file = False
+            log_rotate = False
+        if "cleanup" in loaded:
+            if "auto" in loaded["cleanup"]:
+                cleanup_auto = bool(loaded["cleanup"]["auto"])
+            else:
+                cleanup_auto = False
+            if "auto" in loaded["cleanup"]:
+                cleanup_keep = int(loaded["cleanup"]["keep"])
+            else:
+                cleanup_keep = 100
+        else:
+            cleanup_auto = False
+            cleanup_keep = 100
+        if "paths" in loaded:
+            paths = {}
+            for key in loaded["paths"]:
+                path = loaded["paths"][key]
+                if isinstance(path, list):
+                    paths[key] = []
+                    for p in path:
+                        paths[key].append(PathConfig(p, base=home))
+                else:
+                    paths[key] = PathConfig(path, base=home)
+        else:
+            paths = None
+        if "repos" in loaded:
+            repos = {}
+            for key in loaded["repos"]:
+                repo = loaded["repos"][key]
+                if "url" not in repo:
+                    raise RuntimeError("Missing field 'url' in YAML file")
+                if "ref" in repo:
+                    repos[key] = RepoConfig(repo["url"], ref=repo["ref"])
+                else:
+                    repos[key] = RepoConfig(repo["url"])
+        else:
+            repos = None
+        default_framework = None
+        default_backends = {}
+        if "frameworks" in loaded:
+            frameworks = []
+            default_framework = loaded["frameworks"].pop("default", None)
+            for key in loaded["frameworks"]:
+                framework = loaded["frameworks"][key]
+                if "enabled" in framework:
+                    enabled = bool(framework["enabled"])
+                else:
+                    enabled = False
+                backends = []
+                if "backends" in framework:
+                    default_backend = framework["backends"].pop("default", None)
+                    default_backends[key] = default_backend
+                    for key2 in framework["backends"]:
+                        backend = framework["backends"][key2]
+                        if "enabled" in backend:
+                            enabled2 = bool(backend["enabled"])
+                        else:
+                            enabled2 = True
+                        backend_features = []
+                        if "features" in backend:
+                            for key3 in backend["features"]:
+                                supported = bool(backend["features"][key3])
+                                backend_features.append(BackendFeatureConfig(key3, backend=key2, supported=supported))
+                        backends.append(BackendConfig(key2, enabled=enabled2, features=backend_features))
+                framework_features = []
+                if "features" in framework:
+                    for key2 in framework["features"]:
+                        supported = bool(framework["features"][key2])
+                        framework_features.append(FrameworkFeatureConfig(key2, framework=key, supported=supported))
+                frameworks.append(
+                    FrameworkConfig(
+                        key,
+                        enabled=enabled,
+                        backends=backends,
+                        features=framework_features,
+                    )
+                )
+        else:
+            frameworks = None
+        if "frontends" in loaded:
+            frontends = []
+            for key in loaded["frontends"]:
+                frontend = loaded["frontends"][key]
+
+                if "enabled" in frontend:
+                    enabled = frontend["enabled"]
+                else:
+                    enabled = True
+                frontend_features = []
+                if "features" in frontend:
+                    for key2 in frontend["features"]:
+                        supported = bool(frontend["features"][key2])
+                        frontend_features.append(FrontendFeatureConfig(key2, frontend=key, supported=supported))
+                frontends.append(FrontendConfig(key, enabled=enabled, features=frontend_features))
+        else:
+            frontends = None
+        if "platforms" in loaded:
+            platforms = []
+            for key in loaded["platforms"]:
+                platform = loaded["platforms"][key]
+                if "enabled" in platform:
+                    enabled = platform["enabled"]
+                else:
+                    enabled = True
+                platform_features = []
+                if "features" in platform:
+                    for key2 in platform["features"]:
+                        supported = bool(platform["features"][key2])
+                        platform_features.append(PlatformFeatureConfig(key2, platform=key, supported=supported))
+                platforms.append(PlatformConfig(key, enabled=enabled, features=platform_features))
+        else:
+            platforms = None
+        if "toolchains" in loaded:
+            toolchains = loaded["toolchains"]
+        else:
+            toolchains = None
+        default_target = None
+        if "targets" in loaded:
+            targets = []
+            default_target = loaded["targets"].pop("default", None)
+            for key in loaded["targets"]:
+                target = loaded["targets"][key]
+                if "enabled" in target:
+                    enabled = target["enabled"]
+                else:
+                    enabled = True
+                target_features = []
+                if "features" in target:
+                    for key2 in target["features"]:
+                        supported = bool(target["features"][key2])
+                        target_features.append(TargetFeatureConfig(key2, target=key, supported=supported))
+                targets.append(TargetConfig(key, enabled=enabled, features=target_features))
+        else:
+            targets = None
+        if "vars" in loaded:
+            variables = loaded["vars"]
+        else:
+            variables = None
+        if "flags" in loaded:
+            default_flags = loaded["flags"]
+        else:
+            default_flags = None
+        defaults = DefaultsConfig(
+            log_level=log_level,
+            log_to_file=log_to_file,
+            log_rotate=log_rotate,
+            default_framework=default_framework,
+            default_backends=default_backends,
+            default_target=default_target,
+            cleanup_auto=cleanup_auto,
+            cleanup_keep=cleanup_keep,
+        )
+        env = base(
+            home,
+            defaults=defaults,
+            paths=paths,
+            repos=repos,
+            frameworks=frameworks,
+            frontends=frontends,
+            platforms=platforms,
+            toolchains=toolchains,
+            targets=targets,
+            variables=variables,
+            default_flags=default_flags,
+        )
+        return env

mlonmcu/environment/templates.py

@@ -0,0 +1,69 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Definitions of mlonmcu config templates."""
+import pkgutil
+import os
+import jinja2
+from pathlib import Path
+import pkg_resources
+
+from .config import get_config_dir
+
+
+def get_template_names():
+    template_files = pkg_resources.resource_listdir("mlonmcu", os.path.join("..", "resources", "templates"))
+    names = [name.split(".yml.j2")[0] for name in template_files]
+    return names
+
+
+def get_template_text(name):
+    return pkgutil.get_data("mlonmcu", os.path.join("..", "resources", "templates", name + ".yml.j2"))
+
+
+def fill_template(name, data={}):
+    if isinstance(name, Path):
+        name = str(name)
+    if name.endswith(".j2"):  # Template from file
+        assert Path(name).is_file(), f"Template does not exits: {name}"
+        with open(name, "r") as handle:
+            template_text = handle.read()
+    else:  # Template by name
+        template_text = get_template_text(name)
+    if template_text:
+        if not isinstance(template_text, str):
+            try:
+                template_text = template_text.decode("utf-8")
+            except UnicodeDecodeError as e:
+                raise e
+        tmpl = jinja2.Template(template_text)
+        rendered = tmpl.render(**data)
+        return rendered
+    return None
+
+
+def fill_environment_yaml(template_name, home_dir, config=None):
+    if not config:
+        config = {}
+    return fill_template(template_name, {"home_dir": str(home_dir), "config_dir": str(get_config_dir()), **config})
+
+
+def write_environment_yaml_from_template(path, template_name, home_dir, config=None):
+    with open(path, "w") as yaml:
+        text = fill_environment_yaml(template_name, home_dir, config=config)
+        yaml.write(text)

mlonmcu/environment/writer.py

@@ -0,0 +1,117 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import yaml
+import pathlib
+import logging
+
+from .config import PathConfig
+
+
+def create_environment_dict(environment):
+    data = {}
+    data["home"] = environment.home
+    data["logging"] = {
+        "level": logging.getLevelName(environment.defaults.log_level),
+        "to_file": environment.defaults.log_to_file,
+        "rotate": environment.defaults.log_rotate,
+    }
+    data["cleanup"] = {
+        "auto": environment.defaults.cleanup_auto,
+        "keep": environment.defaults.cleanup_keep,
+    }
+    data["paths"] = {
+        path: str(path_config.path)
+        if isinstance(path_config, PathConfig)
+        else [str(config.path) for config in path_config]
+        for path, path_config in environment.paths.items()
+    }  # TODO: allow relative paths
+    data["repos"] = {repo: vars(repo_config) for repo, repo_config in environment.repos.items()}
+    data["frameworks"] = {
+        "default": environment.defaults.default_framework if environment.defaults.default_framework else None,
+        **{
+            framework.name: {
+                "enabled": framework.enabled,
+                "backends": {
+                    "default": environment.defaults.default_backends[framework.name]
+                    if environment.defaults.default_backends and environment.defaults.default_backends[framework.name]
+                    else None,
+                    **{
+                        backend.name: {
+                            "enabled": backend.enabled,
+                            "features": {
+                                backend_feature.name: backend_feature.supported for backend_feature in backend.features
+                            },
+                        }
+                        for backend in framework.backends
+                    },
+                },
+                "features": {
+                    framework_feature.name: framework_feature.supported for framework_feature in framework.features
+                },
+            }
+            for framework in environment.frameworks
+        },
+    }
+    data["frontends"] = {
+        # "default": None,  # unimplemented?
+        **{
+            frontend.name: {
+                "enabled": frontend.enabled,
+                "features": {
+                    frontend_feature.name: frontend_feature.supported for frontend_feature in frontend.features
+                },
+            }
+            for frontend in environment.frontends
+        },
+    }
+    data["platforms"] = {
+        # "default": None,  # unimplemented?
+        **{
+            platform.name: {
+                "enabled": platform.enabled,
+                "features": {
+                    platform_feature.name: platform_feature.supported for platform_feature in platform.features
+                },
+            }
+            for platform in environment.platforms
+        },
+    }
+    data["toolchains"] = environment.toolchains
+    data["targets"] = {
+        "default": environment.defaults.default_target,
+        **{
+            target.name: {
+                "enabled": target.enabled,
+                "features": {target_feature.name: target_feature.supported for target_feature in target.features},
+            }
+            for target in environment.targets
+        },
+    }
+    data["vars"] = environment.vars
+    data["flags"] = environment.flags
+    return data
+
+
+def write_environment_to_file(environment, filename):
+    """Utility to initialize a mlonmcu environment from a YAML file."""
+    if isinstance(filename, str):
+        filename = pathlib.Path(filename)
+    data = create_environment_dict(environment)
+    with open(filename, "w") as yaml_file:
+        yaml.dump(data, yaml_file, default_flow_style=False, sort_keys=False)

mlonmcu/feature/__init__.py

@@ -0,0 +1,18 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#