lbkit 0.8.7__tar.gz → 0.8.9__tar.gz

{lbkit-0.8.7/lbkit.egg-info → lbkit-0.8.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lbkit
-Version: 0.8.7
+Version: 0.8.9
 Summary: Tools provided by litebmc.com
 Home-page: https://www.litebmc.com
 Author: xuhj@litebmc.com

lbkit-0.8.9/lbkit/__init__.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.8.9'

{lbkit-0.8.7 → lbkit-0.8.9}/lbkit/cli.py

@@ -16,6 +16,7 @@ from lbkit.ci_robot.gitee import Gitee
 from lbkit.log import Logger
 from lbkit import misc
 from lbkit import errors
+from lbkit.ukr.build import UKRBuild
 from lbkit.utils.env_detector import EnvDetector
 
 log = Logger("cli")
@@ -78,7 +79,10 @@ class Command(object):
             组件需要支持多种跨平台构建场景，典型的包括DT（X86-64）、交叉编译（arm64）
         """
         env = EnvDetector()
-        if env.component:
+        if env.ukr:
+            ukr = UKRBuild(env)
+            ukr.run()
+        elif env.component:
             os.chdir(env.component.folder)
             print(os.getcwd())
             arg_parser = ArgParser.new()

{lbkit-0.8.7 → lbkit-0.8.9}/lbkit/tasks/config.py

@@ -145,6 +145,36 @@ class Config(object):
             return product_cfg
         return Config.merge_cfg(global_cfg, product_cfg)
 
+    def _trans_dependencies_to_dict(self, deps):
+        out = {}
+        for dep in deps:
+            name = dep["package"]
+            name = name.split("/")[0]
+            out[name] = dep
+        return out
+
+    def _merge_dependencies(self, dst, src: dict[str, dict]):
+        for key, val in src.items():
+            dst[key] = val
+        return dst
+
+    def get_dependencies(self):
+        debug_cfg = self.get_manifest_config("debug_dependencies", [])
+        debug_dict = self._trans_dependencies_to_dict(debug_cfg)
+        global_cfg = self.get_manifest_config("dependencies", [])
+        global_dict = self._trans_dependencies_to_dict(global_cfg)
+        # debug_cfg优先级更高
+        tmp = self._merge_dependencies(global_dict, debug_dict)
+        key = f"products/{self.product}/dependencies"
+        # product_cfg优先级最高
+        product_cfg = self.get_manifest_config(key, [])
+        product_dict = self._trans_dependencies_to_dict(product_cfg)
+        tmp = self._merge_dependencies(tmp, product_dict)
+        out = []
+        for _, value in tmp.items():
+            out.append(value)
+        return out
+
     def load_manifest(self):
         """加载manifest.yml并验证schema文件"""
         template = {}

{lbkit-0.8.7 → lbkit-0.8.9}/lbkit/tasks/task_build_manifest.py

@@ -91,7 +91,7 @@ class TaskClass(Task):
         # 使用模板生成litebmc组件的配置
         lookup = TemplateLookup(directories=os.path.join(src_cwd, "template"))
         template = lookup.get_template("conanfile.py.mako")
-        conanfile = template.render(lookup=lookup, pkg=manifest)
+        conanfile = template.render(lookup=lookup, pkg=manifest, real_dependencies=self.config.get_dependencies())
 
         recipe = os.path.join(self.conan_build, "litebmc")
         os.makedirs(recipe, exist_ok=True)

{lbkit-0.8.7 → lbkit-0.8.9}/lbkit/tasks/task_download.py

@@ -27,7 +27,7 @@ class DownloadTask():
         self.dst = os.path.join(dst_dir, self.file)
         self.dst = os.path.realpath(self.dst)
         if not self.dst.startswith(dst_dir):
-            raise Exception("Download {self.file} failed because file contain relative paths")
+            raise Exception(f"Download {self.file} failed because file contain relative paths")
         dir = os.path.dirname(self.dst)
         if not os.path.isdir(dir):
             os.makedirs(dir)

{lbkit-0.8.7 → lbkit-0.8.9}/lbkit/tasks/template/conanfile.py.mako

@@ -17,13 +17,13 @@ class LitebmcConan(ConanFile):
 
     def requirements(self):
         """从manifest.yml文件中提取的依赖组件"""
-    % for dep in pkg["dependencies"]:
+    % for dep in real_dependencies:
         self.requires("${dep["package"]}")
     % endfor
         self.requires("rootfs_df190c/0.0.1")
 
     def configure(self):
-    % for dep in pkg["dependencies"]:
+    % for dep in real_dependencies:
         % if len(dep.get("options", [])) > 0:
 <% name = dep["package"].split("/")[0] %>\
             % for op, ctx in dep["options"].items():

lbkit-0.8.9/lbkit/ukr/build.py

@@ -0,0 +1,202 @@
+"""组件构建"""
+import os
+import yaml
+import shutil
+import re
+import json
+import tempfile
+from string import Template
+from multiprocessing import Pool
+import traceback
+from argparse import ArgumentParser
+from jsonschema import validate, ValidationError
+from git import Repo
+from git.exc import InvalidGitRepositoryError
+from mako.lookup import TemplateLookup
+from lbkit.misc import Color, load_yml_with_json_schema_validate, get_json_schema_file, load_json_schema
+from lbkit import errors
+from lbkit.codegen.codegen import CodeGen, history_versions
+from lbkit.tools import Tools
+from lbkit.build_conan_parallel import BuildConanParallel
+from lbkit.codegen.codegen import Version
+from lbkit.tasks.task_download import DownloadTask
+from lbkit.misc import DownloadFlag
+from lbkit.tools import Tools
+from lbkit.utils.env_detector import EnvDetector
+
+
+tools = Tools("comp_build")
+log = tools.log
+lb_cwd = os.path.split(os.path.realpath(__file__))[0]
+
+class SourceDest():
+    def __init__(self, cfg):
+        self.source = cfg.get("source")
+        self.dest = cfg.get("dest")
+
+    def copy(self, source_dir, dest_dir, with_template, **kwargs):
+        source = os.path.join(source_dir, self.source)
+        dest = os.path.join(dest_dir, self.dest)
+        if os.path.isfile(dest):
+            os.unlink(dest)
+        dest_dir = os.path.dirname(dest)
+        if not os.path.isdir(dest_dir):
+            os.makedirs(dest_dir)
+        log.info(f"cp {source} to  {dest}")
+        if with_template:
+            with open(source, "r") as fp:
+                template = Template(fp.read())
+            content = template.safe_substitute(kwargs)
+            with open(dest, "w+") as fp:
+                fp.write(content)
+        else:
+            shutil.copyfile(source, dest)
+
+
+class BuildGeneral():
+    def __init__(self, config, cwd, cfg_key, dir_name):
+        self.cwd = cwd
+        self.name = cfg_key
+        self.work_dir = os.path.join(cwd, ".temp")
+        self.output = os.path.join(self.work_dir, "output")
+        if not os.path.isdir(self.output):
+            os.makedirs(self.output)
+        os.chdir(self.work_dir)
+        self.arch = config.get("base").get("arch")
+        self.cross_compile = config.get("base").get("cross_compile")
+        self.cfg = config.get(cfg_key)
+        self.dir_name = os.path.realpath(dir_name)
+        self.tools = Tools(self.dir_name + ".log")
+
+    def download(self):
+        tmp_file = self.dir_name + ".tar.gz"
+        url = self.cfg.get("url")
+        sha256 = self.cfg.get("sha256")
+        verify = self.cfg.get("verify", True)
+        strip_components = self.cfg.get("strip_components")
+        cfg = {
+            "url": url,
+            "file": tmp_file,
+            "decompress": {
+                "dirname": os.getcwd(),
+                "strip_components": strip_components
+            },
+            "sha256": sha256,
+            "verify": verify
+        }
+        task = DownloadTask(cfg, os.getcwd())
+        task.start()
+        _, sha = DownloadFlag.read(self.dir_name)
+        if sha == sha256:
+            return
+        cmd = f"tar -xf {task.dst} -C {self.dir_name}"
+        if task.strip_components:
+            cmd += f" --strip-components={task.strip_components}"
+        if os.path.isdir(self.dir_name):
+            shutil.rmtree(self.dir_name)
+        os.makedirs(self.dir_name)
+        self.tools.exec(cmd)
+        DownloadFlag.create(self.dir_name, url, sha256)
+
+    def prepare_defconfig(self):
+        defconf = self.cfg.get("defconfig")
+        compiler_path=os.path.join(self.work_dir, "toolchain")
+        sd = SourceDest(defconf)
+        sd.copy(self.cwd, os.path.join(self.work_dir, self.dir_name), True, compiler_path=compiler_path)
+        os.environ["ARCH"] = self.arch
+        os.environ["CROSS_COMPILE"] = self.cross_compile + "-"
+        path = os.environ.get("PATH", "")
+        if compiler_path not in path:
+            path += ":" + compiler_path + "/bin"
+            os.environ["PATH"] = path
+        self.defconfig = os.path.basename(sd.dest)
+
+    def build(self):
+        os.chdir(self.dir_name)
+        cmd = f"make {self.defconfig}"
+        self.tools.exec(cmd, verbose=True)
+        if self.name == "compiler":
+            cmd = f"make sdk -j" + str(os.cpu_count())
+        else:
+            cmd = f"make -j" + str(os.cpu_count())
+        self.tools.exec(cmd, verbose=True)
+
+    def tar_files(self):
+        os.chdir(self.dir_name)
+        cfgs = self.cfg.get("tar", [])
+        for cfg in cfgs:
+            sd = SourceDest(cfg)
+            src = os.path.join(self.dir_name, sd.source)
+            dst = os.path.join(self.dir_name, sd.dest)
+            cmd = f"tar -czf {dst} -C {src} ."
+            self.tools.exec(cmd)
+
+    def package(self):
+        os.chdir(self.dir_name)
+        cfgs = self.cfg.get("output", [])
+        for cfg in cfgs:
+            sd = SourceDest(cfg)
+            sd.copy(self.dir_name, self.output, False)
+
+    def run(self):
+        self.download()
+        self.prepare_defconfig()
+        self.build()
+        self.tar_files()
+        self.package()
+        os.chdir(self.cwd)
+
+class BuildCompiler(BuildGeneral):
+    def __init__(self, config, cwd):
+        super().__init__(config, cwd, "compiler", "buildroot")
+
+    def package(self):
+        super().package()
+        cfgs = self.cfg.get("output", [])
+        for cfg in cfgs:
+            sd = SourceDest(cfg)
+            dest = os.path.join(self.output, sd.dest)
+            install_path = os.path.join(self.work_dir, "toolchain")
+            if os.path.isdir(install_path):
+                shutil.rmtree(install_path)
+            os.makedirs(install_path)
+            cmd = f"tar -xzf {dest} -C {install_path} --strip-components=1"
+            self.tools.exec(cmd)
+
+class BuildRootfs(BuildGeneral):
+    def __init__(self, config, cwd):
+        super().__init__(config, cwd, "buildroot", "buildroot")
+
+    def build(self):
+        os.chdir(self.dir_name)
+        shutil.rmtree("output", ignore_errors=True)
+        return super().build()
+
+class BuildLinux(BuildGeneral):
+    def __init__(self, config, cwd):
+        super().__init__(config, cwd, "linux", "linux")
+
+class BuildUBoot(BuildGeneral):
+    def __init__(self, config, cwd):
+        super().__init__(config, cwd, "uboot", "uboot")
+
+
+class UKRBuild():
+    def __init__(self, env_detector: EnvDetector):
+        os.chdir(env_detector.ukr.folder)
+        self.env_detector = env_detector
+
+    def run(self):
+        cwd = os.getcwd()
+        with open("config.yml") as fp:
+            cfg = yaml.full_load(fp)
+        build = BuildCompiler(cfg, cwd)
+        build.run()
+        build = BuildRootfs(cfg, cwd)
+        build.run()
+        build = BuildLinux(cfg, cwd)
+        build.run()
+        build = BuildUBoot(cfg, cwd)
+        build.run()
+        cmd = f"tar -czf {cwd}/.temp/output/firmware.tar.gz -C {cwd}/.temp/output/images ."
+        tools.exec(cmd)

{lbkit-0.8.7 → lbkit-0.8.9}/lbkit/utils/env_detector.py

@@ -3,6 +3,7 @@
 import os
 import configparser
 from lbkit.tools import Tools
+from lbkit.misc import load_yml_with_json_schema_validate
 
 tools = Tools("EnvDetector")
 log = tools.log
@@ -19,16 +20,30 @@ class LbkitManifest(object):
         self.folder = os.path.realpath(folder)
         self.config = config
 
+class UKR(object):
+    def __init__(self, folder, config: configparser.ConfigParser):
+        self.folder = os.path.realpath(folder)
+        self.config = config
+
 
 class EnvDetector(object):
     def __init__(self):
         """初始化"""
         self.component: LbkitComponent = None
         self.manifest: LbkitManifest = None
+        self.ukr: UKR = None
         self.cwd = os.getcwd()
         """探测环境"""
         cwd = self.cwd
         while cwd != "/":
+            conf_file = os.path.join(cwd, ".lbkit.yml")
+            if os.path.isfile(conf_file):
+                conf = load_yml_with_json_schema_validate(conf_file, "/usr/share/litebmc/schema/lbk_config.v1.json")
+                ukr_conf = conf.get("uboot_kernel_rootfs")
+                if ukr_conf:
+                    folder = ukr_conf.get("folder")
+                    self.ukr = UKR(os.path.join(cwd, folder), ukr_conf)
+                    return
             if os.path.isfile(os.path.join(cwd, "manifest.yml")):
                 self.manifest = LbkitManifest(cwd, None)
                 return

{lbkit-0.8.7 → lbkit-0.8.9/lbkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lbkit
-Version: 0.8.7
+Version: 0.8.9
 Summary: Tools provided by litebmc.com
 Home-page: https://www.litebmc.com
 Author: xuhj@litebmc.com

{lbkit-0.8.7 → lbkit-0.8.9}/lbkit.egg-info/SOURCES.txt

@@ -54,6 +54,8 @@ lbkit/tasks/image_maker/make_qemu_image.py
 lbkit/tasks/image_maker/make_rockchip_image.py
 lbkit/tasks/template/conanfile.py.mako
 lbkit/tasks/template/rootfs.py.mako
+lbkit/ukr/__init__.py
+lbkit/ukr/build.py
 lbkit/utils/__init__.py
 lbkit/utils/env_detector.py
 lbkit/utils/images/__init__.py

lbkit-0.8.7/lbkit/__init__.py

@@ -1,2 +0,0 @@
-
-__version__ = '0.8.7'