lbkit 0.8.5__tar.gz → 0.8.6__tar.gz

{lbkit-0.8.5/lbkit.egg-info → lbkit-0.8.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: lbkit
-Version: 0.8.5
+Version: 0.8.6
 Summary: Tools provided by litebmc.com
 Home-page: https://www.litebmc.com
 Author: xuhj@litebmc.com

lbkit-0.8.6/lbkit/__init__.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.8.6'

{lbkit-0.8.5 → lbkit-0.8.6}/lbkit/misc.py

@@ -5,6 +5,7 @@ import json
 import yaml
 import re
 import requests
+import fcntl
 from string import Template
 from colorama import Fore, Style
 from jsonschema import validate, ValidationError
@@ -104,3 +105,43 @@ def load_yml_with_json_schema_validate(yml_file, default_json_schema_file, **kwa
         raise PackageConfigException(f"validate {yml_file} failed, schema file is {schema_file}, "
                                      f"message: {exc.message}\n"
                                      "installing redhat.vscode-yaml plugin in vscode will help you write odf files")
+
+class DownloadFlag():
+    @staticmethod
+    def clean(filename):
+        """清理文件标记"""
+        filename += ".flag"
+        fp = open(filename, "a+")
+        fcntl.flock(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        fp.truncate(0)
+        fcntl.flock(fp, fcntl.F_UNLCK)
+        fp.close()
+
+    @staticmethod
+    def create(filename, url, new_hash):
+        """创建文件标记"""
+        filename += ".flag"
+        fp = open(filename, "a+")
+        fcntl.flock(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        fp.seek(0, 0)
+        fp.truncate(0)
+        fp.write(url + "|" + new_hash)
+        fcntl.flock(fp, fcntl.F_UNLCK)
+        fp.close()
+
+    @staticmethod
+    def read(filename):
+        """读取文件标记"""
+        filename += ".flag"
+        if not os.path.isfile(filename):
+            return "", ""
+        fp = open(filename, "a+")
+        fcntl.flock(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        fp.seek(0, 0)
+        content = fp.read()
+        fcntl.flock(fp, fcntl.F_UNLCK)
+        fp.close()
+        if len(content) == 0:
+            return "", ""
+        chunk = content.split("|")
+        return chunk[0], chunk[1]

{lbkit-0.8.5 → lbkit-0.8.6}/lbkit/tasks/config.py

@@ -15,6 +15,7 @@ class Config(object):
 
         # 配置项
         self.manifest = os.path.join(os.getcwd(), args.manifest)
+        self.manifest = os.path.realpath(self.manifest)
         # 配置项目录
         self.work_dir = os.path.dirname(self.manifest)
         sys.path.append(self.work_dir)
@@ -32,13 +33,14 @@ class Config(object):
 
         # 编译主机配置项
         self.profile_build = args.profile_build
-        # 编译目标配置项
-        self.profile_host = args.profile
+        # 待所有参数确认后会调用refresh_profile_name设置正确的profile
+        self.profile_host = None
 
         # conan.lock options
         self.using_lockfile = args.lockfile
         self.update_lockfile = args.update_lockfile
         self.target = args.target
+        self.product = args.product
 
         # 设置并创建构建所需目录
         log.info("Work dir: %s", self.work_dir)
@@ -47,6 +49,7 @@ class Config(object):
         self.output_path = os.path.join(self.temp_path, "output")
         self.download_path = os.path.join(self.temp_path, "download")
         self.tool_path = os.path.join(self.temp_path, "tools")
+        self.compiler_path = os.path.join(self.tool_path, "compiler")
         # conan组件打包目录
         self.conan_install = []
         self.mnt_path = os.path.join(self.temp_path, "mnt_path")
@@ -59,8 +62,12 @@ class Config(object):
         os.makedirs(self.tool_path, exist_ok=True)
         os.makedirs(self.output_path, exist_ok=True)
         os.makedirs(self.download_path, exist_ok=True)
+        os.makedirs(self.compiler_path, exist_ok=True)
         # 制作rootfs时需要strip镜像，所以需要单独指定stip路径
         self.strip = "strip"
+        self.check_product()
+        # 刷新conan profile
+        self.refresh_profile_name()
 
     @staticmethod
     def target_list():
@@ -79,12 +86,12 @@ class Config(object):
         parser = argparse.ArgumentParser(description="Build LiteBMC", formatter_class=argparse.RawTextHelpFormatter)
         parser.add_argument("-m", "--manifest", help="Specify the manifest.yml, ignored when -l is specified.", default="./manifest.yml")
         parser.add_argument("-s", "--from_source", help="Build from source", action="store_true")
-        parser.add_argument("-pr", "--profile", help="Apply the specified profile to the host machine", default="litebmc.ini")
         parser.add_argument("-pr:b", "--profile_build", help="Apply the specified profile to the build machine", default="default")
         parser.add_argument("-bt", "--build_type", type=str, choices=['debug', 'release', 'minsize'], help="Set the build type", default="debug")
         parser.add_argument("-r", "--remote", help="specified conan server", default="litebmc")
         parser.add_argument("-l", "--lockfile", help="using conan.lock", action="store_true")
         parser.add_argument("-ul", "--update_lockfile", help="update conan.lock", action="store_true")
+        parser.add_argument("-p", "--product", help="product name, default product is `default`", default="default")
         targets = Config.target_list()
         target_help = "build target:"
         for tgt, _ in targets.items():
@@ -102,6 +109,45 @@ class Config(object):
                 return default
         return manifest
 
+    @staticmethod
+    def merge_cfg(dst, src):
+        """合并两个配置项"""
+        # 如果源为None，则返回dst
+        if src is None:
+            return dst
+        # 如果目标是空的，直接返回src
+        if not dst:
+            return src
+        # 如果是数组、标量的直接覆盖dst即可，所以返回src
+        if not isinstance(dst, dict):
+            return src
+        # 如果目标非空，但源是空，直接返回目标
+        if not src:
+            return dst
+        if not isinstance(src, dict):
+            raise Exception(f"Merge configuration failed, source config {src} is not a dictionary")
+        output = {}
+        for key, val in src.items():
+            dst_val = dst.get(key)
+            output[key] = Config.merge_cfg(dst_val, val)
+        for key, val in dst.items():
+            # 已经合并过，即目标中存在，但源不存在，直接合并
+            if key in output:
+                continue
+            output[key] = val
+        return output
+
+    def get_product_config(self, key: str, default=None):
+        """获取产品配置，注意，key只需要基于products/[name]/即可，如 toolchain"""
+        global_cfg = self.get_manifest_config(key, default)
+        key = f"products/{self.product}/{key}"
+        product_cfg = self.get_manifest_config(key, None)
+        if not product_cfg:
+            return global_cfg
+        if not global_cfg:
+            return product_cfg
+        return Config.merge_cfg(global_cfg, product_cfg)
+
     def load_manifest(self):
         """加载manifest.yml并验证schema文件"""
         template = {}
@@ -110,6 +156,19 @@ class Config(object):
         template["download_path"] = os.path.join(self.download_path)
         return load_yml_with_json_schema_validate(self.manifest, "/usr/share/litebmc/schema/pdf.v1.json", **template)
 
+    def check_product(self):
+        products = self.get_manifest_config("products", {})
+        if products.get(self.product):
+            return
+        log.error(f"Only the following products are supported:")
+        for key, _ in products.items():
+            log.info("    * " + key)
+        # todo: 待manifest.yml整改到位到删除下面注释
+        # raise Exception(f"Unkown product {self.product}")
+
+    def refresh_profile_name(self):
+        self.profile_host = self.get_product_config("toolchain/profile/name", "litebmc")
+
     def set_build_type(self, value):
         self.build_type = value
 

{lbkit-0.8.5 → lbkit-0.8.6}/lbkit/tasks/executor.py

@@ -3,6 +3,7 @@
 import importlib
 import os
 import time
+import traceback
 import sys
 
 from multiprocessing import Process
@@ -138,6 +139,7 @@ def task_handler(te:TaskExecutor):
     try:
         ret = te.run()
     except Exception as e:
+        log.error(traceback.print_exc())
         log.error(f"Task {te.status_key} exit with exceiption: {str(e)}")
         with status_lock:
             status_dict[te.status_key] = TASK_STATUS_EXCEPT

lbkit-0.8.6/lbkit/tasks/task.py

@@ -0,0 +1,132 @@
+"""任务基础类"""
+import importlib
+import os
+import time
+import shutil
+
+from multiprocessing import Process
+from lbkit.log import Logger
+from lbkit.tools import Tools
+from lbkit.misc import load_yml_with_json_schema_validate, DownloadFlag
+
+from lbkit.tasks.config import Config
+
+class ManifestValidateError(OSError):
+    """Raised when validation manifest.yml failed."""
+
+class Task(Process):
+    """任务基础类，提供run和install默认实现以及其它基础该当"""
+    def __init__(self, config: Config, name: str):
+        super().__init__()
+        self.log: Logger = Logger("task")
+        self.tools: Tools = Tools(name)
+        self.config: Config = config
+        self.name = name
+
+    def install(self):
+        """安装任务"""
+        self.log.info("install...........")
+
+    def exec(self, cmd: str, verbose=False, ignore_error = False, sensitive=False, log_prefix="", **kwargs):
+        kwargs["uptrace"] = kwargs.get("uptrace", 0) + 1
+        return self.tools.exec(cmd, verbose, ignore_error, sensitive, log_prefix, **kwargs)
+
+    def pipe(self, cmds: list[str], ignore_error=False, out_file = None, **kwargs):
+        kwargs["uptrace"] = kwargs.get("uptrace", 0) + 1
+        self.tools.pipe(cmds, ignore_error, out_file, **kwargs)
+
+    def exec_easy(self, cmd, ignore_error=False, **kwargs):
+        kwargs["uptrace"] = kwargs.get("uptrace", 0) + 1
+        return self.tools.run(cmd, ignore_error, **kwargs)
+
+    def do_hook(self, path):
+        """执行任务钓子，用于定制化"""
+        try:
+            module = importlib.import_module(path)
+        except TypeError:
+            self.log.info("Load module(%s) failed, skip", path, uptrace=2)
+            return
+        self.log.info(f"load hook: {path}", uptrace=2)
+        hook = module.TaskHook(self.config, "do_hook")
+        hook.run()
+
+    def get_manifest_config(self, key: str, default=None):
+        return self.config.get_manifest_config(key, default)
+
+    def load_manifest(self):
+        """加载manifest.yml并验证schema文件"""
+        return self.config.load_manifest()
+
+    def waitfile(self, src, timeout=1):
+        src = os.path.realpath(src)
+        if os.path.islink(src):
+            raise FileNotFoundError(f"Source file {src} is a symlink, copying failed")
+        if not os.path.isfile(src):
+            if timeout:
+                # 如果有timeout的，可能是正在下载的文件，需要等待超时时间再检查一次
+                time.sleep(timeout)
+                if not os.path.isfile(src):
+                    raise FileNotFoundError(f"Source file {src} does not exist, copying failed")
+            else:
+                raise FileNotFoundError(f"Source file {src} does not exist, copying failed")
+
+        # 检查是否由download下载器下载，如果是，需要检查是否下载完成
+        if src.startswith(self.config.download_path):
+            timeout_cnt = 0
+            timeout_sec = 0
+            while True:
+                _, hash = DownloadFlag.read(src)
+                if hash:
+                    filehash = self.tools.file_digest_sha256(src)
+                    if hash != filehash:
+                        DownloadFlag.clean(src)
+                        raise Exception(f"The hash of file {src} is {filehash}, not equal to {hash}")
+                    break
+                else:
+                    time.sleep(0.1)
+                    timeout_sec += 0.1
+                    timeout_cnt += 0.1
+                # 每10秒打印一次等等日志
+                if timeout_sec == 10:
+                    self.log.info(f"Wait file {src} download success")
+                    timeout_sec = 0
+                # 60秒未创建文件的，判定文件不存在，中止构建
+                if timeout_cnt == 60 and not os.path.isfile(src):
+                    raise Exception(f"Check file {src} failed because source file does not exist")
+                if timeout_cnt == 300:
+                    raise Exception(f"Check file {src} failed because file was not downloaded within 300 seconds")
+
+    def copyfile(self, src, dst):
+        try:
+            self.waitfile(src)
+        except Exception as e:
+            raise Exception(f"Copy file {src} to {dst} failed because source file not ready") from e
+
+        dst = os.path.realpath(dst)
+        if not dst.startswith(self.config.temp_path):
+            raise FileNotFoundError(f"Destination file {dst} is not a subpath of {self.config.temp_path}")
+
+        if os.path.isfile(dst):
+            os.unlink(dst)
+        dst_dir = os.path.dirname(dst)
+        if not os.path.isdir(dst_dir):
+            os.makedirs(dst_dir)
+        if os.path.isfile(dst) or os.path.islink(dst):
+            os.unlink(dst)
+        self.log.debug(f"copy {src} to {dst}")
+        shutil.copyfile(src, dst)
+
+    def deal_conf(self, config_dict):
+        """
+        处理每个Task的私有配置"work_config"
+        当work类有set_xxx类方法时，则可以在target文件中配置xxx
+        """
+        if not config_dict:
+            return
+        for conf in config_dict:
+            try:
+                exist = getattr(self, f"set_{conf}")
+                val = config_dict.get(conf)
+                exist(val)
+            except Exception as e:
+                raise Exception(f"无效配置: {conf}, {e}") from e

{lbkit-0.8.5 → lbkit-0.8.6}/lbkit/tasks/task_build_image.py

@@ -41,5 +41,5 @@ class TaskClass(Task):
 
 if __name__ == "__main__":
     config = Config()
-    build = TaskClass(config)
+    build = TaskClass(config, "test")
     build.run()

{lbkit-0.8.5 → lbkit-0.8.6}/lbkit/tasks/task_build_manifest.py

@@ -159,5 +159,5 @@ class TaskClass(Task):
 
 if __name__ == "__main__":
     config = Config()
-    build = TaskClass(config)
+    build = TaskClass(config, "test")
     build.run()

lbkit-0.8.6/lbkit/tasks/task_build_prepare.py

@@ -0,0 +1,98 @@
+"""环境准备"""
+import os
+import shutil
+import jinja2
+import configparser
+from string import Template
+from lbkit.tasks.config import Config
+from lbkit.tasks.task import Task
+from lbkit.log import Logger
+from lbkit.misc import DownloadFlag
+
+log = Logger("product_prepare")
+
+
+class ManifestValidateError(OSError):
+    """Raised when validation manifest.yml failed."""
+
+src_cwd = os.path.split(os.path.realpath(__file__))[0]
+
+class TaskClass(Task):
+    def decompress_toolchain(self):
+        toolchain = self.config.get_product_config("toolchain")
+        if not toolchain:
+            # todo：toolchain成为强制配置项
+            return
+        file = toolchain.get("file")
+        strip = toolchain.get("strip_components", 0)
+        self.waitfile(file, 10)
+        compiler_file = os.path.join(self.config.compiler_path, "lbkit_cache")
+        _, hash = DownloadFlag.read(compiler_file)
+        if hash:
+            return
+        # 可能标记不匹配，所以尝试删除目录后重新创建
+        shutil.rmtree(self.config.compiler_path)
+        os.makedirs(self.config.compiler_path)
+        cmd = f"tar -xf {file} -C {self.config.compiler_path}"
+        if strip:
+            cmd += f" --strip-components={strip}"
+        self.exec(cmd)
+        sha256 = self.tools.file_digest_sha256(file)
+        DownloadFlag.create(compiler_file, file, sha256)
+
+    def get_conan_profile(self):
+        profile = self.config.get_product_config("toolchain/profile")
+        if profile:
+            file = profile.get("file")
+            name = profile.get("name")
+        else:
+            file = self.get_manifest_config("metadata/profile")
+            name = "litebmc"
+        self.waitfile(file)
+        return file, name
+
+
+    def load_conan_profile(self):
+        profile, name = self.get_conan_profile()
+        log.info("Copy profile %s", profile)
+        profiles_dir = os.path.expanduser("~/.conan2/profiles")
+        if not os.path.isdir(profiles_dir):
+            cmd = "conan profile detect -f"
+            self.exec(cmd, ignore_error=True)
+        dst_profile = os.path.join(profiles_dir, name)
+        with open(dst_profile, "w+") as dst_fp:
+            src_fd = open(profile, "r")
+            template = Template(src_fd.read())
+            src_fd.close()
+            content = template.safe_substitute(compiler_path=self.config.compiler_path)
+            dst_fp.write(content)
+
+        with open(dst_profile, "r") as fp:
+            profile_data = jinja2.Template(fp.read()).render()
+            parser = configparser.ConfigParser()
+            parser.read_string(profile_data)
+            strip = "strip"
+            if parser.has_option("buildenv", "STRIP"):
+                strip = parser.get("buildenv", "STRIP")
+            path = ""
+            if parser.has_option("buildenv", "PATH+"):
+                path = parser.get("buildenv", "PATH+")
+                if path.startswith("(path)"):
+                    path = path[6:]
+            elif parser.has_option("buildenv", "PATH"):
+                path = parser.get("buildenv", "PATH")
+                if path.startswith("(path)"):
+                    path = path[6:]
+            self.config.strip = os.path.join(path, strip)
+
+    def run(self):
+        """任务入口"""
+        self.decompress_toolchain()
+        """检查manifest文件是否满足schema格式描述"""
+        self.config.load_manifest()
+        self.load_conan_profile()
+
+if __name__ == "__main__":
+    config = Config()
+    build = TaskClass(config, "test")
+    build.run()

{lbkit-0.8.5 → lbkit-0.8.6}/lbkit/tasks/task_build_rootfs.py

@@ -157,5 +157,5 @@ class TaskClass(Task):
 
 if __name__ == "__main__":
     config = Config()
-    build = TaskClass(config)
+    build = TaskClass(config, "test")
     build.run()

lbkit-0.8.6/lbkit/tasks/task_download.py

@@ -0,0 +1,129 @@
+"""环境准备"""
+import os
+import requests
+import shutil
+import time
+import traceback
+from functools import partial
+
+from lbkit.tasks.config import Config
+from lbkit.tasks.task import Task
+from lbkit.log import Logger
+from lbkit.tools import Tools
+from multiprocessing.pool import Pool, ApplyResult
+from lbkit.misc import DownloadFlag
+
+log = Logger("build_image")
+
+src_cwd = os.path.split(os.path.realpath(__file__))[0]
+
+class DownloadTask():
+    def __init__(self, record, dst_dir):
+        self.url = record.get("url")
+        self.file = record.get("file")
+        self.dst = os.path.join(dst_dir, self.file)
+        self.dst = os.path.realpath(self.dst)
+        if not self.dst.startswith(dst_dir):
+            raise Exception("Download {self.file} failed because file contain relative paths")
+        dir = os.path.dirname(self.dst)
+        if not os.path.isdir(dir):
+            os.makedirs(dir)
+        self.sha256 = record.get("sha256")
+        self.verify = record.get("verify", True)
+
+    def start(self):
+        last = time.time()
+        if os.path.isfile(self.dst):
+            calc_sha = Tools.file_digest_sha256(self.dst)
+            # 需要校验hash且hash不一致时删除文件
+            if self.sha256 != "any":
+                if self.sha256 != calc_sha:
+                    os.unlink(self.dst)
+                else:
+                    return
+            else:
+                url, hash = DownloadFlag.read(self.dst)
+                # flash标志文件记录的url、hash一致时无需重复下载
+                if  url == self.url and hash == calc_sha:
+                    return
+        print(f"Start downloading {self.file} from {self.url}")
+        DownloadFlag.clean(self.dst)
+        req = requests.get(self.url, stream=True, verify=self.verify, timeout=30)
+        req.raise_for_status()
+        total_size = int(req.headers.get('content-length', 0))
+        total_down = 0
+        fp = open(self.dst, 'wb')
+        for chunk in req.iter_content(chunk_size=16384):
+            if chunk:
+                fp.write(chunk)
+                total_down += len(chunk)
+                now = time.time()
+                # 每30秒打印一次进度
+                if now - last > 30:
+                    print(f"File {self.dst} is downloading, downloaded {total_down} / {total_size} ")
+                    last = now
+        fp.close()
+        calc_sha = Tools.file_digest_sha256(self.dst)
+        if self.sha256 != "any" and calc_sha != self.sha256:
+            os.unlink(self.dst)
+            raise Exception(f"File {self.file} downloaded but sha256 not match, need: {self.sha256}, get: {calc_sha}")
+        DownloadFlag.create(self.dst, self.url, calc_sha)
+
+
+def download_filed(error, pool: Pool):
+    print(f"download file failed, error: {str(error)}")
+    pool.terminate()
+
+
+class TaskClass(Task):
+    def run(self):
+        records = self.config.get_manifest_config("download", [])
+        if not records:
+            return
+        tasks: dict[str, DownloadTask] = {}
+        temps: list[DownloadTask] = []
+        for rec in records:
+            task =  DownloadTask(rec, self.config.download_path)
+            temps.append(task)
+            if not os.path.isfile(task.dst):
+                with open(task.dst, "w+") as _:
+                    pass
+        for task in temps:
+            if not task.url.startswith("file://"):
+                if tasks.get(task.file):
+                    raise Exception(f"manifest.yml configuration error, file {task.file} repeatedly")
+                tasks[task.file] = task
+                continue
+            DownloadFlag.clean(task.dst)
+            src = task.url[7:]
+            # 源文件不存在
+            if not os.path.isfile(src):
+                raise Exception(f"Download failed, file {task.url} not exist")
+            if os.path.isfile(task.dst):
+                os.unlink(task.dst)
+            shutil.copyfile(src, task.dst)
+            calc_hash = self.tools.file_digest_sha256(task.dst)
+            DownloadFlag.create(task.dst, task.url, calc_hash)
+        pool = Pool()
+        results: list[(ApplyResult, DownloadTask)] = []
+        for _, task in tasks.items():
+            ec = partial(download_filed, pool=pool)
+            result = pool.apply_async(task.start, error_callback=ec)
+            results.append((result, task))
+        pool.close()
+        pool.join()
+        for result in results:
+            if not result[0].ready():
+                raise Exception(f"Download file {result[1].file} failed")
+            try:
+                result[0].get()
+            except:
+                log.error("Download with exception")
+                log.error(traceback.print_exc)
+                return -1
+
+
+if __name__ == "__main__":
+    config = Config()
+    build = TaskClass(config, "test")
+    build.run()

{lbkit-0.8.5 → lbkit-0.8.6/lbkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: lbkit
-Version: 0.8.5
+Version: 0.8.6
 Summary: Tools provided by litebmc.com
 Home-page: https://www.litebmc.com
 Author: xuhj@litebmc.com

{lbkit-0.8.5 → lbkit-0.8.6}/lbkit.egg-info/SOURCES.txt

@@ -47,8 +47,8 @@ lbkit/tasks/task_build_image.py
 lbkit/tasks/task_build_manifest.py
 lbkit/tasks/task_build_prepare.py
 lbkit/tasks/task_build_rootfs.py
+lbkit/tasks/task_download.py
 lbkit/tasks/task_test.py
-lbkit/tasks/test.py
 lbkit/tasks/template/conanfile.py.mako
 lbkit/tasks/template/rootfs.py.mako
 lbkit/utils/__init__.py
@@ -57,4 +57,5 @@ lbkit/utils/images/__init__.py
 lbkit/utils/images/emmc.py
 test/__init__.py
 test/test_codegen.py
+test/test_config.py
 test/test_helper.py

lbkit-0.8.6/test/test_config.py

@@ -0,0 +1,66 @@
+import unittest
+from lbkit.tasks.config import Config
+
+class TestConfig(unittest.TestCase):
+    def test_config_merge_config(self):
+        src = "123"
+        dst = "abc"
+        out = Config.merge_cfg(dst, src)
+        self.assertEqual(out, src)
+        src = None
+        dst = "abc"
+        out = Config.merge_cfg(dst, src)
+        self.assertEqual(out, dst)
+        src = "123"
+        dst = None
+        out = Config.merge_cfg(dst, src)
+        self.assertEqual(out, src)
+        src = ["123"]
+        dst = None
+        out = Config.merge_cfg(dst, src)
+        self.assertEqual(out, src)
+        src = ["123"]
+        dst = ["abc"]
+        out = Config.merge_cfg(dst, src)
+        self.assertEqual(out, src)
+        src = []
+        dst = ["abc"]
+        out = Config.merge_cfg(dst, src)
+        src = None
+        dst = ["abc"]
+        out = Config.merge_cfg(dst, src)
+        src = None
+        dst = []
+        out = Config.merge_cfg(dst, src)
+        self.assertEqual(out, dst)
+        src = {"key1": "val1"}
+        dst = {"key1": "val2"}
+        out = Config.merge_cfg(dst, src)
+        self.assertEqual(out, src)
+        src = {"key1": "val1", "key2": [123, 234]}
+        dst = {"key1": "val2"}
+        out = Config.merge_cfg(dst, src)
+        self.assertEqual(out, src)
+        src = {"key2": [123, 234]}
+        dst = {"key1": "val2"}
+        out = Config.merge_cfg(dst, src)
+        self.assertEqual(out, {"key1": "val2", "key2": [123, 234]})
+        src = {"key2": [123, 234]}
+        dst = {"key1": "val2", "key2": [456]}
+        out = Config.merge_cfg(dst, src)
+        self.assertEqual(out, {"key1": "val2", "key2": [123, 234]})
+        src = {"key2": [123, 234]}
+        dst = {"key1": "val2", "key2": []}
+        out = Config.merge_cfg(dst, src)
+        self.assertEqual(out, {"key1": "val2", "key2": [123, 234]})
+        src = {"key2": [123, 234]}
+        dst = {"key1": "val2", "key2": None}
+        out = Config.merge_cfg(dst, src)
+        self.assertEqual(out, {"key1": "val2", "key2": [123, 234]})
+        src = {"key2": None}
+        dst = {"key1": "val2", "key2": []}
+        out = Config.merge_cfg(dst, src)
+        self.assertEqual(out, {"key1": "val2", "key2": []})
+
+if __name__ == "__main__":
+    unittest.main()

lbkit-0.8.5/lbkit/__init__.py

@@ -1,2 +0,0 @@
-
-__version__ = '0.8.5'

lbkit-0.8.5/lbkit/tasks/task.py

@@ -1,70 +0,0 @@
-"""任务基础类"""
-import importlib
-import traceback
-from multiprocessing import Process
-from lbkit.log import Logger
-from lbkit.tools import Tools
-from lbkit.misc import load_yml_with_json_schema_validate
-
-from lbkit.tasks.config import Config
-
-class ManifestValidateError(OSError):
-    """Raised when validation manifest.yml failed."""
-
-class Task(Process):
-    """任务基础类，提供run和install默认实现以及其它基础该当"""
-    def __init__(self, config: Config, name: str):
-        super().__init__()
-        self.log: Logger = Logger("task")
-        self.tools: Tools = Tools(name)
-        self.config: Config = config
-        self.name = name
-
-    def install(self):
-        """安装任务"""
-        self.log.info("install...........")
-
-    def exec(self, cmd: str, verbose=False, ignore_error = False, sensitive=False, log_prefix="", **kwargs):
-        kwargs["uptrace"] = kwargs.get("uptrace", 0) + 1
-        return self.tools.exec(cmd, verbose, ignore_error, sensitive, log_prefix, **kwargs)
-
-    def pipe(self, cmds: list[str], ignore_error=False, out_file = None, **kwargs):
-        kwargs["uptrace"] = kwargs.get("uptrace", 0) + 1
-        self.tools.pipe(cmds, ignore_error, out_file, **kwargs)
-
-    def exec_easy(self, cmd, ignore_error=False, **kwargs):
-        kwargs["uptrace"] = kwargs.get("uptrace", 0) + 1
-        return self.tools.run(cmd, ignore_error, **kwargs)
-
-    def do_hook(self, path):
-        """执行任务钓子，用于定制化"""
-        try:
-            module = importlib.import_module(path)
-        except TypeError:
-            self.log.info("Load module(%s) failed, skip", path, uptrace=2)
-            return
-        self.log.info(f"load hook: {path}", uptrace=2)
-        hook = module.TaskHook(self.config, "do_hook")
-        hook.run()
-
-    def get_manifest_config(self, key: str, default=None):
-        return self.config.get_manifest_config(key, default)
-
-    def load_manifest(self):
-        """加载manifest.yml并验证schema文件"""
-        return self.config.load_manifest()
-
-    def deal_conf(self, config_dict):
-        """
-        处理每个Task的私有配置"work_config"
-        当work类有set_xxx类方法时，则可以在target文件中配置xxx
-        """
-        if not config_dict:
-            return
-        for conf in config_dict:
-            try:
-                exist = getattr(self, f"set_{conf}")
-                val = config_dict.get(conf)
-                exist(val)
-            except Exception as e:
-                raise Exception(f"无效配置: {conf}, {e}") from e

lbkit-0.8.5/lbkit/tasks/task_build_prepare.py

@@ -1,59 +0,0 @@
-"""环境准备"""
-import os
-import shutil
-import jinja2
-import configparser
-from lbkit.tasks.config import Config
-from lbkit.tasks.task import Task
-from lbkit.log import Logger
-
-log = Logger("product_prepare")
-
-
-class ManifestValidateError(OSError):
-    """Raised when validation manifest.yml failed."""
-
-src_cwd = os.path.split(os.path.realpath(__file__))[0]
-
-class TaskClass(Task):
-    def load_conan_profile(self):
-        profile = self.get_manifest_config("metadata/profile")
-        if not os.path.isfile(profile):
-            raise FileNotFoundError(f"profile {profile} not found")
-        log.info("Copy profile %s", profile)
-        profiles_dir = os.path.expanduser("~/.conan2/profiles")
-        if not os.path.isdir(profiles_dir):
-            cmd = "conan profile detect -f"
-            self.exec(cmd, ignore_error=True)
-        dst_profile = os.path.join(profiles_dir, os.path.basename(profile))
-        if os.path.isdir(profiles_dir):
-            shutil.copyfile(profile, dst_profile, follow_symlinks=False)
-
-        with open(dst_profile, "r") as fp:
-            profile_data = jinja2.Template(fp.read()).render()
-            parser = configparser.ConfigParser()
-            parser.read_string(profile_data)
-            strip = "strip"
-            if parser.has_option("buildenv", "STRIP"):
-                strip = parser.get("buildenv", "STRIP")
-            path = ""
-            if parser.has_option("buildenv", "PATH+"):
-                path = parser.get("buildenv", "PATH+")
-                if path.startswith("(path)"):
-                    path = path[6:]
-            elif parser.has_option("buildenv", "PATH"):
-                path = parser.get("buildenv", "PATH")
-                if path.startswith("(path)"):
-                    path = path[6:]
-            self.config.strip = os.path.join(path, strip)
-
-    def run(self):
-        """任务入口"""
-        """检查manifest文件是否满足schema格式描述"""
-        self.config.load_manifest()
-        self.load_conan_profile()
-
-if __name__ == "__main__":
-    config = Config()
-    build = TaskClass(config)
-    build.run()

lbkit-0.8.5/lbkit/tasks/test.py

@@ -1,40 +0,0 @@
-from multiprocessing import Pool
-from multiprocessing.pool import ApplyResult
-import time
-from functools import partial
-
-def square(number):
-    """计算一个数的平方"""
-    print(f"计算 {number} 的平方")
-    time.sleep(1)  # 模拟耗时操作
-    if number % 10 == 9:
-        raise Exception("sssssssssss")
-    # return number * number
-
-def task_error(pool, number, error):
-    print(f"Work {number} exist with error: {error}")
-    pool.terminate()
-
-if __name__ == "__main__":
-    # 创建任务并等待完成
-    results: dict[str, ApplyResult] = {}
-    pool = Pool(3)
-    for work in range(100):
-        error_cb = partial(task_error, pool, work)
-        result = pool.apply_async(square, args=(work, ), error_callback=error_cb)
-        results[work] = result
-    # 检查任务结果
-    pool.close()
-    pool.join()
-    # print("xxxxxxxxxxxxxxxxxx")
-    # time.sleep(100)
-    for number, result in results.items():
-        try:
-            if not result.ready():
-                succ = False
-            else:
-                result.get()
-        except Exception as e:
-            print(f"任务{number}执行异常: {str(e)}")
-            succ = False
-