lbkit 0.8.4__tar.gz → 0.8.6__tar.gz

lbkit-0.8.6/MANIFEST.in

@@ -0,0 +1,3 @@
+recursive-include lbkit/codegen/template *.mako
+recursive-include lbkit/tasks/template *.mako
+recursive-include lbkit/component/template *.mako

{lbkit-0.8.4/lbkit.egg-info → lbkit-0.8.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: lbkit
-Version: 0.8.4
+Version: 0.8.6
 Summary: Tools provided by litebmc.com
 Home-page: https://www.litebmc.com
 Author: xuhj@litebmc.com

lbkit-0.8.6/lbkit/__init__.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.8.6'

{lbkit-0.8.4 → lbkit-0.8.6}/lbkit/cli.py

@@ -10,16 +10,13 @@ from lbkit import __version__ as client_version
 from lbkit.codegen.codegen import CodeGen
 from lbkit.component.build import BuildComponent
 from lbkit.component.test import TestComponent
-from lbkit.integration.build_manifest import BuildManifest
-from lbkit.integration.build_rootfs import BuildRootfs
-from lbkit.integration.build_prepare import BuildPrepare
-from lbkit.integration.build_image import BuildImage
+from lbkit.tasks.executor import Executor
 from lbkit.component.arg_parser import ArgParser
-from lbkit.integration.config import Config
 from lbkit.ci_robot.gitee import Gitee
 from lbkit.log import Logger
 from lbkit import misc
 from lbkit import errors
+from lbkit.utils.env_detector import EnvDetector
 
 log = Logger("cli")
 
@@ -80,26 +77,14 @@ class Command(object):
 
             组件需要支持多种跨平台构建场景，典型的包括DT（X86-64）、交叉编译（arm64）
         """
-        arg_parser = ArgParser.new()
-        build = BuildComponent(arg_parser, sys.argv[2:])
-        build.run()
-
-    # create product package
-    def create(self, *args):
-        """
-        构建组件.
-
-            组件需要支持多种跨平台构建场景，典型的包括DT（X86-64）、交叉编译（arm64）
-        """
-        cfg = Config(sys.argv[2:])
-        build = BuildPrepare(cfg, "create_pre")
-        build.run()
-        build = BuildManifest(cfg, "create_run")
-        build.run()
-        build = BuildRootfs(cfg, "create_rootfs")
-        build.run()
-        build = BuildImage(cfg, "create_image")
-        build.run()
+        env = EnvDetector()
+        if env.component:
+            arg_parser = ArgParser.new()
+            build = BuildComponent(arg_parser, sys.argv[2:])
+            build.run()
+        elif env.manifest:
+            exe = Executor(env)
+            exe.run()
 
     def gitee(self, *args):
         """
@@ -126,7 +111,7 @@ class Command(object):
         """
         grps = [("Code Generate commands", ["gen"]),
                 ("Build Component commands", ["new", "build", "test"]),
-                ("Build Product commands", ["create"]),
+                ("Build Product commands", ["build"]),
                 ("Misc commands", ["help"]),
                 ("CI Robot commands", ["gitee"])
                ]

{lbkit-0.8.4 → lbkit-0.8.6}/lbkit/codegen/template/server.c.mako

@@ -86,7 +86,7 @@ static LBInterface _${class_name}_interface = {
     % if intf.plugin.install_dir:
     .plugin_dir = "${intf.plugin.install_dir}",
     % else:
-    .plugin_dir = NULL,
+    .plugin_dir = "/opt/litebmc/plugins/${class_name}",
     % endif
 #endif
 };

{lbkit-0.8.4 → lbkit-0.8.6}/lbkit/log.py

@@ -12,6 +12,7 @@ class Logger(logging.getLoggerClass()):
         self.logenv = os.environ.get("LOG")
         if self.logenv is None:
             formatter = logging.Formatter('%(message)s')
+            self.setLevel(logging.INFO)
         else:
             formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
             if self.logenv == "info":

{lbkit-0.8.4 → lbkit-0.8.6}/lbkit/misc.py

@@ -5,12 +5,14 @@ import json
 import yaml
 import re
 import requests
+import fcntl
 from string import Template
 from colorama import Fore, Style
 from jsonschema import validate, ValidationError
 from lbkit.errors import PackageConfigException, HttpRequestException
 
 LOG_DIR = "/tmp/lbkit/log"
+TARGETS_DIR = "/usr/share/litebmc/targets"
 
 
 class Color(object):
@@ -103,3 +105,43 @@ def load_yml_with_json_schema_validate(yml_file, default_json_schema_file, **kwa
         raise PackageConfigException(f"validate {yml_file} failed, schema file is {schema_file}, "
                                      f"message: {exc.message}\n"
                                      "installing redhat.vscode-yaml plugin in vscode will help you write odf files")
+
+class DownloadFlag():
+    @staticmethod
+    def clean(filename):
+        """清理文件标记"""
+        filename += ".flag"
+        fp = open(filename, "a+")
+        fcntl.flock(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        fp.truncate(0)
+        fcntl.flock(fp, fcntl.F_UNLCK)
+        fp.close()
+
+    @staticmethod
+    def create(filename, url, new_hash):
+        """创建文件标记"""
+        filename += ".flag"
+        fp = open(filename, "a+")
+        fcntl.flock(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        fp.seek(0, 0)
+        fp.truncate(0)
+        fp.write(url + "|" + new_hash)
+        fcntl.flock(fp, fcntl.F_UNLCK)
+        fp.close()
+
+    @staticmethod
+    def read(filename):
+        """读取文件标记"""
+        filename += ".flag"
+        if not os.path.isfile(filename):
+            return "", ""
+        fp = open(filename, "a+")
+        fcntl.flock(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        fp.seek(0, 0)
+        content = fp.read()
+        fcntl.flock(fp, fcntl.F_UNLCK)
+        fp.close()
+        if len(content) == 0:
+            return "", ""
+        chunk = content.split("|")
+        return chunk[0], chunk[1]

lbkit-0.8.6/lbkit/tasks/config.py

@@ -0,0 +1,187 @@
+"""集成构建配置项"""
+import argparse
+import os, sys
+from lbkit.log import Logger
+from lbkit.misc import load_yml_with_json_schema_validate, TARGETS_DIR
+
+log = Logger("build_config")
+
+
+class Config(object):
+    """集成构建的配置项"""
+    def __init__(self, args = None):
+        parser = self.arg_parser()
+        args = parser.parse_args(args)
+
+        # 配置项
+        self.manifest = os.path.join(os.getcwd(), args.manifest)
+        self.manifest = os.path.realpath(self.manifest)
+        # 配置项目录
+        self.work_dir = os.path.dirname(self.manifest)
+        sys.path.append(self.work_dir)
+        # 是否从源码构建
+        self.from_source = args.from_source
+        # 是否打印详细信息
+        self.verbose = True if os.environ.get("VERBOSE", False) else False
+        # 编译类型
+        self.build_type = args.build_type
+        # conan中心仓
+        self.remote = args.remote
+
+        if not os.path.isfile(self.manifest):
+            raise FileNotFoundError(f"File {args.manifest} not exist")
+
+        # 编译主机配置项
+        self.profile_build = args.profile_build
+        # 待所有参数确认后会调用refresh_profile_name设置正确的profile
+        self.profile_host = None
+
+        # conan.lock options
+        self.using_lockfile = args.lockfile
+        self.update_lockfile = args.update_lockfile
+        self.target = args.target
+        self.product = args.product
+
+        # 设置并创建构建所需目录
+        log.info("Work dir: %s", self.work_dir)
+        self.code_path = os.getcwd()
+        self.temp_path = os.path.join(self.code_path, ".temp")
+        self.output_path = os.path.join(self.temp_path, "output")
+        self.download_path = os.path.join(self.temp_path, "download")
+        self.tool_path = os.path.join(self.temp_path, "tools")
+        self.compiler_path = os.path.join(self.tool_path, "compiler")
+        # conan组件打包目录
+        self.conan_install = []
+        self.mnt_path = os.path.join(self.temp_path, "mnt_path")
+        self.rootfs_img = os.path.join(self.output_path, "rootfs.img")
+        # rootfs、uboot和kernel关键文件路径
+        self.uboot_bin = None
+        self.linux_bin = None
+        self.rootfs_tar = None
+        os.makedirs(self.temp_path, exist_ok=True)
+        os.makedirs(self.tool_path, exist_ok=True)
+        os.makedirs(self.output_path, exist_ok=True)
+        os.makedirs(self.download_path, exist_ok=True)
+        os.makedirs(self.compiler_path, exist_ok=True)
+        # 制作rootfs时需要strip镜像，所以需要单独指定stip路径
+        self.strip = "strip"
+        self.check_product()
+        # 刷新conan profile
+        self.refresh_profile_name()
+
+    @staticmethod
+    def target_list():
+        targets = {}
+        dirname = os.path.join(TARGETS_DIR)
+        for file in os.listdir(dirname):
+            if not file.endswith(".yml"):
+                continue
+            tgt_file = os.path.join(TARGETS_DIR, file)
+            targets[file] = tgt_file
+        return targets
+
+    @staticmethod
+    def arg_parser():
+        """返回配置项支持的参数"""
+        parser = argparse.ArgumentParser(description="Build LiteBMC", formatter_class=argparse.RawTextHelpFormatter)
+        parser.add_argument("-m", "--manifest", help="Specify the manifest.yml, ignored when -l is specified.", default="./manifest.yml")
+        parser.add_argument("-s", "--from_source", help="Build from source", action="store_true")
+        parser.add_argument("-pr:b", "--profile_build", help="Apply the specified profile to the build machine", default="default")
+        parser.add_argument("-bt", "--build_type", type=str, choices=['debug', 'release', 'minsize'], help="Set the build type", default="debug")
+        parser.add_argument("-r", "--remote", help="specified conan server", default="litebmc")
+        parser.add_argument("-l", "--lockfile", help="using conan.lock", action="store_true")
+        parser.add_argument("-ul", "--update_lockfile", help="update conan.lock", action="store_true")
+        parser.add_argument("-p", "--product", help="product name, default product is `default`", default="default")
+        targets = Config.target_list()
+        target_help = "build target:"
+        for tgt, _ in targets.items():
+            target_help += "\n* " + tgt[:-4]
+        parser.add_argument("-t", "--target", help=target_help, default="default")
+        return parser
+
+    def get_manifest_config(self, key: str, default=None):
+        """从manifest中读取配置"""
+        manifest = self.load_manifest()
+        keys = key.split("/")
+        for k in keys:
+            manifest = manifest.get(k, None)
+            if manifest is None:
+                return default
+        return manifest
+
+    @staticmethod
+    def merge_cfg(dst, src):
+        """合并两个配置项"""
+        # 如果源为None，则返回dst
+        if src is None:
+            return dst
+        # 如果目标是空的，直接返回src
+        if not dst:
+            return src
+        # 如果是数组、标量的直接覆盖dst即可，所以返回src
+        if not isinstance(dst, dict):
+            return src
+        # 如果目标非空，但源是空，直接返回目标
+        if not src:
+            return dst
+        if not isinstance(src, dict):
+            raise Exception(f"Merge configuration failed, source config {src} is not a dictionary")
+        output = {}
+        for key, val in src.items():
+            dst_val = dst.get(key)
+            output[key] = Config.merge_cfg(dst_val, val)
+        for key, val in dst.items():
+            # 已经合并过，即目标中存在，但源不存在，直接合并
+            if key in output:
+                continue
+            output[key] = val
+        return output
+
+    def get_product_config(self, key: str, default=None):
+        """获取产品配置，注意，key只需要基于products/[name]/即可，如 toolchain"""
+        global_cfg = self.get_manifest_config(key, default)
+        key = f"products/{self.product}/{key}"
+        product_cfg = self.get_manifest_config(key, None)
+        if not product_cfg:
+            return global_cfg
+        if not global_cfg:
+            return product_cfg
+        return Config.merge_cfg(global_cfg, product_cfg)
+
+    def load_manifest(self):
+        """加载manifest.yml并验证schema文件"""
+        template = {}
+        template["code_path"] = self.code_path
+        template["temp_path"] = self.temp_path
+        template["download_path"] = os.path.join(self.download_path)
+        return load_yml_with_json_schema_validate(self.manifest, "/usr/share/litebmc/schema/pdf.v1.json", **template)
+
+    def check_product(self):
+        products = self.get_manifest_config("products", {})
+        if products.get(self.product):
+            return
+        log.error(f"Only the following products are supported:")
+        for key, _ in products.items():
+            log.info("    * " + key)
+        # todo: 待manifest.yml整改到位到删除下面注释
+        # raise Exception(f"Unkown product {self.product}")
+
+    def refresh_profile_name(self):
+        self.profile_host = self.get_product_config("toolchain/profile/name", "litebmc")
+
+    def set_build_type(self, value):
+        self.build_type = value
+
+    def deal_conf(self, config_dict):
+        """
+        处理Target级别的配置"target_config"
+        当Config类有set_xxx类方法时，则可以在target文件中配置xxx
+        """
+        if not config_dict or not isinstance(config_dict, dict):
+            return
+        for key, conf in config_dict.items():
+            try:
+                method = getattr(self, f"set_{key}")
+                method(conf)
+            except Exception as e:
+                raise Exception(f"目标 config 无效配置: {key}") from e

lbkit-0.8.6/lbkit/tasks/executor.py

@@ -0,0 +1,258 @@
+#! /usr/bin/env python3
+
+import importlib
+import os
+import time
+import traceback
+import sys
+
+from multiprocessing import Process
+from multiprocessing import Manager
+from lbkit.errors import LiteBmcException
+from lbkit.tools import Tools
+from lbkit.tasks.config import Config
+from lbkit.utils.env_detector import EnvDetector
+import lbkit.misc as misc
+from lbkit.misc import load_yml_with_json_schema_validate
+
+# 任务失败状态
+TASK_STATUS_FAILED = "Failed"
+TASK_STATUS_SUCCED = "succed"
+TASK_STATUS_EXCEPT = "Except"
+TASK_STATUS_RUNNING = "Runing"
+tool = Tools("executor")
+log = tool.log
+manager = Manager()
+status_dict = manager.dict()
+status_lock = manager.Lock()
+
+
+def wait_finish(target_name, wait_list):
+    """
+    等待任务结束
+    """
+    if not wait_list:
+        return True
+    start_time = time.time()
+    cnt = 0
+    while True:
+        finish = True
+        time.sleep(0.1)
+        for work_name in wait_list:
+            cur_time = time.time()
+            key = target_name + "/" + work_name
+            status = status_dict.get(key)
+            if status is None:
+                log.warn(f"等待不存在的任务{key}。如果要等待一个任务，这个任务必须在当前任务之前运行，否则触发异常")
+                return False
+            if status == TASK_STATUS_SUCCED:
+                continue
+            if status == TASK_STATUS_FAILED or status == TASK_STATUS_EXCEPT:
+                return False
+            finish = False
+            # 每等待60s打印一次日志
+            if int(cur_time - start_time) >= 60:
+                start_time = time.time()
+                cnt += 60
+                log.info("目标 {} 正在等待任务: {}, 当前已等待 {} 秒".format(target_name, work_name, cnt))
+            break
+        if finish:
+            return True
+
+
+class TaskExecutor():
+    '''
+    '''
+    def __init__(self, target_name, work, config: Config):
+        super().__init__()
+        self.work = work
+        self.config: Config = config
+        self.target_name = target_name
+        self.work_name = self.work.get("task", "")
+        self.status_key = target_name + "/" + self.work_name
+        chunks = self.work_name.split(".", -1)
+        if len(chunks) == 1:
+            self.task_path = ""
+            self.work_name = chunks[1]
+        else:
+            self.task_path = "lbkit.tasks." + chunks[0]
+            self.work_name = chunks[1]
+        self.exception = None
+
+    def load_class(self):
+        if not self.task_path:
+            return None
+        log.debug("工作路径: {}".format(self.task_path))
+        work_py_file = importlib.import_module(self.task_path)
+        return getattr(work_py_file, "TaskClass")
+
+    def run(self):
+        '''
+        功能描述：执行任务
+        '''
+        work_name = self.work_name
+        log.debug(f"任务{self.status_key}已就绪")
+        ret = wait_finish(self.target_name, self.work.get("wait"))
+        if not ret:
+            log.debug(f"任务{self.status_key}等待的其它任务发生错误")
+            return -1
+        work_class = self.load_class()
+        # 如果未指定类时，不需要执行
+        if work_class is not None:
+            work_x = work_class(self.config, work_name)
+            # work配置项和target配置项
+            work_config = self.work.get("config")
+            work_x.deal_conf(work_config)
+            with status_lock:
+                status = status_dict.get(self.status_key)
+                if status is None:
+                    status_dict[self.status_key] = TASK_STATUS_RUNNING
+            if status is None:
+                # 创建进程并且等待完成或超时
+                ret = work_x.run()
+                if ret is not None and ret != 0:
+                    return -1
+            else:
+                # 不需要创建进程，等待任务执行完成即可
+                wait_list = []
+                wait_list.append(work_name)
+                ret = wait_finish(self.target_name, wait_list)
+                if not ret:
+                    log.debug(f"任务{self.status_key}等待的其它任务发生错误")
+                    return -1
+
+            log.debug(f"任务 {work_name} 开始安装步骤")
+
+        # 创建子任务
+        ret = exec_works(self.work.get("subtasks", []), self.config, os.cpu_count())
+        ret = ret and exec_works(self.work.get("seqtasks", []), self.config, 1)
+        if not ret:
+            status_dict[self.status_key] = TASK_STATUS_FAILED
+            return -1
+
+        log.success(f"任务 {work_name} 完成")
+        with status_lock:
+            status_dict[self.status_key] = TASK_STATUS_SUCCED
+        return 0
+
+def task_handler(te:TaskExecutor):
+    try:
+        ret = te.run()
+    except Exception as e:
+        log.error(traceback.print_exc())
+        log.error(f"Task {te.status_key} exit with exceiption: {str(e)}")
+        with status_lock:
+            status_dict[te.status_key] = TASK_STATUS_EXCEPT
+        return -1
+    if ret != 0:
+        with status_lock:
+            status_dict[te.status_key] = TASK_STATUS_FAILED
+        return -1
+    else:
+        with status_lock:
+            status_dict[te.status_key] = TASK_STATUS_SUCCED
+        return 0
+
+
+class TaskInfo():
+    def __init__(self, te: TaskExecutor, proc: Process):
+        self.te = te
+        self.proc = proc
+
+
+def wait_tasks(tasks: dict[str, TaskInfo], alow_processes_alive=0):
+    while True:
+        new_results = {}
+        cnt = 0
+        killall = False
+        for key, ti in tasks.items():
+            if ti.proc.is_alive():
+                new_results[key] = ti
+                cnt += 1
+                continue
+            with status_lock:
+                status = status_dict.get(ti.te.status_key)
+            if status == TASK_STATUS_EXCEPT:
+                killall = True
+                break
+        if killall:
+            for _, ti in tasks.items():
+                if ti.proc.is_alive():
+                    ti.proc.terminate()
+            return None, False
+        tasks = new_results
+        if cnt > 0 and cnt >= alow_processes_alive:
+            time.sleep(0.1)
+        else:
+            return new_results, True
+
+def exec_works(work_list, config, processes):
+    if not work_list:
+        return True
+    # 创建任务并等待完成
+    results: dict[str, TaskInfo] = {}
+    for work in work_list:
+        te = TaskExecutor(config.target, work, config)
+        result = Process(target=task_handler, args=(te, ))
+        results[te.status_key] = TaskInfo(te, result)
+        result.start()
+        results, ok = wait_tasks(results, processes)
+        if not ok:
+            return False
+    _, ok = wait_tasks(results, 0)
+    return ok
+
+
+def target_executor(config):
+    log.info(f"创建新目标 {config.target} 构建计划表")
+    manifest_target = f"{config.code_path}/targets/{config.target}.yml"
+    lbkit_target = os.path.join(misc.TARGETS_DIR, config.target + ".yml")
+    if os.path.isfile(manifest_target):
+        target_file = manifest_target
+    elif os.path.isfile(lbkit_target):
+        target_file = lbkit_target
+    else:
+        raise Exception(f"构建目标文件 [target_]{config.target}.yml 不存在")
+
+    # 读取配置
+    work_list = load_yml_with_json_schema_validate(target_file, os.path.join(misc.TARGETS_DIR, "tdf.v1.json"))
+    target_cfg = work_list.get("config", {})
+    config.deal_conf(target_cfg)
+    environments = work_list.get("env", {})
+    for key, value in environments.items():
+        log.success(f"配置环境变量 {key}: {value}")
+        os.environ[key] = value
+    # 打印任务清单
+    log.debug(f"任务列表:{work_list}")
+    # 创建任务调度器
+    ret = exec_works(work_list.get("subtasks", []), config, os.cpu_count())
+    ret = ret and exec_works(work_list.get("seqtasks", []), config, 1)
+    return ret
+
+
+class Executor(object):
+    def __init__(self, env: EnvDetector):
+        if not env.manifest:
+            raise LiteBmcException("未找到manifest.yml配置文件，当前目录不是一个合法的产品配置仓")
+        os.chdir(env.manifest.folder)
+
+    def run(self):
+        target = ""
+        succ = False
+        try:
+            config = Config(sys.argv[2:])
+            target = config.target
+            succ = target_executor(config)
+        except Exception as e:
+            log.error(str(e))
+            raise Exception(f"任务 {target} 执行失败")
+        if succ:
+            log.success(f"任务 {target} 执行成功")
+            return 0
+        else:
+            raise Exception(f"任务 {target} 执行失败")
+
+if __name__ == "__main__":
+    env = EnvDetector()
+    exec = Executor(env)
+    exec.run()