lbkit 0.9.3__tar.gz → 0.9.4__tar.gz

{lbkit-0.9.3 → lbkit-0.9.4}/AUTHORS

@@ -1,4 +1,4 @@
-Below are just a few of the people who have contributed to litebmc/toolkit.
+Below are just a few of the people who have contributed to litebmc/lbkit.
 
 ----------
 xuhaijun        <xuhj@litebmc.com>

{lbkit-0.9.3/lbkit.egg-info → lbkit-0.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lbkit
-Version: 0.9.3
+Version: 0.9.4
 Summary: Tools provided by litebmc.com
 Home-page: https://www.litebmc.com
 Author: xuhj@litebmc.com

lbkit-0.9.4/lbkit/__init__.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.9.4'

lbkit-0.9.4/lbkit/build_conan_parallel.py

@@ -0,0 +1,61 @@
+"""任务基础类"""
+import os
+import json
+import tempfile
+from lbkit.log import Logger
+from concurrent.futures import ThreadPoolExecutor, Future, as_completed
+from lbkit import misc
+from lbkit.tools import Tools
+from lbkit.misc import Color
+
+tools = Tools("comp_build")
+log = Logger("comp_build")
+
+
+class BuildConanParallel(object):
+    def __init__(self, orderinfo, remote):
+        self.orderinfo = orderinfo
+        self.remote = remote
+
+    def _build_package(self, logfile, build_args, public_args):
+        cmd = f"conan install {build_args} {public_args}"
+        log.info("{}build start:    '{}".format(Color.GREEN, Color.RESET_ALL) + cmd + '  log file: ' + logfile)
+        tools.exec(cmd, ignore_error=True, log_name=logfile, echo_cmd=False)
+        log.info("{}build finished: '{}".format(Color.GREEN, Color.RESET_ALL) + cmd + '  log file: ' + logfile)
+
+    def _build(self):
+        with open(self.orderinfo, "r") as fp:
+            order_info = json.load(fp)
+        public_args = order_info.get("profiles", {}).get("self", {}).get("args", "")
+        for orders in order_info.get("order", []):
+            threadPool = ThreadPoolExecutor(max_workers=(os.cpu_count() + 1) // 2)
+            results: list[Future] = []
+            for order in orders:
+                ref = order.get("ref")
+                pkgname = ref.split("/")[0]
+                logfile = f"{misc.LOG_DIR}/conan_{pkgname}.log"
+                packages = order.get("packages", [])
+                for package in packages:
+                    for pkg in package:
+                        binary = pkg.get("binary", "")
+                        if binary == 'Download':
+                            pkgid = pkg.get("package_id")
+                            cmd = f'conan download {ref}:{pkgid}'
+                            if self.remote:
+                                cmd += f' -r {self.remote}'
+                            tools.exec(cmd)
+                            continue
+                        if binary != "Build":
+                            continue
+                        build_args = pkg.get("build_args", "")
+                        result = threadPool.submit(self._build_package, logfile, build_args, public_args)
+                        results.append(result)
+            for result in as_completed(results):
+                result.result()
+
+    def build(self):
+        cwd = os.getcwd()
+        tmp = tempfile.TemporaryDirectory()
+        os.chdir(tmp.name)
+        self._build()
+        os.chdir(cwd)

{lbkit-0.9.3 → lbkit-0.9.4}/lbkit/component/build.py

@@ -4,6 +4,7 @@ import yaml
 import shutil
 import re
 import json
+import sys
 from multiprocessing import Pool
 import traceback
 from argparse import ArgumentParser
@@ -206,23 +207,25 @@ class BuildComponent():
         export_cmd = "conan export . "
         if self.user != "litebmc" or self.channel != "release":
             export_cmd += f"--user={self.user} --channel={self.channel}"
-        tools.run(export_cmd, capture_output=False)
-        # import sys
-        # sys.exit(-1)
+        tools.run(export_cmd, capture_output=False, stdout=sys.stdout, stderr=sys.stdout)
 
         lockfile = os.path.join(self.cwd, ".temp", "conan.lock")
+        orderfile = os.path.join(self.cwd, ".temp", "graph.order")
+        graph_cmd = f"conan graph build-order . {self.base_cmd} --order-by=recipe -f json --out-file={orderfile}"
+        if self.from_source:
+            graph_cmd += " --build='*'"
+        else:
+            graph_cmd += " --build=missing"
+        graph_cmd += f" --lockfile-out={lockfile}"
+        tools.run(graph_cmd)
+        bcp = BuildConanParallel(orderfile, self.options.remote)
+        bcp.build()
+        cmd = f"conan create {self.base_cmd} --build='{self.name}/*'"
+        tools.run(cmd, capture_output=False)
+
         graphfile = os.path.join(self.cwd, ".temp", "graph.info")
-        lock_cmd = f"conan lock create . {self.base_cmd} --lockfile-out={lockfile}"
-        tools.run(lock_cmd, capture_output=False)
         graph_cmd = f"conan graph info . {self.base_cmd} -f json --lockfile={lockfile}"
         tools.pipe([graph_cmd], out_file=graphfile)
-        log.success(f"start build dependency packages of {self.package}")
-        bcp = BuildConanParallel(graphfile, lockfile, self.base_cmd, self.from_source)
-        bcp.build()
-
-        cmd = f"conan create . {self.base_cmd} --build=missing"
-        log.info(f"start build {self.package}: {cmd}")
-        tools.run(cmd, capture_output=False)
 
         log.success(f"start deploy {self.package} and is's dependency packages")
         self.deploy(graphfile)
@@ -315,7 +318,7 @@ class BuildComponent():
     @property
     def package_id(self):
         cmd = f"conan graph info . {self.base_cmd} --filter=package_id"
-        res = tools.run(cmd)
+        res = tools.run(cmd, capture_output=True)
         match = re.search(r"package_id: ([a-f0-9]{40})", res.stdout)
         if match is None:
             raise errors.LiteBmcException(f"Get package if of {self.package} failed")
@@ -324,13 +327,13 @@ class BuildComponent():
     @staticmethod
     def package_folder(self):
         cmd = f"conan cache path {self.package}#latest:{self.package_id}"
-        res = tools.run(cmd)
+        res = tools.run(cmd, capture_output=True)
         return res.stdout.strip()
 
     @property
     def build_folder(self):
         cmd = f"conan cache path {self.package}#latest:{self.package_id} --folder=build"
-        res = tools.run(cmd)
+        res = tools.run(cmd, capture_output=True)
         return res.stdout.strip()
 
     def test(self):

{lbkit-0.9.3 → lbkit-0.9.4}/lbkit/component/test.py

@@ -52,12 +52,12 @@ class TestComponent():
         shutil.rmtree(coverage_dir, ignore_errors=True)
         os.makedirs(coverage_dir)
         # 覆盖率数据路径由conanbase.make在build配置cflags时定义
-        cmd = f"lcov -c -q -d {build_folder} -o {coverage_dir}/cover.info"
+        cmd = f"lcov --compat-libtool -c -q -d {build_folder} -o {coverage_dir}/cover.info"
         tool.exec(cmd)
         for dir in test_src_folder:
-            cmd = f"lcov -r  {coverage_dir}/cover.info \"{build_folder}/{dir}/*\" -o {coverage_dir}/cover.info"
+            cmd = f"lcov --compat-libtool -r {coverage_dir}/cover.info \"{build_folder}/{dir}/*\" -o {coverage_dir}/cover.info"
             tool.exec(cmd)
-        cmd = f"lcov -r  {coverage_dir}/cover.info \"*/include/*\" -o {coverage_dir}/cover.info"
+        cmd = f"lcov --compat-libtool -r {coverage_dir}/cover.info \"*/include/*\" -o {coverage_dir}/cover.info"
         tool.exec(cmd)
         cmd = f"genhtml -o {coverage_dir}/html --legend {coverage_dir}/cover.info"
         tool.exec(cmd)

{lbkit-0.9.3 → lbkit-0.9.4}/lbkit/log.py

@@ -1,41 +1,47 @@
 import logging
 import inspect
 import os
+import sys
 from lbkit.misc import Color
 
 class Logger(logging.getLoggerClass()):
 
     def __init__(self, *args, **kwargs):
         super(Logger, self).__init__(*args, **kwargs)
-        self.setLevel(logging.INFO)
-        handler = logging.StreamHandler()
         self.logenv = os.environ.get("LOG")
         if self.logenv is None:
             formatter = logging.Formatter('%(message)s')
-            self.setLevel(logging.INFO)
+            self._set_handler(logging.INFO, formatter)
         else:
-            formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
+            formatter = logging.Formatter('%(asctime)s %(message)s')
             if self.logenv == "info":
-                self.setLevel(logging.INFO)
+                self._set_handler(logging.INFO, formatter)
             elif self.logenv == "warn":
-                self.setLevel(logging.WARNING)
-            elif self.logenv == "error":
-                self.setLevel(logging.ERROR)
-            else:
-                self.setLevel(logging.DEBUG)
+                self._set_handler(logging.WARNING, formatter)
+            elif self.logenv != "error":
+                self._set_handler(logging.DEBUG, formatter)
+        self._set_handler(logging.ERROR, formatter)
+
+    def _set_handler(self, level, formatter):
+        if level == logging.ERROR:
+            handler = logging.StreamHandler()
+        else:
+            handler = logging.StreamHandler(sys.stdout)
+            handler.addFilter(lambda record: record.levelno < logging.ERROR)
+        handler.setLevel(level)
         handler.setFormatter(formatter)
-        self.handlers = []
         self.addHandler(handler)
 
+
     def error(self, msg, *args, **kwargs):
         if self.logenv:
             uptrace = kwargs.get("uptrace", 0)
             uptrace += 1
             stack = inspect.stack()[uptrace]
             filename = os.path.basename(stack.filename)
-            msg = f"[{filename}:{stack.lineno}] " + Color.RED + msg + Color.RESET_ALL
+            msg = "ERROR: " + f"[{filename}:{stack.lineno}] " + Color.RED + msg + Color.RESET_ALL
         else:
-            msg = Color.RED + msg + Color.RESET_ALL
+            msg = "ERROR: " + Color.RED + msg + Color.RESET_ALL
         kwargs.pop("uptrace", None)
         super(Logger, self).error(msg, *args, **kwargs)
 
@@ -45,7 +51,9 @@ class Logger(logging.getLoggerClass()):
             uptrace += 1
             stack = inspect.stack()[uptrace]
             filename = os.path.basename(stack.filename)
-            msg = f"[{filename}:{stack.lineno}] " + msg
+            msg = "DEBUG: " + f"[{filename}:{stack.lineno}] " + msg
+        else:
+            msg = "DEBUG: " + msg
         kwargs.pop("uptrace", None)
         super(Logger, self).debug(msg, *args, **kwargs)
 
@@ -55,7 +63,9 @@ class Logger(logging.getLoggerClass()):
             uptrace += 1
             stack = inspect.stack()[uptrace]
             filename = os.path.basename(stack.filename)
-            msg = f"[{filename}:{stack.lineno}] " + msg
+            msg = "INFO:  " + f"[{filename}:{stack.lineno}] " + msg
+        else:
+            msg = "INFO:  " + msg
         kwargs.pop("uptrace", None)
         super(Logger, self).info(msg, *args, **kwargs)
 
@@ -65,9 +75,9 @@ class Logger(logging.getLoggerClass()):
             uptrace += 1
             stack = inspect.stack()[uptrace]
             filename = os.path.basename(stack.filename)
-            msg = f"[{filename}:{stack.lineno}] " + Color.YELLOW + msg + Color.RESET_ALL
+            msg = "WARN:  " + f"[{filename}:{stack.lineno}] " + Color.YELLOW + msg + Color.RESET_ALL
         else:
-            msg = Color.YELLOW + msg + Color.RESET_ALL
+            msg = "WARN:  " + Color.YELLOW + msg + Color.RESET_ALL
         kwargs.pop("uptrace", None)
         super(Logger, self).warning(msg, *args, **kwargs)
 
@@ -77,9 +87,9 @@ class Logger(logging.getLoggerClass()):
             uptrace += 1
             stack = inspect.stack()[uptrace]
             filename = os.path.basename(stack.filename)
-            msg = f"[{filename}:{stack.lineno}] " + Color.GREEN + msg + Color.RESET_ALL
+            msg = "SUCC:  " + f"[{filename}:{stack.lineno}] " + Color.GREEN + msg + Color.RESET_ALL
         else:
-            msg = Color.GREEN + msg + Color.RESET_ALL
+            msg = "SUCC:  " + Color.GREEN + msg + Color.RESET_ALL
         kwargs.pop("uptrace", None)
         super(Logger, self).info(msg, *args, **kwargs)
 

{lbkit-0.9.3 → lbkit-0.9.4}/lbkit/tasks/config.py

@@ -37,7 +37,6 @@ class Config(object):
         self.profile_host = None
 
         # conan.lock options
-        self.using_lockfile = args.lockfile
         self.update_lockfile = args.update_lockfile
         self.target = args.target
         self.product = args.product
@@ -86,7 +85,6 @@ class Config(object):
         parser.add_argument("-pr:b", "--profile_build", help="Apply the specified profile to the build machine", default="default")
         parser.add_argument("-bt", "--build_type", type=str, choices=['debug', 'release', 'minsize'], help="Set the build type", default="debug")
         parser.add_argument("-r", "--remote", help="specified conan server", default="litebmc")
-        parser.add_argument("-l", "--lockfile", help="using conan.lock", action="store_true")
         parser.add_argument("-ul", "--update_lockfile", help="update conan.lock", action="store_true")
         parser.add_argument("-p", "--product", help="product name, default product is `default`", default="default")
         targets = Config.target_list()

{lbkit-0.9.3 → lbkit-0.9.4}/lbkit/tasks/task_build_manifest.py

@@ -43,24 +43,20 @@ class TaskClass(Task):
 
     def deploy(self, graph_file):
         with open(graph_file, "r") as fp:
-            graph = json.load(fp)
-        nodes = graph.get("graph", {}).get("nodes", {})
-        for id, info in nodes.items():
-            ref = info.get("ref")
-            id = info.get("package_id")
-            context = info.get("context")
-            if context != "host" or ref.startswith("litebmc/"):
-                continue
-            cmd = f"conan cache path {ref}:{id}"
-            package_folder = self.tools.run(cmd).stdout.strip()
-            self.config.conan_install.append(package_folder)
-
-    def download_recipe(self, pkg):
-        cmd = f"conan cache path {pkg}"
-        ret = self.exec_easy(cmd, ignore_error=True)
-        if ret is None or ret.returncode != 0:
-            cmd = f"conan download {pkg} -r {self.config.remote} --only-recipe"
-            self.exec(cmd, ignore_error=True)
+            order_info = json.load(fp)
+        for orders in order_info.get("order", []):
+            for order in orders:
+                ref = order.get("ref")
+                packages = order.get("packages", [])
+                for package in packages:
+                    for pkg in package:
+                        binary = pkg.get("context", "")
+                        if binary != "host":
+                            continue
+                        id = pkg.get("package_id", "")
+                        cmd = f"conan cache path {ref}:{id}"
+                        package_folder = self.tools.run(cmd).stdout.strip()
+                        self.config.conan_install.append(package_folder)
 
     def build_rootfs(self):
         """构建产品rootfs包"""
@@ -101,33 +97,24 @@ class TaskClass(Task):
         fp.close()
 
         base_cmd = f"{self.common_args} {self.conan_settings}"
-        threadPool = ThreadPoolExecutor(max_workers=16)
-        if self.config.using_lockfile:
-            lockfile = os.path.join(self.config.code_path, "conan.lock")
+        lockfile = os.path.join(self.config.code_path, "conan.lock")
+        orderfile = os.path.join(self.config.temp_path, "build-order.json")
+        graph_cmd = f"conan graph build-order . {base_cmd} --order-by=recipe -f json --out-file={orderfile}"
+        if self.config.from_source:
+            graph_cmd += " --build='*'"
         else:
-            lockfile = os.path.join(self.config.temp_path, "conan.lock")
-        # 创建新的conan.lock文件
-        if not self.config.using_lockfile or self.config.update_lockfile:
-            lock_cmd = f"conan lock create . {base_cmd} --lockfile-out={lockfile}"
-            self.exec(lock_cmd, verbose=True)
-        if not os.path.isfile(lockfile):
-            raise FileNotFoundError("lockfile ./conan.lock was not found")
-        with open(lockfile, "r") as fp:
-            lock = json.load(fp)
-        for key in ["requires", "build_requires", "python_requires", "config_requires"]:
-            requires = lock.get(key, [])
-            for require in requires:
-                threadPool.submit(self.download_recipe, require)
-        threadPool.shutdown(wait=True)
-        graph_cmd = f"conan graph info . {base_cmd} -f json --lockfile={lockfile}"
-        graphfile = os.path.join(self.config.temp_path, "graph.info")
-        self.pipe([graph_cmd], out_file=graphfile)
-        bcp = BuildConanParallel(graphfile, lockfile, self.common_args, self.config.from_source)
+            graph_cmd += " --build=missing"
+        if self.config.update_lockfile or not os.path.isfile(lockfile):
+            graph_cmd += f" --lockfile-out={lockfile}"
+        else:
+            graph_cmd += f" --lockfile={lockfile} --lockfile-partial"
+        self.exec(graph_cmd, verbose=True)
+        bcp = BuildConanParallel(orderfile, self.config.remote)
         bcp.build()
 
         self.exec(f"sed -i 's@rootfs_df190c/0.0.1#.*\"@rootfs_df190c/0.0.1\"@g' {lockfile}")
         # 部署应用到self.config.conan_install
-        self.deploy(graphfile)
+        self.deploy(orderfile)
 
     def run(self):
         """任务入口"""

{lbkit-0.9.3 → lbkit-0.9.4}/lbkit/tasks/task_build_prepare.py

@@ -74,12 +74,14 @@ class TaskClass(Task):
             src_fd = open(profile, "r")
             template = Template(src_fd.read())
             src_fd.close()
-            content = template.safe_substitute(compiler_path=self.compiler_path, sysroot_path=self.sysroot_path)
+            content = template.safe_substitute(compiler_path=self.compiler_path,
+                                               sysroot_path=self.sysroot_path,
+                                               code_path=self.config.code_path)
             dst_fp.write(content)
 
         with open(dst_profile, "r") as fp:
             profile_data = jinja2.Template(fp.read()).render()
-            parser = configparser.ConfigParser()
+            parser = configparser.ConfigParser(allow_no_value=True, delimiters=('=',))
             parser.read_string(profile_data)
             strip = "strip"
             if parser.has_option("buildenv", "STRIP"):

{lbkit-0.9.3 → lbkit-0.9.4}/lbkit/tools.py

@@ -22,6 +22,33 @@ class Tools(object):
         self.log_name = os.path.join(self.log_dir, f"{log_name}.log")
         self.log: Logger = Logger(log_name)
 
+    class _Tee:
+        """同时写入文件和终端的类"""
+        def __init__(self, file_obj, verbose: bool):
+            self.file = file_obj
+            self.verbose = verbose
+            self.memory = ""
+            self.stdout = sys.stdout
+            self.stderr = sys.stderr
+
+        def writef(self, data):
+            self.file.write(data)
+
+        def write(self, data):
+            # 同时写入文件和终端
+            self.memory += data
+            self.file.write(data)
+            if self.verbose:
+                self.stdout.write(data)
+
+        def fileno(self):
+            return self.file.fileno()
+
+        def flush(self):
+            self.file.flush()
+            if self.verbose:
+                self.stdout.flush()
+
     @staticmethod
     def _real_command(shell: str):
         if not isinstance(shell, str):
@@ -44,50 +71,46 @@ class Tools(object):
         return cmd
 
     def exec(self, cmd: str, verbose=False, ignore_error = False, sensitive=False, log_prefix="", **kwargs):
-        """执行命令"""
+        """执行命令，输出同时写入日志文件和终端"""
         stack = inspect.stack()[1]
         file = os.path.basename(stack.filename)
         line = stack.lineno
         echo_cmd = kwargs.get("echo_cmd", True)
         log_name = kwargs.get("log_name", self.log_name)
         uptrace = kwargs.get("uptrace", 0) + 1
-
         show_cmd = "***" if sensitive else cmd
         if log_prefix:
             show_cmd = "(" + log_prefix + ") " + show_cmd
 
-        fd = os.fdopen(os.open(log_name, os.O_APPEND | os.O_CREAT | os.O_RDWR), "a+")
-        if echo_cmd:
-            self.log.info("{}>>>>{} {}".format(Color.GREEN, Color.RESET_ALL, show_cmd), uptrace=uptrace)
-        last_pos = fd.tell()
-        fd.write(">>>> {}\n".format(show_cmd))
         if os.environ.get("VERBOSE", False):
             verbose = True
+        fd = os.fdopen(os.open(log_name, os.O_APPEND | os.O_CREAT | os.O_RDWR), "a+")
+        # 使用 _Tee 类同时写入文件和终端
+        msg = "{}>>>>{} {}\n".format(Color.GREEN, Color.RESET_ALL, show_cmd)
+        tee = self._Tee(fd, verbose)
+        if echo_cmd:
+            sys.stdout.write(msg)
+        tee.writef(msg)
         real_cmd = self._real_command(cmd)
-        result = subprocess.Popen(real_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+        result = subprocess.Popen(real_cmd, stdout=tee, stderr=tee, universal_newlines=True)
+        tee.flush()
+        fd.close()
         if result is None:
             raise errors.RunCommandException(f"Run command {show_cmd} failed")
-        for msg in result.stdout:
-            fd.write(msg)
-            if verbose:
-                sys.stdout.write(msg)
-        fd.close()
         result.communicate()
-        if result is None or result.returncode != 0:
-            if not ignore_error:
-                msg = f"{file}:{line} > Run command ({show_cmd}) failed, log file is {log_name}"
+        if result.returncode == 0:
+            return result
+        if not ignore_error:
+            msg = f"{file}:{line} > Run command ({show_cmd}) failed, log file is {log_name}"
+            if not verbose:
                 self.log.warn(f">>>>>>>>>> '{show_cmd}' LOG START", uptrace=uptrace)
-                fd = open(log_name, "r")
-                fd.seek(last_pos, 0)
-                logs = fd.readlines()
-                for log in logs:
-                    print(log, end="")
-                self.log.warn(f"<<<<<<<<<< '{show_cmd}' LOG END", uptrace=uptrace)
+                sys.stdout.write(tee.memory)
                 sys.stdout.flush()
-                raise errors.RunCommandException(msg)
-            else:
-                msg = f"{file}:{line} > Run command ({show_cmd}) failed but ignore error"
-            self.log.warn(msg, uptrace=uptrace)
+                self.log.warn(f"<<<<<<<<<< '{show_cmd}' LOG END", uptrace=uptrace)
+            raise errors.RunCommandException(msg)
+        else:
+            msg = f"{file}:{line} > Run command ({show_cmd}) failed but ignore error"
+        self.log.warn(msg, uptrace=uptrace)
         return result
 
     def pipe(self, cmds: list[str], ignore_error=False, out_file = None, **kwargs):
@@ -127,13 +150,16 @@ class Tools(object):
         stdin.close()
         return output
 
-    def run(self, cmd, ignore_error=False, **kwargs):
+    def run(self, cmd, ignore_error=False, stdout=None, stderr=None, **kwargs):
         uptrace = kwargs.get("uptrace", 0) + 1
         capture_output = kwargs.get("capture_output", True)
         self.log.debug("{}>>>>{} {}".format(Color.GREEN, Color.RESET_ALL, cmd), uptrace=uptrace)
         real_cmd = self._real_command(cmd)
         check = False if ignore_error else True
-        return subprocess.run(real_cmd, capture_output=capture_output, check=check, encoding="utf-8")
+        if stdout or stderr:
+            return subprocess.run(real_cmd, check=check, stdout=stdout, stderr=stderr, encoding="utf-8")
+        else:
+            return subprocess.run(real_cmd, capture_output=capture_output, check=check, encoding="utf-8")
 
     @staticmethod
     def file_digest_sha256(filename):

{lbkit-0.9.3 → lbkit-0.9.4/lbkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lbkit
-Version: 0.9.3
+Version: 0.9.4
 Summary: Tools provided by litebmc.com
 Home-page: https://www.litebmc.com
 Author: xuhj@litebmc.com

lbkit-0.9.3/lbkit/__init__.py

@@ -1,2 +0,0 @@
-
-__version__ = '0.9.3'

lbkit-0.9.3/lbkit/build_conan_parallel.py

@@ -1,206 +0,0 @@
-"""任务基础类"""
-import os
-import json
-import tempfile
-from queue import Queue
-from threading import Thread
-from lbkit.log import Logger
-from lbkit.errors import LiteBmcException
-from lbkit import misc
-from lbkit.tools import Tools
-
-tools = Tools("comp_build")
-log = Logger("comp_build")
-
-class ConanPackage():
-    def __init__(self, node):
-        self.node = node
-        self.deps: dict[str, ConanPackage] = {}
-        self.build_deps: dict[str, ConanPackage] = {}
-        self.ref = node.get("ref")
-        self.pkg = self.ref.split("#")[0]
-        self.name = self.pkg.split("/")[0]
-        self.context = node.get("context")
-        self.building = False
-        self.is_host = self.context == "host"
-        binary = node.get("binary")
-        self.binary_exist = (binary in ["Cache"])
-
-    def append_dep(self, dep):
-        self.deps[dep.pkg] = dep
-
-    def append_build_dep(self, dep):
-        self.build_deps[dep.pkg] = dep
-
-    # 间接构建依赖更新为直接依赖
-    def update_indirect_build_dep(self):
-        deps = {}
-        for _, dep in self.build_deps.items():
-            sub_deps = dep.update_indirect_build_dep()
-            deps.update(sub_deps)
-        self.build_deps.update(deps)
-        return self.build_deps
-
-    @property
-    def options(self):
-        return self.node.get("options", {})
-
-    @property
-    def settings(self):
-        return self.node.get("settings", {})
-
-    @property
-    def default_options(self):
-        return self.node.get("default_options", {})
-
-
-class BuildConanParallel(object):
-    def __init__(self, graphinfo, lockfile, cmd, force_build):
-        self.queue = Queue()
-        self.cmd = ""
-        chunks = cmd.split()
-        skip = False
-        for chunk in chunks:
-            if skip:
-                skip = False
-                continue
-            if chunk in ["--user", "--channel", "--version", "--name"]:
-                skip = True
-                continue
-            self.cmd += f"{chunk} "
-        self.graphinfo = os.path.realpath(graphinfo)
-        self.lockfile = os.path.realpath(lockfile)
-        if not os.path.isfile(graphinfo):
-            raise LiteBmcException(f"graph file {graphinfo} not exist")
-        self.exception = None
-        self.force_build = force_build
-
-    def build_dep(self, cp: ConanPackage, options):
-        for name, value in cp.settings.items():
-            options += f" -s {name}={value}"
-
-        cmd = f"conan install --requires={cp.ref} {self.cmd} {options}"
-        if self.force_build:
-            cmd += f" --build=\"{cp.name}/*\""
-        cmd += " --build=missing"
-        cmd += f" --lockfile={self.lockfile}"
-        try:
-            logfile = f"{misc.LOG_DIR}/conan_{cp.name}.log"
-            log.info(f">>>> build {cp.ref} start, logfile: {logfile}")
-            log.debug(f">>>> {cmd}")
-            tools.exec(cmd, echo_cmd=False, log_name=logfile)
-            log.success(f"<<<< build {cp.ref} finished")
-        except Exception as e:
-            self.exception = e
-        self.queue.put(cp)
-
-    def _build(self):
-        with open(self.graphinfo, "r") as fp:
-            grapth = json.load(fp)
-        nodes = grapth.get("graph", {}).get("nodes", {})
-        packages: dict[str, ConanPackage] = {}
-
-        build_works = {}
-        for id, node in nodes.items():
-            if id == "0":
-                continue
-            cp = ConanPackage(node)
-            packages[cp.pkg] = cp
-            build_works[cp.name] = False
-        for id, node in nodes.items():
-            if id == "0":
-                continue
-            ref = node.get("ref")
-            if ref.startswith("litebmc/"):
-                continue
-            cp = packages[ref.split("#")[0]]
-            deps = node.get("dependencies", {})
-            for _, dep in deps.items():
-                dep_ref = dep.get("ref")
-                direct = dep.get("direct")
-                if not direct:
-                    continue
-                dep_cp = packages[dep_ref]
-                if dep_cp.is_host:
-                    cp.append_dep(dep_cp)
-                else:
-                    cp.append_build_dep(dep_cp)
-        options = ""
-        for _, cp in packages.items():
-            for name, value in cp.options.items():
-                if value is None:
-                    continue
-                def_val = cp.default_options.get(name)
-                conan_name = cp.pkg.split("@")[0]
-                if isinstance(def_val, bool):
-                    if (def_val and "False" == value) or (not def_val and "True" == value):
-                        options += f" -o {conan_name}:{name}={value}"
-                elif def_val != value:
-                    options += f" -o {conan_name}:{name}={value}"
-        # 工具间接依赖变更为直接依赖
-        for name, pkg in packages.items():
-            pkg.update_indirect_build_dep()
-        wait_finished = 0
-        while True:
-            for _, cp in packages.items():
-                if wait_finished >= 4:
-                    continue
-                if cp.ref.startswith("litebmc/"):
-                    continue
-                # 如果是构建工具，不参与构建
-                if not cp.is_host:
-                    continue
-                # 如果还有依赖未构建完成，不参与构建
-                if len(cp.deps) != 0:
-                    continue
-                # 如果正在构建
-                if cp.building:
-                    continue
-                # 相同名称的组件正在构建时不启动新的构建
-                if build_works[cp.name]:
-                    continue
-                if not cp.binary_exist or self.force_build:
-                    # 当依赖的构建工具存在正在构建的组件时不能构建
-                    need_build = True
-                    for _, dep in cp.build_deps.items():
-                        # 正在构建且未构建出制品时
-                        if not dep.binary_exist and dep.building:
-                            need_build = False
-                            break
-                    if not need_build:
-                        continue
-                    cp.building = True
-                    # 启动构建前将其依赖的构建工具置为正在构建
-                    for _, dep in cp.build_deps.items():
-                        dep.building = True
-                    build_works[cp.name] = True
-                    wait_finished += 1
-                    thread = Thread(target=self.build_dep, args=(cp, options,))
-                    thread.start()
-                else:
-                    cp.building = True
-                    build_works[cp.name] = True
-                    wait_finished += 1
-                    self.queue.put(cp)
-            if not wait_finished:
-                return
-            cp = self.queue.get()
-            if self.exception:
-                raise self.exception
-            build_works[cp.name] = False
-            wait_finished -= 1
-            for _, sub_cp in packages.items():
-                if sub_cp.deps.get(cp.pkg):
-                    sub_cp.deps.pop(cp.pkg)
-            # 构建完成后，组件的构建依赖工具一定构建完成且制品存在
-            for _, dep in cp.build_deps.items():
-                dep.building = False
-                dep.binary_exist = True
-
-
-    def build(self):
-        cwd = os.getcwd()
-        tmp = tempfile.TemporaryDirectory()
-        os.chdir(tmp.name)
-        self._build()
-        os.chdir(cwd)