openubmc-bingo 0.5.277__py3-none-any.whl → 0.6.2__py3-none-any.whl

bmcgo/utils/tools.py

@@ -26,22 +26,38 @@ import functools
 import inspect
 import configparser
 from typing import Callable
-from tempfile import TemporaryFile
+from tempfile import TemporaryFile, TemporaryDirectory
 from string import Template
 from subprocess import Popen, PIPE
 from collections import OrderedDict
 
 import jsonschema
+import requests
+from tqdm import tqdm
 import yaml
-from conans.model.profile import Profile
-from conans.client.profile_loader import read_profile
 
+from bmcgo import misc
 from bmcgo.logger import Logger
 from bmcgo.errors import BmcGoException, EnvironmentException
-from bmcgo import misc
 from bmcgo import errors
 from bmcgo.logger import Logger
 
+if misc.conan_v2():
+    from conan.internal.api.profile.profile_loader import ProfileLoader
+else:
+    from conans.client.profile_loader import read_profile
+
+
+def env_set(key, value, append=False):
+    if not append:
+        os.environ[key] = value
+        return
+    old_value = os.environ.get(key, "")
+    new_value = ":" + value
+    if new_value in old_value:
+        return
+    os.environ[key] = old_value + new_value
+
 
 class Tools():
     """
@@ -68,7 +84,10 @@ class Tools():
 
     @property
     def conan_home(self):
-        return os.path.expanduser("~/.conan")
+        if misc.conan_v2():
+            return os.path.expanduser("~/.conan2")
+        else:
+            return os.path.expanduser("~/.conan")
 
     @property
     def conan_profiles_dir(self):
@@ -93,10 +112,11 @@ class Tools():
             cmd_args = command
         elif isinstance(command, str):
             cmd_args = shlex.split(command)
-            cmd = shutil.which(cmd_args[0])
-            if cmd is None:
-                raise EnvironmentException(f"{cmd_args[0]}不存在, 请检查命令或环境配置")
-            cmd_args[0] = cmd
+            if cmd_args[0] != "source":
+                cmd = shutil.which(cmd_args[0])
+                if cmd is None:
+                    raise EnvironmentException(f"{cmd_args[0]}不存在, 请检查命令或环境配置")
+                cmd_args[0] = cmd
         else:
             raise BmcGoException(f"不支持的命令参数格式: {command}, 请检查命令格式")
         sudo_cmd = shutil.which("sudo")
@@ -187,17 +207,24 @@ class Tools():
         return profile_help_text
 
     @staticmethod
-    def get_conan_profile(profile: str, build_type: str, enable_luajit=False):
+    def get_conan_profile(profile: str, build_type: str, enable_luajit=False, test=False):
         """
         根据参数与配置获取conan构建使用的profile文件名。
         """
         if profile:
             return str(profile)
 
-        if build_type == "dt":
+        if misc.conan_v1() and build_type == "dt":
             return "profile.dt.ini"
-
-        return "profile.luajit.ini"
+        if misc.conan_v2 and test:
+            return "profile.dt.ini"
+        if misc.community_name() == "openubmc":
+            return "profile.luajit.ini"
+        if not enable_luajit:
+            return "profile.ini"
+        if misc.conan_v1():
+            return "profile.luajit.ini"
+        return "profile.ini"
 
     @staticmethod
     def check_product_dependencies(top, base):
@@ -213,7 +240,6 @@ class Tools():
     @staticmethod
     def check_base_dependencies(base_manifest, com_package, dependency_type):
         name = com_package.get(misc.CONAN)
-        action = com_package.get("action", "")
         pkg_info = name.split("/")
         pkg_name = pkg_info[0]
         has_version = len(pkg_info) > 1
@@ -222,7 +248,7 @@ class Tools():
             sub_pkg_name = sub_name.split("/")[0]
             # 上下层具有相同组件名的组件
             if pkg_name == sub_pkg_name:
-                if has_version or action:
+                if has_version:
                     raise errors.BmcGoException(
                         "配置错误: 不允许删除平台组件或定制平台组件版本号{}".format(name)
                     )
@@ -277,74 +303,26 @@ class Tools():
         if stage == misc.StageEnum.STAGE_DEV.value:
             # 开发者测试模式
             stage = misc.StageEnum.STAGE_RC.value
-        # else: rc和stable模式
-        channel = f"@{Tools().conan_user}/{stage}"
 
         if len(version_split) == 2:
+            # rc和stable模式
+            channel = f"@{misc.conan_user()}/{stage}"
             package[misc.CONAN] += channel
 
-    @staticmethod
-    def install_platform(platform_package, stage, remote):
-        Tools().fix_platform(platform_package, stage)
-        rtos_version = platform_package.get("options", {}).get(
-            "rtos_version", "rtos_v2"
-        )
-        append_cmd = f"-r {remote}" if remote else ""
-        Tools().run_command(
-            f"conan install {platform_package[misc.CONAN]} {append_cmd} -o rtos_version={rtos_version}"
-        )
-        version_split = re.split(r"[/, @]", platform_package[misc.CONAN])
-        package_conan_dir = os.path.join(
-            os.path.expanduser("~"), ".conan/data/", *version_split
-        )
-        cmd = f"conan info {platform_package[misc.CONAN]} {append_cmd} -o rtos_version={rtos_version} -j"
-        cmd_info = Tools().run_command(
-            cmd,
-            capture_output=True,
-        )
-        info = json.loads(cmd_info.stdout)
-        for i in info:
-            package_conan_dir = os.path.join(
-                package_conan_dir, "package", i["id"], rtos_version
-            )
-        return package_conan_dir
-
-    @staticmethod
-    def get_manifest_dependencies(manifest_build_dir, file_path, stage, remote):
-        with open(file_path, "r") as f_:
-            base_manifest = yaml.safe_load(f_)
-        top_dependencies = base_manifest[misc.CONAN_DEPDENCIES_KEY]
-        platform_package = base_manifest.get("platform", {})
-        if platform_package:
-            package_conan_dir = Tools.install_platform(platform_package, stage, remote)
-            top_dependencies.append(platform_package)
-            platform_src = os.path.join(package_conan_dir, "manifest.yml")
-            with open(platform_src, "r") as f_:
-                platform_manifest = yaml.safe_load(f_)
-            Tools.check_product_dependencies(base_manifest, platform_manifest)
-            top_dependencies = Tools.merge_dependencies(
-                top_dependencies, platform_manifest[misc.CONAN_DEPDENCIES_KEY]
-            )
-        inc_filename = base_manifest.get("include")
-        if not inc_filename:
-            return top_dependencies
-        file_path = inc_filename.replace(
-            "${product}", os.path.join(manifest_build_dir, "product")
-        )
-        base_dependencies = Tools.get_manifest_dependencies(manifest_build_dir, file_path, stage, remote)
-        dep = Tools.merge_dependencies(top_dependencies, base_dependencies)
-        return dep
-
     @staticmethod
     def create_common_parser(description):
         parser = argparse.ArgumentParser(description=description)
         parser.add_argument("-cp", "--conan_package", help="软件包名, 示例: kmc/24.0.0.B020", default="")
         parser.add_argument("-uci", "--upload_package", help="是否上传软件包", action=misc.STORE_TRUE)
-        parser.add_argument("-o", "--options", help="组件特性配置, 示例: -o skynet:enable_luajit=True",
-                            action='append')
+        if misc.conan_v2():
+            parser.add_argument("-o", "--options", help="组件特性配置, 示例: -o skynet/*:enable_luajit=True", action="append",)
+            parser.add_argument("-bt", "--build_type", help="构建类型，可选：debug（调试包）, release（正式包）",
+                                default="debug")
+        else:
+            parser.add_argument("-o", "--options", help="组件特性配置, 示例: -o skynet:enable_luajit=True", action='append')
+            parser.add_argument("-bt", "--build_type", help="构建类型，可选：debug（调试包）, release（正式包）, dt（开发者测试包）", 
+                                default="debug")
         parser.add_argument("-r", "--remote", help="指定conan远端")
-        parser.add_argument("-bt", "--build_type", help="构建类型，可选：debug（调试包）, release（正式包）, dt（开发者测试包）", 
-                            default="debug")
         parser.add_argument("--stage", help="包类型，可选值为: dev(调试包), rc（预发布包）, stable（发布包）\n默认：dev", 
                             default="dev")
         parser.add_argument("-jit", "--enable_luajit", help="Enable luajit", action=misc.STORE_TRUE)
@@ -359,7 +337,7 @@ class Tools():
         return parser
 
     @staticmethod
-    def get_module_symver_option(module_symver_path: str):
+    def get_171x_module_symver_option(module_symver_path: str):
         sha256 = Tools.sha256sum(module_symver_path)
         return ("module_symver", sha256)
 
@@ -373,6 +351,70 @@ class Tools():
         else:
             os.makedirs(conan_bin)
 
+    @staticmethod
+    def download_file(url, file_path):
+        response = requests.get(url, stream=True)
+        response.raise_for_status()
+        total_size = int(response.headers.get("content-length", 0))
+
+        with open(file_path, 'wb') as f, tqdm(
+            total=total_size,
+            unit='B',
+            unit_scale=True,
+            desc=file_path,
+            miniters=1
+        ) as pbar:
+            for chunk in response.iter_content(chunk_size=8192):
+                f.write(chunk)
+                pbar.update(len(chunk))
+
+    @staticmethod
+    def get_hi171x_module_symver_option(path: str):
+        sha256 = Tools.sha256sum(path)
+        return ("module_symvers", sha256)
+
+    @staticmethod
+    def install_platform_v1(platform_package, stage, remote):
+        Tools().fix_platform(platform_package, stage)
+        rtos_version = platform_package.get("options", {}).get(
+            "rtos_version", "rtos_v2"
+        )
+        append_cmd = f"-r {remote}" if remote else ""
+        Tools().run_command(
+            f"conan install {platform_package[misc.CONAN]} {append_cmd} -o rtos_version={rtos_version} --build=missing"
+        )
+        version_split = re.split(r"[/, @]", platform_package[misc.CONAN])
+        package_conan_dir = os.path.join(
+            os.path.expanduser("~"), ".conan/data/", *version_split
+        )
+        cmd = f"conan info {platform_package[misc.CONAN]} {append_cmd} -o rtos_version={rtos_version} -j"
+        cmd_info = Tools().run_command(
+            cmd,
+            capture_output=True,
+        )
+        info = json.loads(cmd_info.stdout)
+        for i in info:
+            package_conan_dir = os.path.join(
+                package_conan_dir, "package", i["id"], rtos_version
+            )
+        return package_conan_dir
+
+    @staticmethod
+    def get_package_folder_from_graph_file(graph_file, package):
+        """
+        安装conan包到指定outdir目录
+        """
+        with open(graph_file, "r") as file_handler:
+            package_info = json.load(file_handler)
+        pkg_name = package.split("/")[0] + "/"
+        nodes: dict[str, dict] = package_info.get("graph", {}).get("nodes", {})
+        for _, node in nodes.items():
+            ref = node.get("ref")
+            if not ref or not ref.startswith(pkg_name):
+                continue
+            return node.get("package_folder")
+        return None
+
     @functools.cached_property
     def is_ubuntu(self):
         """
@@ -387,17 +429,54 @@ class Tools():
                 return True
         return False
 
-    @functools.cached_property
-    def conan_user(self):
-        if os.access(misc.GLOBAL_CFG_FILE, os.R_OK):
-            try:
-                conf = configparser.ConfigParser()
-                conf.read(misc.GLOBAL_CFG_FILE)
-                return conf.get(misc.CONAN, "user")
-            except (configparser.NoSectionError, configparser.NoOptionError):
-                pass
+    def install_platform_v2(self, platform_package, remote, update_conan_cache=False):
+        name = platform_package.get(misc.CONAN)
+        if misc.conan_v2():
+            name = name.lower()
+        option_key = name.split("/")[0] + "/*"
+        rtos_version = platform_package.get("options", {}).get(
+            "rtos_version", "rtos_v2"
+        )
+        enable_haf = platform_package.get("options", {}).get("enable_haf", False)
+        append_cmd = f"-o {option_key}:rtos_version={rtos_version}"
+        append_cmd += f" -r {remote}" if remote else ""
+        append_cmd += f" -o {option_key}:enable_haf=True" if enable_haf else ""
+        if update_conan_cache:
+            append_cmd += " -u"
+        append_cmd += " --build=missing"
+        package_folder = self.get_conan_package_folder(name, append_cmd)
+        return os.path.join(package_folder, rtos_version)
+
+    def install_platform(self, platform_package, stage, remote, update_conan_cache=False):
+        if misc.conan_v1():
+            return self.install_platform_v1(platform_package, stage, remote)
+        else:
+            return self.install_platform_v2(platform_package, remote, update_conan_cache)
 
-        return misc.ConanUserEnum.CONAN_USER_RELEASE.value
+    def get_manifest_dependencies(self, manifest_build_dir, file_path, stage, remote, update_conan_cache=False):
+        with open(file_path, "r") as f_:
+            base_manifest = yaml.safe_load(f_)
+        top_dependencies = base_manifest[misc.CONAN_DEPDENCIES_KEY]
+        platform_package = base_manifest.get("platform", {})
+        if platform_package:
+            package_conan_dir = self.install_platform(platform_package, stage, remote, update_conan_cache)
+            top_dependencies.append(platform_package)
+            platform_src = os.path.join(package_conan_dir, "manifest.yml")
+            with open(platform_src, "r") as f_:
+                platform_manifest = yaml.safe_load(f_)
+            Tools.check_product_dependencies(base_manifest, platform_manifest)
+            top_dependencies = Tools.merge_dependencies(
+                top_dependencies, platform_manifest[misc.CONAN_DEPDENCIES_KEY]
+            )
+        inc_filename = base_manifest.get("include")
+        if not inc_filename:
+            return top_dependencies
+        file_path = inc_filename.replace(
+            "${product}", os.path.join(manifest_build_dir, "product")
+        )
+        base_dependencies = self.get_manifest_dependencies(manifest_build_dir, file_path, stage, remote)
+        dep = Tools.merge_dependencies(top_dependencies, base_dependencies)
+        return dep
 
     def get_conan_remote_list(self, remote):
         conan_remote_list = []
@@ -410,13 +489,16 @@ class Tools():
 
     def download_conan_recipes(self, conan_version, conan_remote_list):
         download_flag = False
+        args = "--only-recipe" if misc.conan_v2() else "--recipe"
         for remote in conan_remote_list:
-            try:
-                self.run_command(f"conan download {conan_version} -r {remote} -re", show_error_log=False)
-                download_flag = True
-                break
-            except Exception as e:
-                self.log.info(f"Recipe not fount in {remote}: {conan_version}")
+            # 重试3次
+            for _ in range(1, 3):
+                try:
+                    self.run_command(f"conan download {conan_version} -r {remote} {args}", show_error_log=False)
+                    download_flag = True
+                    break
+                except Exception as e:
+                    self.log.info(f"Recipe not fount in {remote}: {conan_version}")
         if not download_flag:
             raise BmcGoException(f"Download {conan_version} failed")
 
@@ -532,13 +614,10 @@ class Tools():
         os.remove(f"{src_file}_bak")
 
     def make_img(self, img_path, mnt_datafs, size):
-        try:
-            self.run_command(f"/sbin/mkfs.ext4 -d {mnt_datafs} -r 1 -N 0 -m 5 -L \"rootfs\" -I 256 -O \
-            ^64bit,^metadata_csum {img_path} \"{size}M\"", ignore_error=True, sudo=True)
-            user_group = f"{os.getuid()}:{os.getgid()}"
-            self.run_command(f"chown {user_group} {img_path}", sudo=True)
-        except Exception:
-            self.log.error(f"{img_path} 镜像制作失败")
+        self.run_command(f"/sbin/mkfs.ext4 -d {mnt_datafs} -r 1 -N 0 -m 5 -L \"rootfs\" -I 256 -O \
+            ^64bit,^metadata_csum {img_path} \"{size}M\"", sudo=True)
+        user_group = f"{os.getuid()}:{os.getgid()}"
+        self.run_command(f"chown {user_group} {img_path}", sudo=True)
 
     def get_studio_path(self):
         ret = self.run_command("whereis bmc_studio", sudo=True, ignore_error=True,
@@ -561,20 +640,28 @@ class Tools():
         command_key = kwargs.get("command_key", None)
         command_echo = kwargs.get("command_echo", True)
         show_log = kwargs.get("show_log", False)
+        show_error_log = kwargs.get("show_error_log", True)
         uptrace = kwargs.get("uptrace", 0) + 1
         error_log = kwargs.get("error_log")
         warn_log = kwargs.get("warn_log")
+        log_name = kwargs.get("log_name")
         timeout = kwargs.get("timeout", 1200)
         capture_output = kwargs.get("capture_output", False)
         if command_key:
             key = command_key + ":"
         else:
             key = ""
-        if command_echo and not self.log.is_debug:
-            self.log.info(f">> {key}{command}", uptrace=uptrace)
+        if isinstance(command, list):
+            cmd_str = " ".join(command)
+        else:
+            cmd_str = command
+        if command_echo or self.log.is_debug:
+            self.log.info(f">> {key}{cmd_str}", uptrace=uptrace)
         command = self.format_command(command, sudo)
         ret = None
-        log_fd = os.fdopen(os.open(self.log_name, os.O_RDWR | os.O_CREAT | os.O_APPEND,
+        if not log_name:
+            log_name = self.log_name
+        log_fd = os.fdopen(os.open(log_name, os.O_RDWR | os.O_CREAT | os.O_APPEND,
                         stat.S_IWUSR | stat.S_IRUSR), 'a+')
         try:
             check = False if ignore_error else True
@@ -593,8 +680,8 @@ class Tools():
                 self.log.error(error_log, uptrace=uptrace)
             elif warn_log:
                 self.log.warning(warn_log, uptrace=uptrace)
-            else:
-                self.log.error(f"执行命令 {key}{command} 错误, 日志: {self.log_name}", uptrace=uptrace)
+            elif show_error_log:
+                self.log.error(f"执行命令 {key}{cmd_str} 错误, 日志: {log_name}", uptrace=uptrace)
             log_fd.flush()
             log_fd.close()
             raise e
@@ -613,10 +700,11 @@ class Tools():
         stdin = None
         command_echo = kwargs.get("command_echo", True)
         uptrace = kwargs.get("uptrace", 0) + 1
-        ignore_error = kwargs.get('ignore_error', False)
-        capture_output = kwargs.get('capture_output', False)
-        if command_echo and not self.log.is_debug:
-            self.log.info(f">> " + ' | '.join(commands), uptrace=uptrace)
+        ignore_error = kwargs.get("ignore_error", False)
+        capture_output = kwargs.get("capture_output", False)
+        append_output = kwargs.get("append", False)
+        if command_echo or self.log.is_debug:
+            self.log.info(f">> " + " | ".join(commands), uptrace=uptrace)
         for command in commands:
             stdout = TemporaryFile(flag)
             cmd = self.format_command(command, False)
@@ -633,8 +721,14 @@ class Tools():
         if stdin:
             context = stdin.read()
             if out_file:
-                with os.fdopen(os.open(out_file, os.O_WRONLY | os.O_CREAT,
-                                    stat.S_IWUSR | stat.S_IRUSR), flag) as fp:
+                flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
+                if append_output:
+                    flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND
+                mode = stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
+                with os.fdopen(
+                    os.open(out_file, flags, mode),
+                    flag,
+                ) as fp:
                     fp.write(context)
             stdin.close()
         if capture_output:
@@ -658,16 +752,44 @@ class Tools():
         cmd = [f"find {self.conan_data} -maxdepth 4 -mindepth 4 -type f -name *.count*", "xargs rm -f"]
         _, _ = self.pipe_command(cmd, out_file=None)
 
-    def get_profile_config(self, profile_name) -> (Profile, OrderedDict):
-        """根据profile文件名获取~/.conan/profiles对应文件的配置内容"""
-        if not profile_name:
-            return None, None
-        profile_file = os.path.join(self.conan_profiles_dir, profile_name)
-        if not os.path.isfile(profile_file):
-            raise BmcGoException(f"{profile_file} 文件不存在")
+    def get_profile_config(self, profile_name):
+        if misc.conan_v2():
+            return ProfileLoader(".temp").load_profile(
+                profile_name, self.conan_profiles_dir
+            ), {}
+        else:
+            """根据profile文件名获取~/.conan/profiles对应文件的配置内容"""
+            if not profile_name:
+                return None, None
+            profile_file = os.path.join(self.conan_profiles_dir, profile_name)
+            if not os.path.isfile(profile_file):
+                raise BmcGoException(f"{profile_file} 文件不存在")
+
+            profile, profile_vars = read_profile(profile_name, os.getcwd(), self.conan_profiles_dir)
+            return profile, profile_vars
 
-        profile, profile_vars = read_profile(profile_name, os.getcwd(), self.conan_profiles_dir)
-        return profile, profile_vars
+    def get_conan_package_folder(self, package, install_args):
+        """
+        安装conan包到指定outdir目录
+        """
+        tmpdir = TemporaryDirectory()
+        graph_file = os.path.join(tmpdir.name, "package.json")
+        cmd = f"conan install --requires='{package}' {install_args}"
+        cmd += f" -f json --out-file={graph_file}"
+        self.run_command(cmd, ignore_error=False)
+        return self.get_package_folder_from_graph_file(graph_file, package)
+
+    def install_conan_package_to(self, package, install_args, outdir):
+        """
+        安装conan包到指定outdir目录
+        """
+        package_folder = self.get_conan_package_folder(package, install_args)
+        if not package_folder:
+            raise errors.BmcGoException(f"未找到{package}包的打包目录，可能包不存在或构建失败")
+        for file in os.listdir(package_folder):
+            source = os.path.join(package_folder, file)
+            cmd = ["/usr/bin/cp", "-arf", source, outdir]
+            self.run_command(cmd, ignore_error=False)
 
     def _save_tempfile_safety(self, temp_fd, target_file, show_log=False):
         lock_fd = open(self.lock_file, "r")

bmcgo/worker.py

@@ -120,7 +120,7 @@ class WorkStatusClient():
     @staticmethod
     def __comm(msg, ignore_error):
         fd = socket(AF_INET, SOCK_STREAM)
-        fd.settimeout(2)
+        fd.settimeout(10)
         while True:
             try:
                 fd.connect(("", WORK_SERVER_PORT))
@@ -202,7 +202,7 @@ class WorkerScheduler(Process):
         if self.klass == "":
             return None
         # bmcgo的任务类名固定为TaskClass
-        if self.klass.startswith("bmcgo"):
+        if self.klass.startswith("bingo") or self.klass.startswith("bmcgo"):
             work_path = self.klass
             class_name = "TaskClass"
         else:

{openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.2.dist-info}/METADATA

@@ -1,13 +1,13 @@
 Metadata-Version: 2.1
 Name: openubmc-bingo
-Version: 0.5.277
+Version: 0.6.2
 Summary: Tools provided by openubmc
 Home-page: https://openubmc.cn
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: Mulan Permissive Software License v2 (MulanPSL-2.0)
 Classifier: Operating System :: OS Independent
 Description-Content-Type: text/markdown
-Requires-Dist: conan <=1.62.0
+Requires-Dist: conan
 Requires-Dist: cryptography <=42.0.7
 Requires-Dist: dbus-python
 Requires-Dist: ecdsa
@@ -19,6 +19,7 @@ Requires-Dist: jsonschema
 Requires-Dist: launchpadlib
 Requires-Dist: mako
 Requires-Dist: meson
+Requires-Dist: node-semver ==0.6.1
 Requires-Dist: paramiko <4.0.0
 Requires-Dist: protobuf ==3.19.1
 Requires-Dist: psutil
@@ -27,6 +28,7 @@ Requires-Dist: pyinstaller
 Requires-Dist: pyopenssl <=24.1.0,>=23.2.0
 Requires-Dist: pysftp
 Requires-Dist: pyyaml-include
+Requires-Dist: tqdm ==4.67.1
 Requires-Dist: typeguard ==2.13.3
 
 build and code generate tools