openubmc-bingo 0.5.277__py3-none-any.whl → 0.6.2__py3-none-any.whl

bmcgo/component/build.py

@@ -14,7 +14,8 @@ import os
 import shutil
 import subprocess
 import stat
-import pathlib
+from tempfile import NamedTemporaryFile
+import yaml
 
 from mako.lookup import TemplateLookup
 
@@ -26,7 +27,9 @@ from bmcgo.errors import BmcGoException
 from bmcgo.component.package_info import InfoComp
 from bmcgo.component.component_helper import ComponentHelper
 from bmcgo import misc
+from bmcgo import errors
 from bmcgo.utils.json_validator import JSONValidator
+from bmcgo.component.gen import GenComp
 
 log = Logger()
 tool = Tools()
@@ -35,12 +38,14 @@ cwd_script = os.path.split(os.path.realpath(__file__))[0]
 
 
 class BuildComp():
-    def __init__(self, bconfig: BmcgoConfig, args=None, gen_conanbase=True, service_json="mds/service.json"):
+    def __init__(self, bconfig: BmcgoConfig, args=None, gen_conanbase=True,
+                 service_json="mds/service.json", enable_upload=True):
         self.init_common_params(bconfig)
-        self.info: InfoComp = InfoComp(args, service_json, self.bconfig.partner_mode)
+        self.info: InfoComp = InfoComp(args, service_json, self.bconfig.partner_mode, enable_upload)
         self.set_remote_url()
         self.gen_conanbase(gen_conanbase, service_json)
         self.gen_log()
+        self.graph_file = NamedTemporaryFile(suffix=".graph.json")
 
     @staticmethod
     def get_remote_urls():
@@ -66,10 +71,17 @@ class BuildComp():
         os.environ["LUA_PATH"] = f"{conan_bin}/?.lua"
 
     def gen_conanbase(self, gen_conanbase, service_json):
-        lookup = TemplateLookup(directories=os.path.join(cwd_script, "template"))
+        if misc.conan_v1():
+            lookup = TemplateLookup(directories=os.path.join(cwd_script, "template"))
+        else:
+            lookup = TemplateLookup(directories=os.path.join(cwd_script, "template_v2"))
         if gen_conanbase:
             template = lookup.get_template("conanbase.py.mako")
-            language = ComponentHelper.get_language(service_json=service_json, allow_non_service_json=True)
+            language = self.info.language
+            if language == "c" and misc.conan_v2():
+                args = ["-s", service_json]
+                gen = GenComp(args)
+                gen.run(self.info.codegen_base_version)
             conanbase = template.render(lookup=lookup, pkg=self.info, remote_url=self.remote_url,
                                         codegen_version=self.info.codegen_base_version,
                                         language=language)
@@ -79,7 +91,7 @@ class BuildComp():
             file_handler.close()
 
     def gen_log(self):
-        self.log_file = os.path.join("/tmp", self.info.name + ".log")
+        self.log_file = os.path.join(misc.CACHE_DIR, self.info.name + ".log")
         file_handler = os.fdopen(os.open(self.log_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
                                          stat.S_IWUSR | stat.S_IRUSR), 'w')
         file_handler.close()
@@ -141,15 +153,36 @@ class BuildComp():
 
     def check_conan_profile(self):
         profile = os.path.join(tool.conan_profiles_dir, self.info.profile)
-        luajit_profile = os.path.join(tool.conan_profiles_dir, "profile.luajit.ini")
         if not os.path.isfile(profile):
             raise BmcGoException(f"Profile文件{profile} 不存在，系统可能未初始化，" +
-                                 "请在manifest仓执行bingo build安装产品配套构建工具")
-        if self.info.enable_luajit and not os.path.isfile(luajit_profile):
-            raise BmcGoException(f"Profile文件{luajit_profile} 不存在，系统可能未初始化，" +
-                                "请在manifest仓(2024.07.04之后的版本)执行bingo build安装产品配套构建工具")
+                                 f"请在manifest仓执行{misc.tool_name()} build安装产品配套构建工具")
+        if misc.conan_v1():
+            luajit_profile = os.path.join(tool.conan_profiles_dir, "profile.luajit.ini")
+            if self.info.enable_luajit and not os.path.isfile(luajit_profile):
+                raise BmcGoException(f"Profile文件{luajit_profile} 不存在，系统可能未初始化，" +
+                                    "请在manifest仓(2024.07.04之后的版本)执行bingo build安装产品配套构建工具")
+
+    def upload_conan_v2(self):
+        cmd = f"conan cache path {self.info.package}"
+        ret = tool.run_command(cmd, capture_output=True)
+        recipe_path = ret.stdout.strip()
+        conandata = os.path.join(recipe_path, "conandata.yml")
+        with open(conandata, mode="r") as fp:
+            config_data = yaml.safe_load(fp)
+            cfg = config_data.get("sources", {}).get(self.info.version, {})
+            if not cfg:
+                raise errors.BmcGoException(f"conandata.yml不存在sources/{self.info.version}配置或配置为空")
+            if cfg.get("pwd"):
+                log.error("检查到错误的conandata.yml配置")
+                fp.seek(0, os.SEEK_SET)
+                log.error(fp.read())
+                raise errors.BmcGoException(f"组件目录存在脏数据或未推送到远程仓，禁止上传")
+
+        cmd = [misc.CONAN, "upload"]
+        cmd += ("%s -r %s --force" % (self.info.package, self.info.remote)).split()
+        tool.run_command(cmd, show_log=True)
 
-    def upload(self):
+    def upload_conanv1(self):
         if not self.info.upload:
             return
         pkg = "%s/%s%s" % (self.info.name, self.info.version, self.info.channel)
@@ -172,7 +205,7 @@ class BuildComp():
         cmd += ("%s:%s -r %s --all" % (pkg, pkg_id, self.info.remote)).split()
         tool.run_command(cmd, show_log=True)
 
-    def run(self):
+    def run_conan_v1(self):
         tool.clean_locks()
         self.check_conan_profile()
         self.check_luac()
@@ -184,11 +217,40 @@ class BuildComp():
             shutil.rmtree(cache_dir, ignore_errors=True)
         # 构建当前组件
         append = "%s %s -tf None" % (self.info.cmd_base, from_source)
+        if self.info.no_cache:
+            append += " -u"
         cmd = [misc.CONAN, "create"]
         cmd += append.split()
         tool.run_command(cmd, show_log=True)
         self._check_sr_validation(os.path.join(cache_dir, "package"))
-        self.upload()
+        if self.info.upload:
+            self.upload_conanv1()
+
+    def run_conan_v2(self):
+        self.check_conan_profile()
+        from_source = f"--build={self.info.name}/*"
+        if self.info.from_source:
+            from_source = "--build=*"
+        else:
+            from_source += " --build=missing"
+        # # 构建当前组件
+        args = "%s %s" % (self.info.cmd_base, from_source)
+        if self.info.no_cache:
+            args += " -u"
+        cmd = [misc.CONAN, "create", "--name", self.info.name, "--version", self.info.version]
+        args += f" -f json --out-file={self.graph_file.name}"
+        cmd += args.split()
+        tool.run_command(cmd, show_log=True)
+        package_folder = tool.get_package_folder_from_graph_file(self.graph_file.name, self.info.package)
+        self._check_sr_validation(package_folder)
+        if self.info.upload:
+            self.upload_conan_v2()
+
+    def run(self):
+        if misc.conan_v1():
+            self.run_conan_v1()
+        else:
+            self.run_conan_v2()
 
     def test(self):
         if os.path.isdir("test_package"):

bmcgo/component/component_dt_version_parse.py

@@ -200,7 +200,7 @@ class ManifestConfigParse:
 class ComponentDtVersionParse:
     """分析组件 dt 的依赖组件版本, 实现自动配置 service.json 文件, 达成 dt 构建门禁
     """
-    def __init__(self, parser=None, serv_file="mds/service.json", args=None):
+    def __init__(self, parser=None, serv_file="mds/service.json", args=None, exclude_dt=False):
         """初始化，读取配置
 
         Args:
@@ -221,7 +221,8 @@ class ComponentDtVersionParse:
             self.mds_conf = json.load(mds_conf_fp)
             mds_conf_fp.close()
         # 生成dt与build的依赖列表, 检查时如果发现问题, 直接报出组件名, 不分类
-        conan_dt_list = Function.get_key_value(self.mds_conf, "dependencies/test") or []
+        if not exclude_dt:
+            conan_dt_list = Function.get_key_value(self.mds_conf, "dependencies/test") or []
         conan_build_list = Function.get_key_value(self.mds_conf, "dependencies/build") or []
         # 根据配置文件，已确定为列表，有告警，无需使用list转化，影响效率
         self.conan_list = conan_dt_list + conan_build_list

bmcgo/component/component_helper.py

@@ -19,7 +19,7 @@ from tempfile import NamedTemporaryFile
 from packaging import version
 
 from bmcgo.utils.tools import Tools
-from bmcgo.bmcgo_config import misc
+from bmcgo import misc
 from bmcgo.errors import BmcGoException
 
 STAGE_DEV = "dev"
@@ -40,9 +40,15 @@ class DownloadComponentRecipe(Process):
         self.remote_list = remote_list
 
     def run(self):
-        dep_path = os.path.join(self.tools.conan_data, self.comp.replace("@", "/"))
-        if os.path.isdir(dep_path):
-            return
+        if misc.conan_v1():
+            dep_path = os.path.join(self.tools.conan_data, self.comp.replace("@", "/"))
+            if os.path.isdir(dep_path):
+                return
+        else:
+            cmd = f"conan cache path {self.comp}"
+            ret = self.tools.run_command(cmd, capture_output=True, ignore_error=True)
+            if ret.returncode == 0:
+                return
         self.tools.download_conan_recipes(self.comp, self.remote_list)
 
 
@@ -123,17 +129,26 @@ class ComponentHelper:
     @staticmethod
     def download_recipes(depdencies: List[str], tools: Tools, remote_list):
         pools = []
+        ignore_error = os.getenv("CLOUD_BUILD_IGNORE_ERROR")
         for dep in depdencies:
             task = DownloadComponentRecipe(tools, dep, remote_list)
             task.start()
             pools.append(task)
-
+            # 减少并发数，避免CI场景连接过多导致服务器无法处理导致的失败
+            while len(pools) > 4:
+                time.sleep(0.1)
+                for pool in pools.copy():
+                    if pool.is_alive():
+                        continue
+                    if pool.exitcode is not None and pool.exitcode != 0 and not ignore_error:
+                        tools.log.warning(f"下载组件 ({pool.comp}) 的构建配方(recipe)失败, 退出码: {pool.exitcode}")
+                    pools.remove(pool)
         while pools:
             time.sleep(0.1)
             for pool in pools.copy():
                 if pool.is_alive():
                     continue
-                if pool.exitcode is not None and pool.exitcode != 0:
+                if pool.exitcode is not None and pool.exitcode != 0 and not ignore_error:
                     raise BmcGoException(f"下载组件 ({pool.comp}) 的构建配方(recipe)失败, 退出码: {pool.exitcode}")
                 pools.remove(pool)
 
@@ -144,18 +159,31 @@ class ComponentHelper:
         dependencies = set()
         tempfile = NamedTemporaryFile()
         for comp in components:
-            if not re.match(ComponentHelper.CONAN_PAKCAGE_RE, comp):
-                raise BmcGoException(f"组件 {comp} 不符合正则 {ComponentHelper.CONAN_PAKCAGE_RE}")
-            cmd = f'conan info "{comp}" --remote {remote} --json {tempfile.name}'
+            if misc.conan_v1():
+                if not re.match(ComponentHelper.CONAN_PAKCAGE_RE, comp):
+                    raise BmcGoException(f"组件 {comp} 不符合正则 {ComponentHelper.CONAN_PAKCAGE_RE}")
+                cmd = f'conan info "{comp}" --remote {remote} --json {tempfile.name}'
+            else:
+                if not misc.conan_package_match(comp):
+                    raise BmcGoException(f"组件 {comp} 不符合正则 {misc.CONAN_NAME_RESTR}")
+                cmd = f'conan graph info --requires="{comp}" --remote {remote} -f json --out-file={tempfile.name}'
             tools.run_command(cmd)
 
             file_handler = open(tempfile.name, "r")
             conan_comps = json.load(file_handler)
-            for conan_comp in conan_comps:
-                comp_ref = conan_comp.get("reference", "")
-                if not comp_ref or comp_ref == comp:
-                    continue
-                dependencies.add(comp_ref)
+            if misc.conan_v1():
+                for conan_comp in conan_comps:
+                    comp_ref = conan_comp.get("reference", "")
+                    if not comp_ref or comp_ref == comp:
+                        continue
+                    dependencies.add(comp_ref)
+            else:
+                deps = conan_comps.get("graph", {}).get("nodes", {}).get("0", {}).get("dependencies", {})
+                for _, conan_comp in deps.items():
+                    comp_ref = conan_comp.get("ref", "")
+                    if not comp_ref or comp_ref == comp:
+                        continue
+                    dependencies.add(comp_ref)
             file_handler.close()
 
         return list(dependencies)
@@ -164,5 +192,5 @@ class ComponentHelper:
     def get_user_channel(stage: str):
         if stage == misc.StageEnum.STAGE_DEV.value:
             stage = misc.StageEnum.STAGE_RC.value
-        user_channel = f"@{Tools().conan_user}/{stage}"
+        user_channel = f"@{misc.conan_user()}/{stage}"
         return user_channel

bmcgo/component/coverage/incremental_cov.py

@@ -85,9 +85,9 @@ class GcovHTMLParser(HTMLParser):
             for a in attrs:
                 if a == ("class", "lineNum"):
                     self.is_line_num = True
-                if a == ("class", "lineNoCov"):
+                if a == ("class", "lineNoCov") or a == ("class", "tlaUNC"):
                     self.uncovers.append(self.line_num)
-                if a == ("class", "lineCov"):
+                if a == ("class", "lineCov") or a == ("class", "tlaGNC"):
                     self.covers.append(self.line_num)
 
     def handle_data(self, data):

bmcgo/component/deploy.py

@@ -12,6 +12,7 @@
 import os
 import shutil
 import stat
+import json
 import yaml
 
 from mako.lookup import TemplateLookup
@@ -28,6 +29,24 @@ log = Logger("deploy")
 cwd_script = os.path.split(os.path.realpath(__file__))[0]
 
 
+class GraphNode:
+    def __init__(self, node):
+        if not node:
+            self.name = None
+            self.package_folder = None
+            self.recipe_folder = None
+            self.package_id = None
+            self.ref = None
+            self.node = node
+            return
+        self.name = node.get("name")
+        self.package_folder = node.get("package_folder")
+        self.recipe_folder = node.get("recipe_folder")
+        self.package_id = node.get("package_id")
+        self.ref = node.get("ref")
+        self.node = node
+
+
 class DeployComp():
     def __init__(self, bconfig: BmcgoConfig, info: InfoComp = None):
         self.info: InfoComp = info
@@ -35,6 +54,19 @@ class DeployComp():
         self.folder = bconfig.component.folder
         os.chdir(self.folder)
         self.temp_path = os.path.join(self.folder, "temp")
+        self.graph_nodes: dict[str, GraphNode] = {}
+
+    @staticmethod
+    def copy_packages_v1(install_path, rootfs_path):
+        for sub_dir in os.listdir(install_path):
+            dir_path = os.path.join(install_path, sub_dir)
+            if os.path.isfile(dir_path):
+                os.unlink(dir_path)
+                continue
+            for file in os.listdir(dir_path):
+                source = os.path.join(dir_path, file)
+                cmd = ["/usr/bin/cp", "-arf", source, rootfs_path]
+                Helper.run(cmd)
 
     def get_dt_dependencies(self):
         user_channel = ComponentHelper.get_user_channel(self.info.stage)
@@ -61,12 +93,15 @@ class DeployComp():
         return dependencies
 
     def gen_conanfile(self):
-        dependencies = [f"{self.info.name}/{self.info.version}{self.info.channel}"]
-        if self.info.build_type == "dt":
+        dependencies = [self.info.package]
+        if self.info.test:
             dependencies += self.get_dt_dependencies()
 
         # 构建虚拟deploy组件，生成conanfile.py文件
-        lookup = TemplateLookup(directories=os.path.join(cwd_script, "template"))
+        if misc.conan_v1():
+            lookup = TemplateLookup(directories=os.path.join(cwd_script, "template"))
+        else:
+            lookup = TemplateLookup(directories=os.path.join(cwd_script, "template_v2"))
         template = lookup.get_template("conanfile.deploy.py.mako")
         conanfile = template.render(lookup=lookup, pkg=self.info, dependencies=dependencies)
         file_handler = os.fdopen(os.open("conanfile.py", os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
@@ -74,6 +109,25 @@ class DeployComp():
         file_handler.write(conanfile)
         file_handler.close()
 
+    def copy_packages_v2(self, graph_file, rootfs_path=None):
+        if not rootfs_path:
+            rootfs_path = self.temp_path
+        with open(graph_file, "r") as file_handler:
+            package_info = json.load(file_handler)
+        nodes: dict[str, dict] = package_info.get("graph", {}).get("nodes", {})
+        self.graph_nodes = {}
+        for _, node in nodes.items():
+            if node.get("context") != "host":
+                continue
+            gn = GraphNode(node)
+            self.graph_nodes[gn.name] = gn
+            if not gn.package_folder:
+                continue
+            for file in os.listdir(gn.package_folder):
+                source = os.path.join(gn.package_folder, file)
+                cmd = ["/usr/bin/cp", "-arf", source, rootfs_path]
+                Helper.run(cmd)
+
     def run(self):
         # 生成虚拟deploy组件，仅用于安装
         deploy_conan = os.path.join(self.temp_path, ".deploy")
@@ -86,8 +140,13 @@ class DeployComp():
         log.info("安装所有依赖到目录 %s", install_path)
         shutil.rmtree(install_path, ignore_errors=True)
         cmd = [misc.CONAN, "install"]
-        append_cmd = ("%s -if=%s -g deploy" % (self.info.cmd_base, install_path))
-        append_cmd = append_cmd.replace(self.info.package, self.info.channel)
+        graph_file = "package.info.json"
+        if misc.conan_v2():
+            append_cmd = ("%s -of=%s -d=full_deploy -f json --out-file=%s" %
+                          (self.info.cmd_base, install_path, graph_file))
+        else:
+            append_cmd = ("%s -if=%s -g deploy" % (self.info.cmd_base, install_path))
+            append_cmd = append_cmd.replace(self.info.package, self.info.channel)
         cmd += append_cmd.split()
         cmd.append("--build=missing")
         log.success("运行部署命令: %s", " ".join(cmd))
@@ -96,13 +155,8 @@ class DeployComp():
         rootfs_path = self.temp_path
         log.info("复制所有依赖到目录 %s", rootfs_path)
         os.makedirs(rootfs_path, exist_ok=True)
-        for sub_dir in os.listdir(install_path):
-            dir_path = os.path.join(install_path, sub_dir)
-            if os.path.isfile(dir_path):
-                os.unlink(dir_path)
-                continue
-            for file in os.listdir(dir_path):
-                source = os.path.join(dir_path, file)
-                cmd = ["/usr/bin/cp", "-arf", source, rootfs_path]
-                Helper.run(cmd)
+        if misc.conan_v2():
+            self.copy_packages_v2(graph_file, rootfs_path)
+        else:
+            self.copy_packages_v1(install_path, rootfs_path)
         shutil.rmtree(install_path, ignore_errors=True)

bmcgo/component/gen.py

@@ -107,7 +107,7 @@ class GenComp():
     def run(self, base_version=-1):
         language = ComponentHelper.get_language().lower()
         parser = argparse.ArgumentParser(description="代码自动生成.",
-                                         prog="bingo gen/当前只支持C和lua语言代码生成",
+                                         prog=f"{misc.tool_name()} gen/当前只支持C和lua语言代码生成",
                                          formatter_class=SmartFormatter)
         parser.add_argument("-v", "--version", help="use specified version", type=int, default=-1)
         parser.add_argument("-r", "--remote", help="conan remote, 如不设置则默认读取本地conan配置")