openubmc-bingo 0.6.25__py3-none-any.whl → 0.6.27__py3-none-any.whl

bmcgo/__init__.py

@@ -9,4 +9,4 @@
 # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
 # See the Mulan PSL v2 for more details.
 
-__version__ = '0.6.25'
+__version__ = '0.6.27'

bmcgo/tasks/task_build_conan.py

@@ -25,6 +25,7 @@ import time
 import subprocess
 import random
 import pathlib
+from tempfile import NamedTemporaryFile
 from multiprocessing import Process, Queue
 from copy import deepcopy
 from collections import Counter
@@ -699,6 +700,30 @@ class TaskClass(Task):
             options += " -o */*:gcov=True"
         return options
 
+    def clean_folder_not_exist_packages(self):
+        """
+        检查缓存的所有包路径是否存在，不存在的删除对应的包
+        缓解"folder must exist"错误
+        """
+        tmp = NamedTemporaryFile(suffix=".json")
+        cmd = f"conan list -f json '*/*@*/*#*:*' --out-file={tmp.name}"
+        self.run_command(cmd)
+        with open(tmp.name, "r") as fd:
+            cache = json.load(fd)
+        for name, recipe in cache.get("Local Cache", {}).items():
+            for rrid, revision in recipe.get("revisions", {}).items():
+                for pid, _ in revision.get("packages", {}).items():
+                    cmd = f"conan cache path {name}#{rrid}:{pid}"
+                    path = self.run_command(cmd, capture_output=True).stdout.strip()
+                    if not os.path.isdir(path):
+                        cmd = f"conan remove {name}#{rrid}:{pid} -c"
+                        self.run_command(cmd)
+                cmd = f"conan cache path {name}#{rrid}"
+                path = self.run_command(cmd, capture_output=True).stdout.strip()
+                if not os.path.isdir(path):
+                    cmd = f"conan remove {name}#{rrid} -c"
+                    self.run_command(cmd)
+
     def install_openubmc_v2(self):
         profile_file = os.path.join(self.tools.conan_profiles_dir, self.config.profile)
         if not os.path.isfile(profile_file):
@@ -716,12 +741,17 @@ class TaskClass(Task):
         self.success(f"start build dependency packages of {self.config.board_name}")
         if self.skip_install_comp:
             return
-        bcp = BuildConans(self.graphfile, self.lockfile, base_cmd, self.config.from_source, self.config.log_path)
-        bcp.build()
-
+        if self.config.from_source:
+            """
+            多进程并行构建组件
+            conan原生不支持多组件并行构建，BuildConans会尝试构建所有组件但忽略构建失败
+            """
+            bcp = BuildConans(self.graphfile, self.lockfile, base_cmd, self.config.from_source, self.config.log_path)
+            bcp.build()
+            self.clean_folder_not_exist_packages()
         cmd = f"conan create . {base_cmd} --build=missing"
         self.info(f"start build {self.config.board_name}: {cmd}")
-        self.run_command(cmd, capture_output=False)
+        self.run_command(cmd)
 
         # 检查使用到的组件是否都在单板目录 manifest.yml 中配置了
         self.component_check.run()
@@ -990,9 +1020,7 @@ class TaskClass(Task):
         for name, node in self.graph_nodes.items():
             if node.package_folder.startswith(self.conan_install):
                 continue
-            cust = os.path.join(node.package_folder, "include", "customization.py")
-            if not os.path.isfile(cust):
-                continue
+
             comp_dir = os.path.join(self.conan_install, name)
             shutil.copytree(node.package_folder, comp_dir)
 

bmcgo/utils/build_conans.py

@@ -37,8 +37,6 @@ class GraphNode():
         self.pkg = self.ref.split("#")[0] + "-" + self.context
         self.is_host = self.context == "host"
         self.building = False
-        binary = node.get("binary")
-        self.binary_exist = (binary in ["Cache"])
         self.builded = False
 
     @property
@@ -79,7 +77,6 @@ class BuildConans(object):
                 skip = True
                 continue
             self.cmd += f"{arg} "
-        self.exception = None
         self.force_build = force_build
         self.log_dir = log_dir
         if not os.path.isdir(log_dir):
@@ -93,45 +90,35 @@ class BuildConans(object):
         cmd += f" --build=missing -f json --lockfile={self.lockfile}"
         if self.force_build:
             cmd += f" --build=\"{node.name}/*\""
-        try:
-            # 最多重试5次
-            for i in range(1, 5):
-                logfile = f"{self.log_dir}/conan_build_{node.name}.log"
-                log.debug(f">>>> {cmd}")
-                log.info(f">>>> build {node.ref} start, logfile: {logfile}")
-                with os.fdopen(os.open(logfile, os.O_RDWR | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), "a+") as f:
-                    real_cmd = shlex.split(cmd)
-                    pipe = subprocess.Popen(real_cmd, stdout=f, stderr=f)
-                    start_time = time.time()
-                    while True:
-                        if pipe.poll() is not None:
-                            break
-                        # 1800秒超时
-                        if time.time() - start_time > 1800:
-                            pipe.kill()
-                            raise errors.BmcGoException(f"================== {node.name} 构建超时 ==================")
-                        time.sleep(1)
-                    if pipe.returncode != 0:
-                        # 首次增量构建失败时切换到源码构建模式
-                        if i == 1 and not self.force_build:
-                            cmd += f" --build=\"{node.name}/*\""
-                            continue
-
-                        f.seek(0)
-                        conan_log = f.read()
-                        # conan的缓存数据库不支持并发访问，会概率性存在数据库锁错误，当前通过重试解决
-                        if "sqlite3.OperationalError: database is locked" in conan_log:
-                            log.info("rebuild because database is locked")
-                            continue
-                        log.error(f"================== {node.name} 构建失败日志起始位置 ==================")
-                        log.error(conan_log)
-                        log.error(f"================== {node.name} 构建失败日志结束位置 ==================")
-                        raise errors.BmcGoException(f"================== {node.name} 构建失败 ==================")
-                    else:
-                        break
-            log.success(f"<<<< build {node.ref} finished")
-        except Exception as e:
-            self.exception = e
+        logfile = f"{self.log_dir}/conan_build_{node.name}.log"
+        logfd = os.fdopen(os.open(logfile, os.O_RDWR | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), "a+")
+        # 最多重试3次
+        max_cnt = 3
+        for i in range(0, max_cnt):
+            log.debug(f">>>> {cmd}")
+            log.info(f">>>> build {node.ref} start, logfile: {logfile}")
+            real_cmd = shlex.split(cmd)
+            pipe = subprocess.Popen(real_cmd, stdout=logfd, stderr=logfd)
+            start_time = time.time()
+            while True:
+                if pipe.poll() is not None:
+                    break
+                # 1800秒超时
+                if time.time() - start_time > 1800:
+                    pipe.kill()
+                    log.info(f"================== {node.name} 构建超时 ==================")
+                    self.queue.put(node)
+                    return
+                time.sleep(1)
+            if pipe.returncode != 0:
+                log.info(f"<<<< build {node.ref} not ok")
+                # 首次增量构建失败时切换到源码构建模式
+                if i == 0 and not self.force_build:
+                    cmd += f" --build=\"{node.name}/*\""
+                    continue
+            else:
+                break
+        log.success(f"<<<< build {node.ref} finished")
         self.queue.put(node)
 
     def build(self):
@@ -213,28 +200,25 @@ class BuildConans(object):
                     continue
                 if dep.builded:
                     continue
-                if not dep.binary_exist or self.force_build:
-                    # 当依赖的构建工具存在正在构建的组件时不能构建
-                    skip_build = False
-                    for _, build_dep in dep.build_packages.items():
-                        # 正在构建且未构建出制品时
-                        if build_dep.building and not build_dep.builded:
-                            skip_build = True
-                            break
-                    if skip_build:
-                        continue
-                    # 启动构建前将其依赖的构建工具置为正在构建
-                    for _, build_dep in dep.build_packages.items():
-                        build_dep.building = True
-                    tasks_cnt += 1
-                    thread = Thread(target=self.build_task, args=(dep, options,))
-                    thread.start()
+                # 当依赖的构建工具存在正在构建的组件时不能构建
+                skip_build = False
+                for _, build_dep in dep.build_packages.items():
+                    # 正在构建且未构建出制品时
+                    if build_dep.building and not build_dep.builded:
+                        skip_build = True
+                        break
+                if skip_build:
+                    continue
+                # 启动构建前将其依赖的构建工具置为正在构建
+                for _, build_dep in dep.build_packages.items():
+                    build_dep.building = True
+                tasks_cnt += 1
+                thread = Thread(target=self.build_task, args=(dep, options,))
+                thread.start()
                 build_tasks[dep.name] = True
             if not tasks_cnt:
                 return
             dep = self.queue.get()
-            if not dep or self.exception:
-                raise self.exception
             dep.builded = True
             build_tasks[dep.name] = False
             tasks_cnt -= 1

bmcgo/utils/tools.py

@@ -77,6 +77,7 @@ class Tools():
         file_handle = os.fdopen(os.open(self.lock_file, os.O_WRONLY | os.O_CREAT,
                                 stat.S_IWUSR | stat.S_IRUSR), 'a')
         file_handle.close()
+        self.last_succ_remote = ""
 
     @property
     def user_home(self):
@@ -488,19 +489,20 @@ class Tools():
         return conan_remote_list
 
     def download_conan_recipes(self, conan_version, conan_remote_list):
-        download_flag = False
         args = "--only-recipe" if misc.conan_v2() else "--recipe"
-        for remote in conan_remote_list:
+        remote_list = conan_remote_list
+        if self.last_succ_remote:
+            remote_list.insert(0, self.last_succ_remote)
+        for remote in remote_list:
             # 重试3次
             for _ in range(1, 3):
                 try:
                     self.run_command(f"conan download {conan_version} -r {remote} {args}", show_error_log=False)
-                    download_flag = True
-                    break
+                    self.last_succ_remote = remote
+                    return
                 except Exception as e:
                     self.log.info(f"Recipe not fount in {remote}: {conan_version}")
-        if not download_flag:
-            raise BmcGoException(f"Download {conan_version} failed")
+        raise BmcGoException(f"Download {conan_version} failed")
 
     def yaml_load_template(self, yaml_name: str, template: dict, need_validate=True):
         with open(yaml_name, "r", encoding="utf-8") as fp:

{openubmc_bingo-0.6.25.dist-info → openubmc_bingo-0.6.27.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: openubmc-bingo
-Version: 0.6.25
+Version: 0.6.27
 Summary: Tools provided by openubmc
 Home-page: https://openubmc.cn
 Classifier: Programming Language :: Python :: 3

{openubmc_bingo-0.6.25.dist-info → openubmc_bingo-0.6.27.dist-info}/RECORD

@@ -1,4 +1,4 @@
-bmcgo/__init__.py,sha256=UIcAX3HFWIv4SVR6dvGzKjsZ-Hcy6x6_GwpA6css2D4,562
+bmcgo/__init__.py,sha256=oqZ8oOARtejTeAecTilQ7hon25T3svORSMnhnL6HQwY,562
 bmcgo/bmcgo.py,sha256=uD4TsfjrFB5aQPIS6WRUVc9ShXX-dSImY9ezkB13g1w,685
 bmcgo/bmcgo_config.py,sha256=-HZcTsnccE5wUsgGJ59kq8sDdu8feftIbpyoFvyvBhU,11264
 bmcgo/errors.py,sha256=QW1ndrJcJ2Ws7riOznPKVvZsNlrYk73eZol7w8gJTPU,3076
@@ -224,7 +224,7 @@ bmcgo/tasks/__init__.py,sha256=VapgzhPMmB7dzPIRohoVm1jjfVn_v97cYNCRmkxScaU,539
 bmcgo/tasks/download_buildtools_hm.py,sha256=f4UxStARc8Z8DnT_5O6ONajQ7P0sKyJ8brixr43fHVQ,5988
 bmcgo/tasks/misc.py,sha256=GK_bSDLGZW0FxywB2ICG1iIEz2y2QoCb1YQQk8SYOIA,711
 bmcgo/tasks/task.py,sha256=w8tdU5NwoUfVF4xKFvSzcCWCf0FGTBHJAToNLlP0p4w,18424
-bmcgo/tasks/task_build_conan.py,sha256=cpGFUgwzp2WmrZr9ISFdPUDO5kPuEWqJWTFFdL_1MLk,51173
+bmcgo/tasks/task_build_conan.py,sha256=EpxDBOSmj9YIDSF8ifG0lREIAhOfQbWjt29lmO6mI1g,52546
 bmcgo/tasks/task_build_rootfs_img.py,sha256=JKEvldJnLWu2IdVSntuVowocQwdVtBQUpxzhplYauPI,29209
 bmcgo/tasks/task_build_wbd_up.py,sha256=X9-0Qqad-s3mGfJBMeBQvfZ99KlWcgaMluDr_zv6Z-o,3122
 bmcgo/tasks/task_buildgppbin.py,sha256=epBxxrYFPgoTzpBgPzSRXcUs-ia_BJWsLGflylEJS7I,6614
@@ -241,7 +241,7 @@ bmcgo/tasks/conan/conanfile.py,sha256=MI6c2QNKixrkA5Tx4i9EKw7bZ20GVDIKgkaMwNHQKc
 bmcgo/utils/__init__.py,sha256=BDXz8BcSlCkfo5UYt6j2rm89-HiYA1ZzfpFhy99MH-0,538
 bmcgo/utils/basic_enums.py,sha256=L5VtHtzSvs6duAnphgqDGXX80Wit3Y7DjMlpi9MCxyI,1348
 bmcgo/utils/buffer.py,sha256=t1SkWntWksboNFMPsltslwRdXZi3FAe8eV0JAAE7Vto,4004
-bmcgo/utils/build_conans.py,sha256=3uGn9MWlW3d8WNKZQPs4rEd6E6KxP_eyqSxSlSQQG4o,10172
+bmcgo/utils/build_conans.py,sha256=27aPzS7Q6e6xNlK9pkGgqJBe_pIs8YJHJXxqbaPoCN8,8920
 bmcgo/utils/combine_json_schemas.py,sha256=08JrAlLeo_JgUqzYcZNgSwJZPLfjbWVJ4esPPt9bPMY,7967
 bmcgo/utils/component_post.py,sha256=rTSMv36geI6rbm6ZFQenZfG0mn1nVPpdJqn7g8bYtKA,2330
 bmcgo/utils/component_version_check.py,sha256=nKfav7EnJ_JWfCvH_SiyhhIapGXUzhFs1QvLb58MPBs,6266
@@ -252,7 +252,7 @@ bmcgo/utils/json_validator.py,sha256=_k5wU78wfYGrzvSDaqOEtT4otgKUjquVhZNpVf2PW_c
 bmcgo/utils/mapping_config_patch.py,sha256=_gKfZnrvsLPgHn1yXhEJRVTAeuGpeGD9T-Pqyw5Ydys,16827
 bmcgo/utils/merge_csr.py,sha256=JNxHCfW1au84WQshdz0aQy8wMTEOHonjQT3s6LjlZN4,4580
 bmcgo/utils/perf_analysis.py,sha256=fh6lV9AAKVhpPkGPwAJ8EWfGfUoHjqGYQxrvc32Xiac,4767
-bmcgo/utils/tools.py,sha256=NrYd4bNzRiBGu7FklN6PT1gj-sdqUWB_lj11Q7TjI8Y,32480
+bmcgo/utils/tools.py,sha256=dB7lX0jv-mE6pBalzm-VUZEnxPSbYkgC9soD_FAAnFg,32587
 bmcgo/utils/installations/README.md,sha256=hKXnFYmeHEuROFMFE-vzlGLSHg71bei5ZYwyYZphWNk,2
 bmcgo/utils/installations/__init__.py,sha256=BDXz8BcSlCkfo5UYt6j2rm89-HiYA1ZzfpFhy99MH-0,538
 bmcgo/utils/installations/base_installer.py,sha256=3UZiKWoa41ygRbLD3QsE2FTp-VFp79V0I53QLRIf4E8,5902
@@ -264,8 +264,8 @@ bmcgo/utils/installations/install_plans/qemu.yml,sha256=lT7aKag60QUH6hTGFKa3hGRq
 bmcgo/utils/installations/install_plans/studio.yml,sha256=APR3GM-3Q11LFfe8vh7t3LztDn4LLEpNHRUNjkaVekA,143
 bmcgo/utils/installations/installers/apt_installer.py,sha256=nPaCb4cobSi9InN_aHsEPtQ0k4FgsCUWE5_VgBPvcRE,3769
 bmcgo/utils/installations/installers/pip_installer.py,sha256=dDdios1EQ7fzt90r02pZeoM3jCmjslLzkSvzd2hgRVM,3241
-openubmc_bingo-0.6.25.dist-info/METADATA,sha256=9npzn3E3oo6oiNlQGepfcjj0msWe2jDEmSlhBefEm5E,1010
-openubmc_bingo-0.6.25.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-openubmc_bingo-0.6.25.dist-info/entry_points.txt,sha256=UUoUP-vAWTgg9vEYbRwYqOBHgpRtkngdzMPb-ocz90g,42
-openubmc_bingo-0.6.25.dist-info/top_level.txt,sha256=9AcvCAt1nZcOgMsGt6T07mg2Bgtdet-3mHTwg91axgI,6
-openubmc_bingo-0.6.25.dist-info/RECORD,,
+openubmc_bingo-0.6.27.dist-info/METADATA,sha256=RJul5-nRe2BswGFKly97xf_nQAlfpNu2_AeO1OOOAs0,1010
+openubmc_bingo-0.6.27.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+openubmc_bingo-0.6.27.dist-info/entry_points.txt,sha256=UUoUP-vAWTgg9vEYbRwYqOBHgpRtkngdzMPb-ocz90g,42
+openubmc_bingo-0.6.27.dist-info/top_level.txt,sha256=9AcvCAt1nZcOgMsGt6T07mg2Bgtdet-3mHTwg91axgI,6
+openubmc_bingo-0.6.27.dist-info/RECORD,,