lbkit 0.9.8__tar.gz → 0.9.10__tar.gz

{lbkit-0.9.8/lbkit.egg-info → lbkit-0.9.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lbkit
-Version: 0.9.8
+Version: 0.9.10
 Summary: Tools provided by litebmc.com
 Home-page: https://www.litebmc.com
 Author: xuhj@litebmc.com

lbkit-0.9.10/lbkit/__init__.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.9.10'

{lbkit-0.9.8 → lbkit-0.9.10}/lbkit/component/build.py

@@ -33,29 +33,23 @@ def _clang_format_file(args):
 
 
 class DeployComponent():
-    def __init__(self, package_ref, package_id, rootfs_dir):
+    def __init__(self, package_ref, package_id, rootfs_dir, package_folder=None):
         self.package_ref = package_ref
         self.package_id = package_id
         self.rootfs_dir = rootfs_dir
+        self.package_folder = package_folder
 
     def run(self):
         if self.package_ref.startswith("deploy"):
             return
-        cmd = f"conan cache path {self.package_ref}:{self.package_id}"
-        package_folder = tools.run(cmd).stdout.strip()
+        if self.package_folder:
+            package_folder = self.package_folder
+        else:
+            cmd = f"conan cache path {self.package_ref}:{self.package_id}"
+            package_folder = tools.run(cmd).stdout.strip()
         log.info(f">>>> deploy {self.package_ref}")
-        cmd = f"cp -rT {package_folder}/ {self.rootfs_dir}"
-        cnt = 10
-        while cnt > 0:
-            try:
-                cnt -= 1
-                tools.exec(cmd)
-                return
-            except Exception as e:
-                if cnt == 0:
-                    log.warn("Copy failed, msg: " + str(e))
-                    raise e
-                log.info("Copy {self.package_ref} failed, try again")
+        cmd = f"rsync -aHK {package_folder}/ {self.rootfs_dir}"
+        tools.exec(cmd, verbose=True)
 
 
 class BuildComponent():
@@ -255,7 +249,7 @@ class BuildComponent():
                 continue
             if ref.startswith(self.package):
                 self.package_id = id
-            dep = DeployComponent(ref, id, self.rootfs_dir)
+            dep = DeployComponent(ref, id, self.rootfs_dir, info.get("package_folder"))
             pool.apply_async(dep.run, error_callback=self._copy_failed)
         pool.close()
         pool.join()
@@ -280,7 +274,7 @@ class BuildComponent():
         cmd = f"conan create {self.base_cmd} --build='{self.name}/*'"
         tools.exec(cmd, verbose=True)
 
-        graph_cmd = f"conan graph info . {self.base_cmd} -f json --lockfile={self.lockfile}"
+        graph_cmd = f"conan install . {self.base_cmd} --lockfile={self.lockfile} -f json"
         tools.pipe([graph_cmd], out_file=self.graphfile)
 
         log.success(f"start deploy {self.package} and is's dependency packages")

{lbkit-0.9.8 → lbkit-0.9.10}/lbkit/component/test.py

@@ -66,13 +66,27 @@ class TestComponent():
         index_file = os.path.join(coverage_dir, "html/index.html")
         with open(index_file, "r") as fp:
             content = fp.read()
-        matches = re.search(r"Lines:</td\>\n.*headerCovTableEntry\"\>([0-9]+).*\n.*headerCovTableEntry\"\>([0-9]+).*\n.*headerCovTableEntry(Lo|Hi|Med)\"\>([0-9.]+) %", content)
-        if matches is None:
-            raise errors.LiteBmcException(f"Read line coverage data from {index_file} failed")
-        line_hit = int(matches.group(1))
-        line_total = int(matches.group(2))
-        line_cov = float(matches.group(4))
-        line_level = matches.group(3)
+
+        # LCOV版本间HTML格式差异：
+        # LCOV 1.x (Ubuntu 22.04): 列顺序为 Hit, Total, Coverage%
+        # LCOV 2.0 (Ubuntu 24.04+): 列顺序为 Coverage%, Total, Hit
+        # 优先匹配LCOV 2.0格式，失败则回退LCOV 1.x格式
+        lcov2_cov_line = r"Lines:</td\>\n.*headerCovTableEntry(Lo|Hi|Med)\"\>([0-9.]+).*%.*\n.*headerCovTableEntry\"\>([0-9]+).*\n.*headerCovTableEntry\"\>([0-9]+)"
+        lcov1_cov_line = r"Lines:</td\>\n.*headerCovTableEntry\"\>([0-9]+).*\n.*headerCovTableEntry\"\>([0-9]+).*\n.*headerCovTableEntry(Lo|Hi|Med)\"\>([0-9.]+).*%"
+        matches = re.search(lcov2_cov_line, content)
+        if matches:
+            line_level = matches.group(1)
+            line_cov = float(matches.group(2))
+            line_total = int(matches.group(3))
+            line_hit = int(matches.group(4))
+        else:
+            matches = re.search(lcov1_cov_line, content)
+            if matches is None:
+                raise errors.LiteBmcException(f"Read line coverage data from {index_file} failed")
+            line_hit = int(matches.group(1))
+            line_total = int(matches.group(2))
+            line_level = matches.group(3)
+            line_cov = float(matches.group(4))
         log.info("Coverage info:")
         if line_level == "Hi":
             log.success(f"Line:     hit %-10d total %-10d coverage %.02f %% (High)" % (line_hit, line_total, line_cov))
@@ -80,13 +94,23 @@ class TestComponent():
             log.warn(f"Line:     hit %-10d total %-10d coverage %.02f %% (Medium)" % (line_hit, line_total, line_cov))
         elif line_level == "Lo":
             log.warn(f"Line:     hit %-10d total %-10d coverage %.02f %% (Low!!!)" % (line_hit, line_total, line_cov))
-        matches = re.search(r"Functions:</td\>\n.*headerCovTableEntry\"\>([0-9]+).*\n.*headerCovTableEntry\"\>([0-9]+).*\n.*headerCovTableEntry(Lo|Hi|Med)\"\>([0-9.]+) %", content)
-        if matches is None:
-            raise errors.LiteBmcException(f"Read function coverage data from {index_file} failed")
-        func_hit = int(matches.group(1))
-        func_total = int(matches.group(2))
-        func_cov = float(matches.group(4))
-        func_level = matches.group(3)
+
+        lcov2_cov_func = r"Functions:</td\>\n.*headerCovTableEntry(Lo|Hi|Med)\"\>([0-9.]+).*%.*\n.*headerCovTableEntry\"\>([0-9]+).*\n.*headerCovTableEntry\"\>([0-9]+)"
+        lcov1_cov_func = r"Functions:</td\>\n.*headerCovTableEntry\"\>([0-9]+).*\n.*headerCovTableEntry\"\>([0-9]+).*\n.*headerCovTableEntry(Lo|Hi|Med)\"\>([0-9.]+).*%"
+        matches = re.search(lcov2_cov_func, content)
+        if matches:
+            func_level = matches.group(1)
+            func_cov = float(matches.group(2))
+            func_total = int(matches.group(3))
+            func_hit = int(matches.group(4))
+        else:
+            matches = re.search(lcov1_cov_func, content)
+            if matches is None:
+                raise errors.LiteBmcException(f"Read function coverage data from {index_file} failed")
+            func_hit = int(matches.group(1))
+            func_total = int(matches.group(2))
+            func_level = matches.group(3)
+            func_cov = float(matches.group(4))
         if func_level == "Hi":
             log.success(f"Function: hit %-10d total %-10d coverage %.02f %% (High)" % (func_hit, func_total, func_cov))
         elif func_level == "Med":

lbkit-0.9.10/lbkit/download_cache.py

@@ -0,0 +1,112 @@
+"""下载缓存模块
+
+提供共享下载缓存，避免重复下载相同的文件。
+缓存文件以 sha256 命名存储在 ~/.cache/litebmc/download/ 目录下，
+支持多进程并发安全（临时文件 + 原子重命名模式）。
+"""
+import os
+import time
+import shutil
+import requests
+
+from lbkit.misc import DOWNLOAD_CACHE_DIR
+from lbkit.tools import Tools
+
+
+def get_cache_dir():
+    """获取缓存目录路径。
+
+    解析优先级：
+    1. LBKIT_DOWNLOAD_CACHE 环境变量（设为空字符串则禁用缓存）
+    2. 默认路径 ~/.cache/litebmc/download/
+
+    Returns:
+        str | None: 缓存目录路径，禁用时返回 None
+    """
+    env = os.environ.get("LBKIT_DOWNLOAD_CACHE")
+    if env is not None:
+        if env == "":
+            return None
+        cache_dir = env
+    else:
+        cache_dir = DOWNLOAD_CACHE_DIR
+    try:
+        os.makedirs(cache_dir, exist_ok=True)
+        return cache_dir
+    except OSError:
+        return None
+
+
+def cache_lookup(cache_dir, sha256):
+    """在缓存中查找文件。
+
+    Args:
+        cache_dir: 缓存目录路径
+        sha256: 期望的文件 sha256 值
+
+    Returns:
+        str | None: 缓存文件路径，未命中返回 None
+    """
+    cache_file = os.path.join(cache_dir, sha256)
+    if not os.path.isfile(cache_file):
+        return None
+    calc_sha = Tools.file_digest_sha256(cache_file)
+    if calc_sha != sha256:
+        return None
+    return cache_file
+
+
+def cache_download_and_store(cache_dir, url, sha256, verify=True):
+    """下载文件并存入缓存。
+
+    下载到临时文件，校验 sha256 后原子重命名为 {sha256}。
+
+    Args:
+        cache_dir: 缓存目录路径
+        url: 下载地址
+        sha256: 期望的 sha256 值
+        verify: SSL 证书校验开关
+
+    Returns:
+        str: 缓存文件路径
+
+    Raises:
+        Exception: 下载失败或 sha256 不匹配
+    """
+    os.makedirs(cache_dir, exist_ok=True)
+    tmp_path = os.path.join(cache_dir, f".tmp_{os.getpid()}_{time.monotonic_ns()}")
+    try:
+        req = requests.get(url, stream=True, verify=verify, timeout=30)
+        req.raise_for_status()
+        total_size = int(req.headers.get('content-length', 0))
+        total_down = 0
+        last = time.time()
+        with open(tmp_path, 'wb') as fp:
+            for chunk in req.iter_content(chunk_size=16384):
+                if chunk:
+                    fp.write(chunk)
+                    total_down += len(chunk)
+                    now = time.time()
+                    if now - last > 30:
+                        print(f"Downloading to cache, {total_down} / {total_size}")
+                        last = now
+        calc_sha = Tools.file_digest_sha256(tmp_path)
+        if sha256 != "any" and calc_sha != sha256:
+            raise Exception(f"Cache download sha256 mismatch, need: {sha256}, get: {calc_sha}")
+        cache_file = os.path.join(cache_dir, sha256)
+        os.rename(tmp_path, cache_file)
+        return cache_file
+    except Exception:
+        if os.path.isfile(tmp_path):
+            os.unlink(tmp_path)
+        raise
+
+
+def cache_copy_to_dst(cache_file, dst):
+    """从缓存复制文件到目标路径。
+
+    Args:
+        cache_file: 缓存文件路径
+        dst: 目标文件路径
+    """
+    shutil.copyfile(cache_file, dst)

{lbkit-0.9.8 → lbkit-0.9.10}/lbkit/misc.py

@@ -13,6 +13,7 @@ from jsonschema import validate, ValidationError
 from lbkit.errors import PackageConfigException, HttpRequestException
 
 LOG_DIR = os.path.join(Path.home(), ".cache", "lbkit", "log")
+DOWNLOAD_CACHE_DIR = os.path.join(Path.home(), ".cache", "litebmc", "download")
 TARGETS_DIR = "/usr/share/litebmc/targets"
 
 

{lbkit-0.9.8 → lbkit-0.9.10}/lbkit/tasks/image_maker/make_qemu_image.py

@@ -1,9 +1,9 @@
 """环境准备"""
 import os
-import tempfile
 from lbkit.tasks.config import Config
 from lbkit.utils.images.emmc import MakeImage as MekeEmmcImage
 from lbkit.tasks.image_maker.make_image import TaskMakeImage
+from lbkit.utils.fakeroot import Fakeroot
 
 src_cwd = os.path.split(os.path.realpath(__file__))[0]
 
@@ -17,7 +17,6 @@ class TaskMakeQemuImage(TaskMakeImage):
         self.chip_model = chip_model
         self.uboot = self.config.get_product_config("flash/uboot")
         self.kernel = self.config.get_product_config("flash/kernel")
-        self.rootfs = self.config.rootfs_img
 
     @staticmethod
     def _trans_to_blk_1m(cfg: str):
@@ -37,26 +36,38 @@ class TaskMakeQemuImage(TaskMakeImage):
 
     def run(self):
         """任务入口"""
-        """检查manifest文件是否满足schema格式描述"""
         os.chdir(self.config.output_path)
+        mnt_path = self.config.mnt_path
+        # 加载task_build_rootfs保存的fakeroot状态，继承权限配置
+        fakeroot_file = os.path.join(self.config.temp_path, "make_rootfs", "fakeroot.db")
+        fakeroot = Fakeroot(fakeroot_file)
+        fakeroot.start()
+        try:
+            self.config.fakeroot = fakeroot
+            # 复制内核文件到boot/Image
+            cmd = f"cp {self.kernel} {mnt_path}/boot/Image"
+            self.exec(cmd)
+            cmd = f"chown 0:0 {mnt_path}/boot/Image"
+            self.exec(cmd)
+            # 重新生成rootfs镜像
+            if os.path.exists(self.config.rootfs_img):
+                os.unlink(self.config.rootfs_img)
+            rootfs_size = self.config.get_product_config("rootfs/size")
+            size_mb = int(self._trans_to_blk_1m(rootfs_size)) if rootfs_size else 1024
+            cmd = f'mkfs.ext4 -d {mnt_path} -r 1 -N 0 -m 5 -L "rootfs" -O ^64bit {self.config.rootfs_img} "{size_mb}M"'
+            self.exec(cmd)
+        finally:
+            self.config.fakeroot = None
+            fakeroot.stop()
 
-        mk = MekeEmmcImage(os.path.join(self.config.temp_path, "emmc_tmp_dir"))
+        # 制作eMMC镜像
+        mk = MekeEmmcImage()
         rootfs_size = self.config.get_product_config("rootfs/size")
         if rootfs_size:
             mk.rootfs_blk_1m = self._trans_to_blk_1m(rootfs_size)
         emmc_size = self.config.get_product_config("flash/size")
         if emmc_size:
             mk.size_1m = self._trans_to_blk_1m(emmc_size)
-        tmpdir = tempfile.TemporaryDirectory()
-        cmd = f"fuse2fs {self.rootfs} {tmpdir.name} -o fakeroot"
-        self.exec(cmd)
-        # 复制内核文件到boot/extlinux/Image
-        cmd = f"cp {self.kernel} {tmpdir.name}/boot/Image"
-        self.exec(cmd)
-        cmd = f"chown 0:0 {tmpdir.name}/boot/Image"
-        self.exec(cmd)
-        cmd = f"umount {tmpdir.name}"
-        self.exec(cmd)
         mk.run(self.config.rootfs_img, "qemu.img")
         cmd = 'cp /usr/share/litebmc/qemu.conf qemu.conf'
         self.exec(cmd)

{lbkit-0.9.8 → lbkit-0.9.10}/lbkit/tasks/task.py

@@ -26,10 +26,16 @@ class Task(Process):
         self.log.info("install...........")
 
     def exec(self, cmd: str, verbose=False, ignore_error = False, sensitive=False, log_prefix="", **kwargs):
+        fakeroot = getattr(self.config, 'fakeroot', None)
+        if fakeroot and fakeroot.active:
+            kwargs["extra_env"] = fakeroot.env
         kwargs["uptrace"] = kwargs.get("uptrace", 0) + 1
         return self.tools.exec(cmd, verbose, ignore_error, sensitive, log_prefix, **kwargs)
 
     def pipe(self, cmds: list[str], ignore_error=False, out_file = None, **kwargs):
+        fakeroot = getattr(self.config, 'fakeroot', None)
+        if fakeroot and fakeroot.active:
+            kwargs["extra_env"] = fakeroot.env
         kwargs["uptrace"] = kwargs.get("uptrace", 0) + 1
         return self.tools.pipe(cmds, ignore_error, out_file, **kwargs)
 

{lbkit-0.9.8 → lbkit-0.9.10}/lbkit/tasks/task_build_manifest.py

@@ -43,20 +43,15 @@ class TaskClass(Task):
 
     def deploy(self, graph_file):
         with open(graph_file, "r") as fp:
-            order_info = json.load(fp)
-        for orders in order_info.get("order", []):
-            for order in orders:
-                ref = order.get("ref")
-                packages = order.get("packages", [])
-                for package in packages:
-                    for pkg in package:
-                        binary = pkg.get("context", "")
-                        if binary != "host":
-                            continue
-                        id = pkg.get("package_id", "")
-                        cmd = f"conan cache path {ref}:{id}"
-                        package_folder = self.tools.run(cmd).stdout.strip()
-                        self.config.conan_install.append(package_folder)
+            graph = json.load(fp)
+        nodes = graph.get("graph", {}).get("nodes", {})
+        for id, info in nodes.items():
+            context = info.get("context")
+            if context != "host":
+                continue
+            package_folder = info.get("package_folder", "")
+            if package_folder:
+                self.config.conan_install.append(package_folder)
 
     def build_rootfs(self):
         """构建产品rootfs包"""
@@ -113,8 +108,11 @@ class TaskClass(Task):
         bcp.build()
 
         self.exec(f"sed -i 's@rootfs_df190c/0.0.1#.*\"@rootfs_df190c/0.0.1\"@g' {lockfile}")
+        # 用conan install获取所有依赖的package_folder信息，避免逐个调用conan cache path
+        graphfile = os.path.join(self.config.temp_path, "graph.info")
+        self.pipe([f"conan install . {base_cmd} --lockfile={lockfile} -f json"], out_file=graphfile)
         # 部署应用到self.config.conan_install
-        self.deploy(orderfile)
+        self.deploy(graphfile)
 
     def run(self):
         """任务入口"""