lbkit 0.5.9__tar.gz → 0.5.10__tar.gz

{lbkit-0.5.9/lbkit.egg-info → lbkit-0.5.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lbkit
-Version: 0.5.9
+Version: 0.5.10
 Summary: Tools provided by litebmc.com
 Home-page: https://www.litebmc.com
 Author: xuhj@litebmc.com
@@ -20,5 +20,6 @@ Requires-Dist: conan==2.1
 Requires-Dist: requests
 Requires-Dist: gitpython
 Requires-Dist: inflection
+Requires-Dist: meson==1.4.0
 
 build and code generate tools

lbkit-0.5.10/lbkit/__init__.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.5.10'

{lbkit-0.5.9 → lbkit-0.5.10}/lbkit/cli.py

@@ -13,6 +13,7 @@ from lbkit.component.test import TestComponent
 from lbkit.integration.build_manifest import BuildManifest
 from lbkit.integration.build_rootfs import BuildRootfs
 from lbkit.integration.build_prepare import BuildPrepare
+from lbkit.integration.build_image import BuildImage
 from lbkit.component.arg_parser import ArgParser
 from lbkit.integration.config import Config
 from lbkit.ci_robot.gitee import Gitee
@@ -93,6 +94,8 @@ class Command(object):
         build.run()
         build = BuildRootfs(cfg, "create_rootfs")
         build.run()
+        build = BuildImage(cfg, "create_image")
+        build.run()
 
     def gitee(self, *args):
         """

{lbkit-0.5.9 → lbkit-0.5.10}/lbkit/codegen/idf_interface.py

@@ -5,7 +5,7 @@ import hashlib
 from lbkit.log import Logger
 from lbkit.codegen.renderer import Renderer
 from lbkit.codegen.ctype_defination import CTYPE_OBJS, RefObjArrayValidator, RefObjValidator
-from lbkit.misc import validate_yml_with_json_schema
+from lbkit.misc import load_yml_with_json_schema_validate
 
 log = Logger("gen_interface")
 
@@ -786,10 +786,8 @@ class IdfInterface(IdfInterfaceBase):
     def load_elements(self):
         # 使用schema校验数据，确保IDF文件符合格式要求，减少程序处理过程中的异常处理
         # 验证失败时抛异常，此处不用处理，由外层处理
-        validate_yml_with_json_schema(self.file, "/usr/share/litebmc/schema/idf.v1.json")
+        idf = load_yml_with_json_schema_validate(self.file, "/usr/share/litebmc/schema/idf.v1.json")
         log.info(f"validate {self.file} successfully")
-        with open(self.file, "r") as fp:
-            idf = yaml.load(fp, yaml.FullLoader)
         self.version = idf.get("version")
         self.description = idf.get("description", "")
         # 别名

{lbkit-0.5.9 → lbkit-0.5.10}/lbkit/component/build.py

@@ -10,7 +10,7 @@ from jsonschema import validate, ValidationError
 from git import Repo
 from git.exc import InvalidGitRepositoryError
 from mako.lookup import TemplateLookup
-from lbkit.misc import Color, validate_yml_with_json_schema, get_json_schema_file, load_json_schema
+from lbkit.misc import Color, load_yml_with_json_schema_validate, get_json_schema_file, load_json_schema
 from lbkit import errors
 from lbkit.codegen.codegen import CodeGen
 from lbkit.tools import Tools
@@ -32,7 +32,7 @@ class DeployComponent():
             return
         cmd = f"conan cache path {self.package_ref}:{self.package_id}"
         package_folder = tools.run(cmd).stdout.strip()
-        log.info(f">>copy {package_folder} to {self.rootfs_dir}")
+        log.info(f">>>> deploy {self.package_ref}")
         cmd = f"cp -rT {package_folder}/ {self.rootfs_dir}"
         tools.exec(cmd)
 
@@ -98,13 +98,9 @@ class BuildComponent():
         if not os.path.isfile(package_yml):
             raise FileNotFoundError("metadata/package.yml文件不存在")
         # 验证失败时抛异常，此处不用处理，由外层处理
-        validate_yml_with_json_schema(package_yml, "/usr/share/litebmc/schema/cdf.v1.json")
+        pkg = load_yml_with_json_schema_validate(package_yml, "/usr/share/litebmc/schema/cdf.v1.json")
         log.info(f"validate {package_yml} successfully")
 
-        # 加载package.yml文件
-        fp = open(package_yml, "r")
-        pkg = yaml.safe_load(fp)
-        fp.close()
         self.user = pkg.get("user")
         if self.user is None:
             raise errors.PackageConfigException("metadata/package.yml未正确配置user字段")
@@ -193,13 +189,16 @@ class BuildComponent():
         os.chdir(dir)
 
         cmd = self.base_cmd.replace(self.package, "")
-        cmd = f"conan graph info . {cmd} --build=missing -f json --filter=ref --filter=package_id"
+        cmd = f"conan graph info . {cmd} --build=missing -f json --filter=ref --filter=package_id --filter=context"
         res = tools.run(cmd).stdout.strip()
         graph = json.loads(res)
         nodes = graph.get("graph", {}).get("nodes", {})
         for id, info in nodes.items():
             ref = info.get("ref")
             id = info.get("package_id")
+            context = info.get("context")
+            if context != "host":
+                continue
             dep = DeployComponent(ref, id, self.rootfs_dir)
             dep.run()
 
@@ -264,7 +263,7 @@ class BuildComponent():
                     continue
                 log.info(f"start validate {file} with schema {schema_file}")
                 # 验证全局odf验证
-                validate_yml_with_json_schema(file, schema_file)
+                load_yml_with_json_schema_validate(file, schema_file)
                 odf_ok = self._validate_odf_file(file)
                 if not odf_ok:
                     ok = False

{lbkit-0.5.9 → lbkit-0.5.10}/lbkit/errors.py

@@ -79,3 +79,12 @@ class OdfValidateException(Exception):
 
     def __str__(self):
         return super(OdfValidateException, self).__str__()
+
+class DigestNotMatchError(OSError):
+    """Raised when source and destination are the same file."""
+
+class ExtractRootfsTarFileError(OSError):
+    """Raised when extract rootfs.tar.gz failed."""
+
+class PermissionFormatError(OSError):
+    """permission.ini with format."""

lbkit-0.5.10/lbkit/integration/build_image.py

@@ -0,0 +1,85 @@
+"""环境准备"""
+import os
+import tempfile
+import tarfile
+import shutil
+from lbkit.integration.config import Config
+from lbkit.integration.task import Task
+from lbkit.log import Logger
+from lbkit import errors
+
+log = Logger("build_image")
+
+src_cwd = os.path.split(os.path.realpath(__file__))[0]
+
+class BuildImage(Task):
+    def download_uboot(self, img_file):
+        tar_file = os.path.join(self.config.download_path, "uboot.tar.gz")
+
+        url = self.get_manifest_config("components/uboot/url")
+        sha256 = self.get_manifest_config("components/uboot/sha256")
+        self.tools.download(url, tar_file, sha256)
+        tar = tarfile.open(tar_file)
+        members = tar.getmembers()
+        for member in members:
+            if not member.isfile():
+                continue
+            if member.name != "uboot.bin" and member.name != "u-boot.bin":
+                continue
+            io = tar.extractfile(member)
+            fp = open(img_file, "wb+")
+            while True:
+                buf = io.read(65536)
+                if len(buf) == 0:
+                    break
+                fp.write(buf)
+            fp.close()
+            return
+        raise errors.ExtractRootfsTarFileError("Extract failed, the u-boot.bin can't be found in " + url)
+
+    def download_kernel(self, img_file):
+        tar_file = os.path.join(self.config.download_path, "kernel.tar.gz")
+
+        url = self.get_manifest_config("components/kernel/url")
+        sha256 = self.get_manifest_config("components/kernel/sha256")
+        self.tools.download(url, tar_file, sha256)
+        tar = tarfile.open(tar_file)
+        members = tar.getmembers()
+        for member in members:
+            if not member.isfile():
+                continue
+            if member.name != "Image":
+                continue
+            io = tar.extractfile(member)
+            fp = open(img_file, "wb+")
+            while True:
+                buf = io.read(65536)
+                if len(buf) == 0:
+                    break
+                fp.write(buf)
+            fp.close()
+            return
+        raise errors.ExtractRootfsTarFileError("Extract failed, the Image can't be found in " + url)
+
+    def run(self):
+        """任务入口"""
+        """检查manifest文件是否满足schema格式描述"""
+        os.chdir(self.config.output_path)
+        self.download_uboot("u-boot.bin")
+        self.download_kernel("Image")
+        cmd = "qemu-system-aarch64 -M virt -cpu cortex-a57 -M virt,dumpdtb=virt.dtb"
+        self.exec(cmd)
+
+        cmd = 'lbpack_emmc.sh ./Image ./virt.dtb ./rootfs.img ./qemu.img'
+        self.exec(cmd)
+        cmd = 'cp /usr/share/litebmc/qemu.conf ./qemu.conf'
+        self.exec(cmd)
+        output_img = os.path.join(self.config.output_path, "litebmc_qemu.tar.gz")
+        cmd = f'tar -czf {output_img} -C . qemu.img u-boot.bin qemu.conf'
+        self.exec(cmd)
+        log.success(f"Create litebmc image {output_img} successfully")
+
+if __name__ == "__main__":
+    config = Config()
+    build = BuildImage(config)
+    build.run()

{lbkit-0.5.9 → lbkit-0.5.10}/lbkit/integration/build_manifest.py

@@ -1,6 +1,7 @@
 """应用构建任务"""
 import os
 import shutil
+import json
 from mako.lookup import TemplateLookup
 from lbkit.integration.config import Config
 from lbkit.integration.task import Task
@@ -15,14 +16,13 @@ class ManifestValidateError(OSError):
 src_cwd = os.path.split(os.path.realpath(__file__))[0]
 
 class BuildManifest(Task):
-    """根据产品配置构建所有app并将应用安装到self.config.conan_install目录"""
-    def __init__(self, cfg: Config):
-        super().__init__(cfg)
+    """根据产品配置构建所有app,记录待安装应用路径到self.config.conan_install路径"""
+    def __init__(self, cfg: Config, name: str):
+        super().__init__(cfg, name)
         self.conan_build = os.path.join(self.config.temp_path, "conan")
-        shutil.rmtree(self.conan_build, ignore_errors=True)
+        if os.path.isdir(self.conan_build):
+            shutil.rmtree(self.conan_build)
         os.makedirs(self.conan_build)
-        shutil.rmtree(self.config.conan_install, ignore_errors=True)
-        os.makedirs(self.config.conan_install)
         if self.config.build_debug:
             self.conan_settings = " -s build_type=Debug"
         else:
@@ -30,11 +30,29 @@ class BuildManifest(Task):
         self.common_args = "-r " + self.config.remote
         self.common_args += " -pr:b {} -pr:h {}".format(self.config.profile_build, self.config.profile_host)
 
+    def deploy(self):
+        cmd = f"conan graph info . {self.common_args} --build=missing -f json --filter=ref --filter=package_id"
+        cmd += f" --filter=context"
+        res = self.tools.run(cmd).stdout.strip()
+        graph = json.loads(res)
+        nodes = graph.get("graph", {}).get("nodes", {})
+        for id, info in nodes.items():
+            ref = info.get("ref")
+            id = info.get("package_id")
+            context = info.get("context")
+            if context != "host":
+                continue
+            cmd = f"conan cache path {ref}:{id}"
+            package_folder = self.tools.run(cmd).stdout.strip()
+            self.config.conan_install.append(package_folder)
+
     def build_litebmc(self):
         """构建产品conan包"""
         log.info("build litebmc")
 
         manifest = self.load_manifest()
+        hook_name = "hook.prepare_manifest"
+        self.do_hook(hook_name)
         # 使用模板生成litebmc组件的配置
         lookup = TemplateLookup(directories=os.path.join(src_cwd, "template"))
         template = lookup.get_template("conanfile.py.mako")
@@ -49,7 +67,7 @@ class BuildManifest(Task):
 
         # 安装所有应用
         if self.config.from_source:
-            cmd = "conan install . --build " + self.common_args
+            cmd = "conan install . --build=\"*\" " + self.common_args
         else:
             cmd = "conan install . --build=missing " + self.common_args
         cmd = cmd + self.conan_settings
@@ -59,10 +77,7 @@ class BuildManifest(Task):
         cmd = cmd + self.conan_settings
         self.exec(cmd, verbose=True)
         # 部署应用到self.config.conan_install
-        log.info("install litebmc to %s", self.config.conan_install)
-        cmd = "conan install . {} -d direct_deploy --deployer-folder={}".format(self.common_args, self.config.conan_install)
-        cmd = cmd + self.conan_settings
-        self.exec(cmd, verbose=self.config.verbose)
+        self.deploy()
 
     def run(self):
         """任务入口"""
@@ -71,4 +86,4 @@ class BuildManifest(Task):
 if __name__ == "__main__":
     config = Config()
     build = BuildManifest(config)
-    build.run()
+    build.run()

lbkit-0.5.10/lbkit/integration/build_prepare.py

@@ -0,0 +1,56 @@
+"""环境准备"""
+import os
+import shutil
+import jinja2
+import configparser
+from lbkit.integration.config import Config
+from lbkit.integration.task import Task
+from lbkit.log import Logger
+
+log = Logger("product_prepare")
+
+
+class ManifestValidateError(OSError):
+    """Raised when validation manifest.yml failed."""
+
+src_cwd = os.path.split(os.path.realpath(__file__))[0]
+
+class BuildPrepare(Task):
+    def load_conan_profile(self):
+        profile = self.get_manifest_config("metadata/profile")
+        if not os.path.isfile(profile):
+            raise FileNotFoundError(f"profile {profile} not found")
+        log.info("Copy profile %s", profile)
+        profiles_dir = os.path.expanduser("~/.conan2/profiles")
+        dst_profile = os.path.join(profiles_dir, os.path.basename(profile))
+        if os.path.isdir(profiles_dir):
+            shutil.copyfile(profile, dst_profile, follow_symlinks=False)
+
+        with open(dst_profile, "r") as fp:
+            profile_data = jinja2.Template(fp.read()).render()
+            parser = configparser.ConfigParser()
+            parser.read_string(profile_data)
+            strip = "strip"
+            if parser.has_option("buildenv", "STRIP"):
+                strip = parser.get("buildenv", "STRIP")
+            path = ""
+            if parser.has_option("buildenv", "PATH+"):
+                path = parser.get("buildenv", "PATH+")
+                if path.startswith("(path)"):
+                    path = path[6:]
+            elif parser.has_option("buildenv", "PATH"):
+                path = parser.get("buildenv", "PATH")
+                if path.startswith("(path)"):
+                    path = path[6:]
+            self.config.strip = os.path.join(path, strip)
+
+    def run(self):
+        """任务入口"""
+        """检查manifest文件是否满足schema格式描述"""
+        self.load_manifest()
+        self.load_conan_profile()
+
+if __name__ == "__main__":
+    config = Config()
+    build = BuildPrepare(config)
+    build.run()

{lbkit-0.5.9 → lbkit-0.5.10}/lbkit/integration/build_rootfs.py

@@ -3,20 +3,15 @@ import os
 import shutil
 import tarfile
 from lbkit.integration.config import Config
-from lbkit.integration.task import Task, DigestNotMatchError
+from lbkit.integration.task import Task
 from lbkit.log import Logger
+from lbkit import errors
 
 log = Logger("build_rootfs")
 
 src_cwd = os.path.split(os.path.realpath(__file__))[0]
 IMG_FILE = "rootfs.img"
 
-class ExtractRootfsTarFileError(OSError):
-    """Raised when extract rootfs.tar.gz failed."""
-
-class PermissionFormatError(OSError):
-    """permission.ini with format."""
-
 class BuildRootfs(Task):
     """构建rootfs镜像"""
     def do_permission(self, per_file: str):
@@ -32,14 +27,14 @@ class BuildRootfs(Task):
                 log.debug("Permission line: %s", line)
                 chunk = line.split()
                 if len(chunk) < 5:
-                    raise PermissionFormatError(f"Permission format with error, line: {line}")
+                    raise errors.PermissionFormatError(f"Permission format with error, line: {line}")
                 if not chunk[0].startswith("/"):
-                    raise PermissionFormatError(f"Permission file error, must begin with \"/\", get: {chunk[0]}")
+                    raise errors.PermissionFormatError(f"Permission file error, must begin with \"/\", get: {chunk[0]}")
                 if not chunk[3].isnumeric() or not chunk[4].isnumeric():
-                    raise PermissionFormatError(f"Permission uid or gid error, must is numeric, uid({chunk[3]}), gid({chunk[4]})")
+                    raise errors.PermissionFormatError(f"Permission uid or gid error, must is numeric, uid({chunk[3]}), gid({chunk[4]})")
                 chunk[0] = chunk[0].lstrip("/")
                 if chunk[1] != "f" and chunk[1] != "d" and chunk[1] != "s" and chunk[1] != "r":
-                    raise PermissionFormatError(f"Permission type error, only support 'f' 's' 'd' 'r', get({chunk[1]}), ignore")
+                    raise errors.PermissionFormatError(f"Permission type error, only support 'f' 's' 'd' 'r', get({chunk[1]}), ignore")
                 if chunk[1] == "d" and not os.path.isdir(chunk[0]):
                     log.error("Permission error, %s is not directory", chunk[0])
                 if chunk[1] == "s" and not os.path.islink(chunk[0]):
@@ -70,21 +65,7 @@ class BuildRootfs(Task):
 
         url = self.get_manifest_config("components/rootfs/url")
         sha256 = self.get_manifest_config("components/rootfs/sha256")
-        digest = None
-        if os.path.isfile(rootfs_tar):
-            digest = self.file_digest_sha256(rootfs_tar)
-        if digest != sha256:
-            art_user = os.environ.get("ART_USER", None)
-            art_token = os.environ.get("ART_TOKEN", None)
-            if art_user is None or art_token is None:
-                log.info("The environment variables ART_USER and ART_TOKEN are not set, the file is downloaded directily")
-                cmd = f"curl -fL \"{url}\" -o {rootfs_tar}"
-            else:
-                cmd = f"curl -fL -u {art_user}:{art_token} \"{url}\" -o {rootfs_tar}"
-            self.exec(cmd, verbose=True, sensitive=True)
-            digest = self.file_digest_sha256(rootfs_tar)
-        if digest != sha256:
-            raise DigestNotMatchError(f"File {rootfs_tar} with sha256 error, need: {sha256}, get: {digest}")
+        self.tools.download(url, rootfs_tar, sha256)
         tar = tarfile.open(rootfs_tar)
         members = tar.getmembers()
         for member in members:
@@ -97,11 +78,11 @@ class BuildRootfs(Task):
             while True:
                 buf = io.read(65536)
                 if len(buf) == 0:
-                    break;
+                    break
                 fp.write(buf)
             fp.close()
             return
-        raise ExtractRootfsTarFileError("Extract failed, the rootfs.img can't be found in rootfs.tar")
+        raise errors.ExtractRootfsTarFileError("Extract failed, the rootfs.img can't be found in rootfs.tar")
 
     def merge_rootfs(self, rootfs_img):
         """将产品依赖的所有组件安装到rootfs镜像中"""
@@ -117,13 +98,9 @@ class BuildRootfs(Task):
         # 切换到rootfs挂载目录
         os.chdir(mnt_path)
         log.info("Copy customization rootfs......")
-        deploy_dir = os.path.join(self.config.conan_install, "direct_deploy")
-        for component in os.listdir(deploy_dir):
-            src_dir = os.path.join(deploy_dir, component)
-            if not os.path.isdir(src_dir):
-                continue
+        for src_dir in self.config.conan_install:
             log.info("copy %s to %s", src_dir, mnt_path)
-            cmd = f"rsync -auHK {src_dir}/ {mnt_path}"
+            cmd = f"rsync -aHK {src_dir}/ {mnt_path}"
             self.exec(cmd)
             per_file = os.path.join(src_dir, "permissions")
             self.do_permission(per_file)
@@ -131,7 +108,7 @@ class BuildRootfs(Task):
         # copy product self-owned rootfs
         product_rootfs = os.path.join(self.config.work_dir, "rootfs")
         if os.path.isdir(product_rootfs):
-            cmd = f"rsync -auKH {product_rootfs}/ {mnt_path}"
+            cmd = f"rsync -aKH {product_rootfs}/ {mnt_path}"
             self.exec(cmd)
             per_file = os.path.join(product_rootfs, "permissions")
             self.do_permission(per_file)
@@ -142,18 +119,21 @@ class BuildRootfs(Task):
         self.do_hook(hook_name)
 
         # 清理冗余文件
-        cmd = "rm -rf " + os.path.join(self.config.mnt_path, "include")
-        self.exec(cmd)
+        inc_dir = os.path.join(self.config.mnt_path, "include")
+        if os.path.isdir(inc_dir):
+            cmd = "rm -rf " + inc_dir
+            self.exec(cmd)
         cmd = "rm " + os.path.join(self.config.mnt_path, "permissions")
         self.exec(cmd)
 
         # 清理冗余静态文件
-        cmd = "find " + os.path.join(self.config.mnt_path, "include") + " -name *.a -type f | xargs rm -f"
-        self.exec(cmd)
+        cmds = ["find " + self.config.mnt_path + " -name *.a -type f", "xargs -i{} rm {} -f"]
+        self.pipe(cmds)
 
         # 按manifest配置的大小调整rootfs
         strip = self.get_manifest_config("metadata/strip")
         if strip:
+            log.info("Start strip files")
             file_list = os.path.join(self.config.temp_path, "strip.filelist")
             unstrip_regex = [
                 "*.yaml$",
@@ -172,15 +152,19 @@ class BuildRootfs(Task):
                 "*.conf$"]
             cmds = ["find " + self.config.mnt_path + " -type f"]
             for unstrip in unstrip_regex:
-                cmds.append[f"grep -vE \"{unstrip}\""]
+                cmds.append(f"grep -vE {unstrip}")
             self.pipe(cmds, out_file=file_list)
 
             cmds = [f"cat {file_list}", "xargs file", "grep 'not stripped'", "awk -F':' '{{print $1}}'"]
-            cmds.append[f"xargs {self.config.strip} -s"]
-            self.pipe(cmd)
-
-        cmds = [f"find {self.config.mnt_path} -name *.a", "xargs rm"]
-        self.pipe(cmd)
+            cmds.append(f"xargs -i{{}} {self.config.strip} -s {{}}")
+            self.pipe(cmds)
+
+        log.info("remove static libraries")
+        cmds = [f"find {self.config.mnt_path} -name *.a", "xargs -i{} rm {}"]
+        self.pipe(cmds)
+        log.info("remove all .fuse_hiddeng* files")
+        cmds = [f"find {self.config.mnt_path} -name .fuse_hidden*", "xargs -i{} rm {}"]
+        self.pipe(cmds)
         self.exec("umount " + mnt_path)
 
     def run(self):
@@ -190,8 +174,8 @@ class BuildRootfs(Task):
         # 任务入口
         self.download_rootfs(rootfs_img)
         self.merge_rootfs(rootfs_img)
-        tgt_img = os.path.join(self.config.output_path, IMG_FILE)
-        os.rename(rootfs_img, tgt_img)
+        os.rename(rootfs_img, self.config.rootfs_img)
+        log.success(f"Create image {self.config.rootfs_img} successfully")
 
 if __name__ == "__main__":
     config = Config()

{lbkit-0.5.9 → lbkit-0.5.10}/lbkit/integration/config.py

@@ -41,14 +41,14 @@ class Config(object):
         self.tool_path = os.path.join(self.temp_path, "tools")
         self.download_path = os.path.join(self.temp_path, "download")
         self.rootfs_path = os.path.join(self.temp_path, "rootfs")
-        self.conan_install = os.path.join(self.temp_path, "apps")
+        self.conan_install = []
         self.mnt_path = os.path.join(self.temp_path, "mnt_path")
+        self.rootfs_img = os.path.join(self.output_path, "rootfs.img")
         os.makedirs(self.temp_path, exist_ok=True)
         os.makedirs(self.tool_path, exist_ok=True)
         os.makedirs(self.output_path, exist_ok=True)
         os.makedirs(self.download_path, exist_ok=True)
         os.makedirs(self.rootfs_path, exist_ok=True)
-        os.makedirs(self.conan_install, exist_ok=True)
         # 制作rootfs时需要strip镜像，所以需要单独指定stip路径
         self.strip = "strip"
 

lbkit-0.5.10/lbkit/integration/task.py

@@ -0,0 +1,63 @@
+"""任务基础类"""
+import shutil
+import importlib
+import os
+import hashlib
+import requests
+import yaml
+from multiprocessing import Process
+from lbkit.log import Logger
+from lbkit.tools import Tools
+from lbkit.misc import load_yml_with_json_schema_validate
+
+from lbkit.integration.config import Config
+
+class ManifestValidateError(OSError):
+    """Raised when validation manifest.yml failed."""
+
+class Task(Process):
+    """任务基础类，提供run和install默认实现以及其它基础该当"""
+    def __init__(self, config: Config, name: str):
+        super().__init__()
+        self.log: Logger = Logger("task")
+        self.tools: Tools = Tools(name)
+        self.config: Config = config
+
+    def install(self):
+        """安装任务"""
+        self.log.info("install...........")
+
+    def exec(self, cmd: str, verbose=False, ignore_error = False, sensitive=False, log_prefix=""):
+        return self.tools.exec(cmd, verbose, ignore_error, sensitive, log_prefix)
+
+    def pipe(self, cmds: list[str], ignore_error=False, out_file = None):
+        self.tools.pipe(cmds, ignore_error, out_file)
+
+    def run(self, cmd, ignore_error=False):
+        return self.tools.run(cmd, ignore_error)
+
+    def do_hook(self, path):
+        """执行任务钓子，用于定制化"""
+        try:
+            module = importlib.import_module(path)
+        except TypeError:
+            self.log.info("Load module(%s) failed, skip", path)
+            return
+        self.log.info(module)
+        hook = module.TaskHook(self.config, "do_hook")
+        hook.run()
+
+    def get_manifest_config(self, key: str):
+        """从manifest中读取配置"""
+        with open(self.config.manifest, "r", encoding="utf-8") as fp:
+            manifest = yaml.load(fp, yaml.FullLoader)
+        keys = key.split("/")
+        for k in keys:
+            manifest = manifest.get(k, None)
+            if manifest is None:
+                return None
+        return manifest
+
+    def load_manifest(self):
+        """加载manifest.yml并验证schema文件"""
+        return load_yml_with_json_schema_validate(self.config.manifest, "/usr/share/litebmc/schema/pdf.v1.json")

{lbkit-0.5.9 → lbkit-0.5.10}/lbkit/misc.py

@@ -73,7 +73,7 @@ def load_json_schema(schema_file):
             tmp = fp.read()
             return json.loads(tmp)
 
-def validate_yml_with_json_schema(yml_file, default_json_schema_file):
+def load_yml_with_json_schema_validate(yml_file, default_json_schema_file):
     """使用json schema文件校验yml_file配置文件"""
     schema_file = get_json_schema_file(yml_file, default_json_schema_file)
     if schema_file is None:
@@ -85,6 +85,7 @@ def validate_yml_with_json_schema(yml_file, default_json_schema_file):
         data = yaml.safe_load(fp)
         fp.close()
         validate(data, schema)
+        return data
     except ValidationError as exc:
         raise PackageConfigException(f"validate {yml_file} failed, schema file is {schema_file}, "
                                      f"message: {exc.message}\n"

{lbkit-0.5.9 → lbkit-0.5.10}/lbkit/tools.py

@@ -5,7 +5,9 @@ import inspect
 import os
 import sys
 import tempfile
+import requests
 import shlex
+import hashlib
 from lbkit.log import Logger
 from lbkit import errors
 from lbkit import misc
@@ -51,8 +53,8 @@ class Tools(object):
             show_cmd = "(" + log_prefix + ") " + show_cmd
 
         fd = os.fdopen(os.open(self.log_name, os.O_APPEND | os.O_CREAT | os.O_RDWR), "a+")
-        self.log.debug("{}>>>>{} {}".format(Color.GREEN, Color.RESET_ALL, show_cmd))
-        fd.write(">>>> {}".format(show_cmd))
+        self.log.info("{}>>>>{} {}".format(Color.GREEN, Color.RESET_ALL, show_cmd))
+        fd.write(">>>> {}\n".format(show_cmd))
         if os.environ.get("VERBOSE", False):
             verbose = True
         real_cmd = self._real_command(cmd)
@@ -66,16 +68,19 @@ class Tools(object):
         fd.close()
         result.communicate()
         if result is None or result.returncode != 0:
-            self.log.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
-            msg = f"{file}:{line} > Run command failed, cmd: {show_cmd}, error code: {result.returncode}, log: {self.log_name}"
+            msg = f"{file}:{line} > Run command ({show_cmd}) failed but ignore, log file is {self.log_name}"
             if not ignore_error:
+                self.log.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
                 raise errors.RunCommandException(msg)
-            self.log.error(msg)
+            self.log.warn(msg)
         return result
 
     def pipe(self, cmds: list[str], ignore_error=False, out_file = None):
         if not isinstance(cmds, list):
             raise errors.ArgException("Command ({}) with type error, only list[str] can be accepted".format(cmds))
+        if out_file and os.path.isfile(out_file):
+            fp = open(out_file, "w")
+            fp.close()
         stdin = None
         for cmd in cmds:
             self.log.debug("{}>>>>{} {}".format(Color.GREEN, Color.RESET_ALL, cmd))
@@ -85,7 +90,7 @@ class Tools(object):
             if ret is None:
                 raise errors.RunCommandException(f"Run command {real_cmd[0]} failed")
             ret.communicate()
-            if ret is None or ret.returncode != 0:
+            if ret.returncode != 0:
                 if ignore_error:
                     self.log.info("Run command ({}) failed".format(cmd))
                     return
@@ -93,6 +98,8 @@ class Tools(object):
             if stdin:
                 stdin.close()
             stdin = stdout
+            stdin.seek(0)
+
         if out_file:
             stdin.seek(0)
             with open(out_file, "w+b") as fp:
@@ -105,3 +112,44 @@ class Tools(object):
         check = False if ignore_error else True
         return subprocess.run(real_cmd, capture_output=True, check=check, encoding="utf-8")
 
+    @staticmethod
+    def file_digest_sha256(filename):
+        """计算文件的sha256值"""
+        sha256 = hashlib.sha256()
+        fp = open(filename, "rb")
+        while True:
+            data = fp.read(65536)
+            if len(data) == 0:
+                break
+            sha256.update(data)
+        fp.close()
+        return sha256.hexdigest()
+
+    def download(self, url, dst_file, sha256sum = None):
+        """下载文件"""
+        self.log.info("Start download %s", url)
+        is_local = False
+        if url.startswith("file://"):
+            path = url[7:]
+            shutil.copyfile(path, dst_file)
+            is_local = True
+        if os.path.isfile(dst_file):
+            digest = self.file_digest_sha256(dst_file)
+            if sha256sum is None or digest == sha256sum:
+                return
+            if is_local:
+                raise errors.DigestNotMatchError(f"File {dst_file} with sha256 error, need: {sha256sum}, get: {digest}")
+            os.unlink(dst_file)
+        verify = os.environ.get("HTTPS_VERIFY", True)
+        if verify:
+            response = requests.get(url, timeout=30, verify=True)
+        else:
+            response = requests.get(url, timeout=30)
+        fp = open(dst_file, "wb")
+        fp.write(response.content)
+        fp.close()
+        digest = self.file_digest_sha256(dst_file)
+        if sha256sum is None or digest == sha256sum:
+            self.log.info("Download %s successfully", url)
+            return
+        raise errors.DigestNotMatchError(f"File {dst_file} with sha256 error, need: {sha256sum}, get: {digest}")

{lbkit-0.5.9 → lbkit-0.5.10/lbkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lbkit
-Version: 0.5.9
+Version: 0.5.10
 Summary: Tools provided by litebmc.com
 Home-page: https://www.litebmc.com
 Author: xuhj@litebmc.com
@@ -20,5 +20,6 @@ Requires-Dist: conan==2.1
 Requires-Dist: requests
 Requires-Dist: gitpython
 Requires-Dist: inflection
+Requires-Dist: meson==1.4.0
 
 build and code generate tools

{lbkit-0.5.9 → lbkit-0.5.10}/lbkit.egg-info/SOURCES.txt

@@ -39,6 +39,7 @@ lbkit/component/test.py
 lbkit/component/template/conanbase.mako
 lbkit/component/template/deploy.mako
 lbkit/integration/__init__.py
+lbkit/integration/build_image.py
 lbkit/integration/build_manifest.py
 lbkit/integration/build_prepare.py
 lbkit/integration/build_rootfs.py

{lbkit-0.5.9 → lbkit-0.5.10}/lbkit.egg-info/requires.txt

@@ -7,3 +7,4 @@ conan==2.1
 requests
 gitpython
 inflection
+meson==1.4.0

{lbkit-0.5.9 → lbkit-0.5.10}/setup.py

@@ -19,7 +19,7 @@ setup(
     description="Tools provided by litebmc.com",
     long_description="build and code generate tools",
     long_description_content_type="text/markdown",
-    install_requires=["pyyaml", "colorama", "mako", "node-semver==0.6.1", "jsonschema", "conan==2.1", "requests", "gitpython", "inflection"],
+    install_requires=["pyyaml", "colorama", "mako", "node-semver==0.6.1", "jsonschema", "conan==2.1", "requests", "gitpython", "inflection", "meson==1.4.0"],
     url="https://www.litebmc.com",
     packages=find_packages(),
     include_package_data=True,

lbkit-0.5.9/lbkit/__init__.py

@@ -1,2 +0,0 @@
-
-__version__ = '0.5.9'

lbkit-0.5.9/lbkit/integration/build_prepare.py

@@ -1,89 +0,0 @@
-"""环境准备"""
-import os
-import shutil
-import tarfile
-from lbkit.integration.config import Config
-from lbkit.integration.task import Task
-from lbkit.log import Logger
-
-log = Logger("product_prepare")
-
-
-class ManifestValidateError(OSError):
-    """Raised when validation manifest.yml failed."""
-
-src_cwd = os.path.split(os.path.realpath(__file__))[0]
-
-class BuildPrepare(Task):
-    def check_manifest(self):
-        """检查manifest文件是否满足schema格式描述"""
-        self.load_manifest()
-
-    def install_tools(self):
-        url = self.get_manifest_config("tools/compiler/url")
-        sha256 = self.get_manifest_config("tools/compiler/sha256")
-        path = self.get_manifest_config("tools/compiler/path")
-        compiler_path = path if path else os.path.join(self.config.tool_path, "linaro")
-
-        need_untar = True
-
-        # 下载工具
-        tool_file = os.path.join(self.config.download_path, "gcc.tar.bz2")
-        # 先检查安装路径下是否存在sha256.lock文件，如果存在且值相同的不处理
-        sha256_flag = os.path.join(compiler_path, "sha256.lock")
-        if os.path.isfile(sha256_flag):
-            fp = open(sha256_flag, "r", encoding="utf-8")
-            lock_sha256 = fp.readline()
-            fp.close
-            if lock_sha256 == sha256:
-                log.info("%s exist, skip untar", sha256_flag)
-                need_untar = False
-        # 如果需要解压值url或sha256为空时异常
-        if need_untar and url is None or sha256 is None:
-            raise ManifestValidateError(self.config.manifest + " with error, tools.compiler.url or tools.compiler.sha256 is None")
-
-        # 解压到tools/linaro目录
-        self.download_url(url, tool_file, sha256)
-        tar = tarfile.open(tool_file)
-        top_dir = tar.next().name
-        if (need_untar):
-            log.info("Start untar %s", tool_file)
-            shutil.rmtree(compiler_path, ignore_errors=True)
-            os.makedirs(compiler_path, exist_ok=True)
-            tar.extractall(compiler_path)
-            # 加了工具的目录名生成真实的工具目录地址
-        compiler_path = os.path.join(compiler_path, top_dir)
-        self.modify_profile(compiler_path)
-        with open(sha256_flag, "w+", encoding="utf-8") as fp:
-            fp.write(sha256)
-        log.info("Set strip to %s", self.config.strip)
-
-    def modify_profile(self, compiler_path):
-        host = self.get_manifest_config("tools/compiler/host")
-        # 制作rootfs时需要strip镜像，所以需要单独指定stip路径
-        self.config.strip = os.path.join(compiler_path, "bin", host + "-strip")
-        profile = self.get_manifest_config("tools/conan/profile")
-        src_profile = os.path.join(self.config.work_dir, profile)
-        log.info("Copy profile %s", src_profile)
-        if not os.path.isfile(src_profile):
-            raise FileNotFoundError(f"profile {src_profile} not found")
-        profiles_dir = os.path.expanduser("~/.conan2/profiles")
-        dst_profile = os.path.join(profiles_dir, os.path.basename(profile))
-        if os.path.isdir(profiles_dir):
-            shutil.copyfile(src_profile, dst_profile, follow_symlinks=False)
-        cmd = f"sed -i 's@^toolchain=.*@toolchain={compiler_path}@g' {dst_profile}"
-        self.exec(cmd)
-        cmd = f"sed -i 's@^target_host=.*@target_host={host}@g' {dst_profile}"
-        self.exec(cmd)
-        # 重写profile配置
-        log.info("Overwrite profile to %s", dst_profile)
-        self.config.profile_host = os.path.basename(dst_profile)
-
-    def run(self):
-        """任务入口"""
-        self.check_manifest()
-        self.install_tools()
-if __name__ == "__main__":
-    config = Config()
-    build = BuildPrepare(config)
-    build.run()

lbkit-0.5.9/lbkit/integration/task.py

@@ -1,126 +0,0 @@
-"""任务基础类"""
-import shutil
-import importlib
-import os
-import hashlib
-import requests
-import jsonschema
-import json, yaml
-from multiprocessing import Process
-from lbkit.log import Logger
-from lbkit.tools import Tools
-
-log = Logger("product_test")
-
-from lbkit.integration.config import Config
-
-class ManifestValidateError(OSError):
-    """Raised when validation manifest.yml failed."""
-
-class DigestNotMatchError(OSError):
-    """Raised when source and destination are the same file."""
-
-class Task(Process):
-    """任务基础类，提供run和install默认实现以及其它基础该当"""
-    def __init__(self, config: Config, name: str):
-        super().__init__()
-        self.tools = Tools(name)
-        self.config = config
-
-    def install(self):
-        """安装任务"""
-        log.info("install...........")
-
-    def exec(self, cmd: str, verbose=False, ignore_error = False, sensitive=False, log_prefix=""):
-        return self.tools.exec(cmd, verbose, ignore_error, sensitive, log_prefix)
-
-    def pipe(self, cmds: list[str], ignore_error=False, out_file = None):
-        self.tools.pipe(cmds, out_file, ignore_error)
-
-    def run(self, cmd, ignore_error=False):
-        return self.tools.run(cmd, ignore_error)
-
-    def do_hook(self, path):
-        """执行任务钓子，用于定制化"""
-        try:
-            module = importlib.import_module(path)
-        except TypeError:
-            log.info("Load module(%s) failed, skip", path)
-            return
-        log.info(module)
-        hook = module.TaskHook(self.config)
-        hook.run()
-
-    def get_manifest_config(self, key: str):
-        """从manifest中读取配置"""
-        with open(self.config.manifest, "r", encoding="utf-8") as fp:
-            manifest = yaml.load(fp, yaml.FullLoader)
-        keys = key.split("/")
-        for k in keys:
-            manifest = manifest.get(k, None)
-            if manifest is None:
-                return None
-        return manifest
-
-    @staticmethod
-    def file_digest_sha256(filename):
-        """计算文件的sha256值"""
-        sha256 = hashlib.sha256()
-        fp = open(filename, "rb")
-        while True:
-            data = fp.read(65536)
-            if len(data) == 0:
-                break
-            sha256.update(data)
-        fp.close()
-        return sha256.hexdigest()
-
-    def download_url(self, url, dst_file, sha256sum = None):
-        """下载文件"""
-        log.info("Start download %s", url)
-        is_local = False
-        if url.startswith("file://"):
-            path = url[7:]
-            shutil.copyfile(path, dst_file)
-            is_local = True
-        if os.path.isfile(dst_file):
-            digest = self.file_digest_sha256(dst_file)
-            if sha256sum is None or digest == sha256sum:
-                return
-            if is_local:
-                raise DigestNotMatchError(f"File {dst_file} with sha256 error, need: {sha256sum}, get: {digest}")
-            os.unlink(dst_file)
-        verify = os.environ.get("HTTPS_VERIFY", True)
-        if verify:
-            response = requests.get(url, timeout=30, verify=True)
-        else:
-            response = requests.get(url, timeout=30)
-        fp = open(dst_file, "wb")
-        fp.write(response.content)
-        fp.close()
-        digest = self.file_digest_sha256(dst_file)
-        if sha256sum is None or digest == sha256sum:
-            log.info("Download %s successfully", url)
-            return
-        raise DigestNotMatchError(f"File {dst_file} with sha256 error, need: {sha256sum}, get: {digest}")
-
-    def load_manifest(self):
-        """加载manifest.yml并验证schema文件"""
-        with open(self.config.manifest, "r", encoding="utf-8") as fp:
-            manifest = yaml.load(fp, yaml.FullLoader)
-        schema_version = manifest.get("schema", None)
-        if schema_version is None or not isinstance(schema_version, int):
-            raise ManifestValidateError("Validation manifest.yml failed, Property schema not exist.")
-        # 加载schmea文件验证文件合法性
-        schema_url = "https://www.litebmc.com/standards/metadata/manifest.yml/schema_v" + str(schema_version)
-        # 下载schema文件
-        schema_file = os.path.join(self.config.temp_path, "manifest.schema.json")
-        self.download_url(schema_url, schema_file)
-        with open (schema_file, "rb") as fp:
-            schema = json.load(fp)
-        # 验证manifest.yml是否合法
-        try:
-            jsonschema.validate(manifest, schema)
-        except jsonschema.exceptions.ValidationError as e:
-            raise ManifestValidateError("manifest validate failed") from e
-        return manifest