openubmc-bingo 0.5.240__py3-none-any.whl

bmcgo/functional/check.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+# encoding=utf-8
+# 描述：BMC Studio语法正确性与模型一致性检查
+# Copyright (c) 2024 Huawei Technologies Co., Ltd.
+# openUBMC is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#         http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+import argparse
+import json
+import os
+import subprocess
+import tempfile
+from datetime import datetime, timedelta, timezone
+
+from git import Repo
+
+from bmcgo.codegen.c.helper import Helper
+from bmcgo.utils.tools import Tools
+from bmcgo.bmcgo_config import BmcgoConfig
+from bmcgo.utils.fetch_component_code import FetchComponentCode
+from bmcgo import misc
+from bmcgo.functional.fetch import BmcgoCommand as FetchAgent
+
+log = Tools().log
+cwd = os.getcwd()
+command_info: misc.CommandInfo = misc.CommandInfo(
+    group=misc.GRP_STUDIO,
+    name="check",
+    description=["BMC Studio语法正确性与模型一致性检查"],
+    hidden=False
+)
+ISSUE_TEMPLATE = '''问题{0}：
+【规则名称】{1}
+【文件路径】{2}
+【错误提示】{3}
+【修复建议】{4}
+'''
+_PACKAGE_INFO_HELP = """
+1. 通过组件包名及版本拉取单个组件代码，格式：package/version@user/channel
+2. 通过配置文件拉取部分指定版本的组件代码。支持一下2种配置文件：
+    a. yml格式
+        dependencies:
+        - conan: "package/version@user/channel"
+    b. 文本格式
+        package/version@user/channel
+"""
+MODEL_CHOICES = ["all", "mds", "resource_tree", "csr", "interface_mapping"]
+
+
+def if_available(bconfig: BmcgoConfig):
+    return True
+
+
+class BmcgoCommand:
+    def __init__(self, bconfig: BmcgoConfig, *args):
+        self.bconfig = bconfig
+        parser = argparse.ArgumentParser(prog="bmcgo check", description="语法正确性与模型一致性检查", add_help=True,
+            formatter_class=argparse.RawTextHelpFormatter)
+        parser.add_argument("-b", "--board_name", help="指定单板获取配套全量源码，可选值为build/product目录下的单板名\n默认：openUBMC",
+                            default="openUBMC")
+        parser.add_argument("--stage", help="包类型，可选值为：rc(预发布包), stable(发布包)\n默认: stable", default="stable")
+        parser.add_argument("-r", "--remote")
+        parser.add_argument("-m", "--model", help=f"检查的模型，可选值为: {', '.join(MODEL_CHOICES)}\n默认: all",
+                            choices=MODEL_CHOICES, default="all")
+        parsed_args, _ = parser.parse_known_args(*args)
+        self.board_name = parsed_args.board_name
+        self.stage = parsed_args.stage
+        self.remote = parsed_args.remote
+        self.model = parsed_args.model
+        self.community_issues = set()
+        self.service_dict = {}
+        self.packages = {}
+        self.repo_brance = Repo(cwd).active_branch.name
+        self.repo_name = "manifest"
+        self.disabled = False
+        self.studio_dir = "/usr/share/bmc_studio/server"
+        self.studio_path = "/usr/share/bmc_studio/server/bmcstudio"
+        # 初始化fetch命令对象
+        if self.bconfig.manifest:
+            self.fetch_agent = FetchAgent(self.bconfig, *args)
+    
+    @staticmethod
+    def check_overdue(issue: dict):
+        data = issue.get("deadline")
+        if date is None:
+            return False
+        try:
+            utc_8 = timezone(timedelta(hours=8))
+            deadline = datetime.strptime(data, '%Y/%m/%d').replace(tzinfo=utc_8)
+            return datetime.now(tz=utc_8) >= deadline + timedelta(days=1)
+        except Exception as e:
+            log.warning("日期 %s 解析失败：%s", date, e)
+            return False
+    
+    @staticmethod
+    def process_issues_group(issues, prefix, add_message=""):
+        result = ""
+        if issues:
+            issues.sort()
+            for index, (rule, filepath, error_message, repair_suggestion) in enumerate(issues):
+                result += ISSUE_TEMPLATE.format(index + 1, rule, filepath, error_message, repair_suggestion)
+            result = result.strip()
+            if result and os.getenv("CLOUD_BUILD_RECORD_ID") is not None:
+                result = "\n".join(map(lambda s: f'{prefix} {s}', result.split("\n")))
+            if add_message:
+                result = f"{result}\n{add_message}\n"
+        return result
+    
+    @staticmethod
+    def filter_output():
+        output_path = os.path.join(os.environ["HOME"], "bmc_studio", "var", "data", "cli_data", "issues.json")
+        try:
+            with open(output_path, "r") as output_fp:
+                items = json.load(output_fp)
+        except json.decoder.JSONDecoderError as e:
+            log.error("检查结果解析失败：%s", e.msg)
+            return "", ""
+        ci_enabled_issues = []
+        disabled_issues = []
+        for item in items:
+            rule = item.get("rule")
+            filepath = item.get("filepath")
+            ci_enabled = item.get("ciEnabled")
+            if ci_enabled:
+                ci_enabled_issues.append(
+                    (rule, filepath, item.get("errorMessage", ""), item.get("repairSuggestion", ""))
+                )
+            else:
+                disabled_issues.append((rule, filepath, item.get("errorMessage", ""), item.get("repairSuggestion", "")))
+        # 门禁环境下需要在每一行前加入ERROR，以红色显示
+        error = BmcgoCommand.process_issues_group(ci_enabled_issues, "ERROR")
+        warning = BmcgoCommand.process_issues_group(disabled_issues, "WARNING", "!!!!!以上是告警提示，后续门禁会逐步生效!!!!!")
+        return error, warning
+
+    def find_component_packages(self):
+        dependencies = self.service_dict.get(misc.CONAN_DEPDENCIES_KEY, {})
+        dep_list = dependencies.get("test", [])
+        dep_list.extend(dependencies.get("build", []))
+        for dep in dep_list:
+            package = dep.get(misc.CONAN, "")
+            comp_name = package.split("/")[0]
+            if "@" not in package:
+                package += f"@{misc.ConanUserEnum.CONAN_USER_RELEASE.value}/{misc.StageEnum.STAGE_RC.value}"
+            self.packages[comp_name] = package
+
+    def run(self):
+        service_path = os.path.join(cwd, "mds", "service.json")
+        if self.bconfig.manifest is None:
+            if not os.path.isfile(service_path):
+                raise RuntimeError("mds/service.json 文件不存在")
+            with open(service_path, "r") as service_fp:
+                self.service_dict = json.load(service_fp)
+            if "name" not in self.service_dict:
+                raise RuntimeError("mds/service.json 文件中缺少 name 配置")
+            self.repo_name = self.service_dict.get("name")
+            self.find_component_packages()
+        if self.disabled:
+            log.info("%s 仓库没有开启语法正确性和模型一致性检查", self.repo_name)
+            return 0
+        with tempfile.TemporaryDirectory(prefix="dependencies_repo_") as tempdir:
+            if self.bconfig.manifest:
+                self.fetch_agent.code_path = tempdir
+                self.fetch_agent.run()
+            elif self.packages:
+                FetchComponentCode(self.packages, tempdir, self.remote, include_open_source=False).run()
+            cmd = [self.studio_path, "check", "--repo", cwd, "--dependencies", tempdir, "--model", self.model]
+            if self.bconfig.manifest:
+                cmd.append("--manifest")
+                cmd.append("true")
+            os.chdir(self.studio_dir)
+            Helper.run(cmd, stdout=subprocess.DEVNULL)
+            os.chdir(cwd)
+        error, warning = self.filter_output()
+        if error:
+            log.warning(warning)
+            log.error("语法正确性和模型一致性检查不通过：\n%s\n", error)
+            return -1
+        log.warning(warning)
+        log.success("语法正确性和模型一致性检查通过\n")
+        return 0

bmcgo/functional/conan_index_build.py

@@ -0,0 +1,251 @@
+#!/usr/bin/env python3
+# encoding=utf-8
+# 描述：组件维护工具
+# Copyright (c) 2024 Huawei Technologies Co., Ltd.
+# openUBMC is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#         http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+import os
+import pathlib
+import subprocess
+import shutil
+import stat
+import json
+import yaml
+
+from bmcgo import errors
+from bmcgo import misc
+from bmcgo.misc import CommandInfo
+from bmcgo.utils.tools import Tools
+from bmcgo.bmcgo_config import BmcgoConfig
+
+tool = Tools("conanIdx")
+log = tool.log
+command_info: CommandInfo = CommandInfo(
+    group="Conan Index commmands",
+    name="build",
+    description=["构建conan index仓下的组件"],
+    hidden=False
+)
+
+
+def if_available(bconfig: BmcgoConfig):
+    return bconfig.conan_index is not None and bconfig.manifest is None
+
+
+class Function:
+    """功能类, 用于做一些通用操作
+    """
+    @classmethod
+    def git_check(cls, tag_list: list):
+        """检查 tag 是否在远端 url 内
+
+        Args:
+            tag_list (list): 依赖仓列表
+
+        Raises:
+            AttributeError: 如果其中有一个不是 tag, 则报错
+        """
+        check = True
+        for tag in tag_list:
+            cmd = f"git ls-remote {tag['url']} --tags {tag['tag']} | grep tags"
+            ret = subprocess.getstatusoutput(cmd)
+            if ret[0] != 0:
+                log.error("%s not a tag in %s", tag['tag'], tag['url'])
+                check = False
+        if check is True:
+            log.info("tag list check successfully !!!")
+        else:
+            raise AttributeError("tag list check fail !!!")
+
+
+class BmcgoCommand:
+    def __init__(self, bconfig: BmcgoConfig, *args):
+        self.bconfig = bconfig
+        parser = tool.create_common_parser("Conan Index Build")
+        self.args, _ = parser.parse_known_args(*args)
+        self.path = ""
+        self.version = ""
+        self.conan_package = ""
+        self.upload = False
+        self.remote = None
+        self.options = []
+        self.stage = misc.StageEnum.STAGE_DEV.value
+        self.enable_luajit = False
+        self.from_source = False
+        self.build_type = 'debug'
+        self.asan = False
+        self.profile = ''
+        self.initialize()
+        self.channel = f"@{misc.ConanUserEnum.CONAN_USER_DEV.value}/{self.stage}"
+
+    @staticmethod
+    def run_command(command, ignore_error=False, sudo=False, **kwargs):
+        """
+        如果ignore_error为False，命令返回码非0时则打印堆栈和日志并触发异常，中断构建
+        """
+        uptrace = kwargs.get("uptrace", 1)
+        kwargs["uptrace"] = uptrace
+        return tool.run_command(command, ignore_error, sudo, **kwargs)
+
+    def initialize(self):
+        self.set_package(self.args.conan_package)
+        self.upload = self.args.upload_package
+        if self.args.remote:
+            self.remote = self.args.remote
+        if self.args.options:
+            self.options = self.args.options
+        self.stage = self.args.stage
+        self.enable_luajit = self.args.enable_luajit
+        self.from_source = self.args.from_source
+        self.build_type = self.args.build_type
+        self.asan = self.args.asan
+        self.profile = Tools.get_conan_profile(None, self.build_type, self.enable_luajit)
+        if self.args.profile:
+            self.profile = self.args.profile
+
+    # 入参可以是huawei_secure_c/1.0.0样式
+    def set_package(self, path: str):
+        os.chdir(self.bconfig.conan_index.folder)
+        split = path.split("/")
+        if len(split) != 2:
+            raise errors.BmcGoException(f"包名称({path})错误，例：kmc/1.0.1")
+
+        if not os.path.isdir(split[0]):
+            raise errors.BmcGoException(f"包路径({split[0]})不存在，或不是文件夹")
+
+        config_yaml = os.path.join(self.bconfig.conan_index.folder, split[0], "config.yml")
+        with open(config_yaml) as f:
+            config_data = yaml.safe_load(f)
+            config_data = config_data.get('versions', None)
+            if config_data is None:
+                raise errors.BmcGoException(f"Config format error, config.yml path: {config_yaml}")
+            config_data = config_data.get(split[1], None)
+            if config_data is None:
+                raise errors.BmcGoException(f"Unkown version, config.yml path: {config_yaml}, version: {split[1]}")
+            folder = config_data.get('folder', None)
+            if config_data is None:
+                raise errors.BmcGoException(f"Unkown folder, config.yml path: {config_yaml}, version: {split[1]}")
+            self.path = "{}/{}".format(split[0], folder)
+            self.version = split[1]
+            if self.stage != "dev":
+                self.tag_check()
+
+    def install_luac(self):
+        conan_bin = os.path.join(os.path.expanduser('~'), ".conan", "bin")
+        # 设置PLD_LIBRARY_PATH环境变量，luajit运行时需要加载so动态库
+        ld_library_path = conan_bin + ":" + os.environ.get("LD_LIBRARY_PATH", "")
+        os.environ["LD_LIBRARY_PATH"] = ld_library_path
+        # 设置PATH环境变量，luajit无需指定全路径
+        path = conan_bin + ":" + os.environ.get("PATH", "")
+        os.environ["PATH"] = path
+        os.environ["LUA_PATH"] = f"{conan_bin}/?.lua"
+
+        # 待安装的luajit版本
+        luajit_pkg = f"luajit/2.1.0.B012@{tool.conan_user}/{misc.StageEnum.STAGE_STABLE.value}"
+        luajit_flag = luajit_pkg.split("@")[0].replace("/", "_")
+        luajit_flag = os.path.join(conan_bin, luajit_flag)
+
+        # 使能luajit时需要安装luajit
+        luac_path = os.path.join(conan_bin, "luajit")
+        # luajit版本一致且luac/luajit存在时赋权即可
+        if os.path.isfile(luajit_flag) and os.path.exists(luac_path):
+            os.chmod(luac_path, stat.S_IRWXU)
+            return
+        Tools.clean_conan_bin(conan_bin)
+        # 其它情况都尝试安装luajit
+        cmd = ["conan", "install", luajit_pkg]
+        cmd += ("-pr profile.dt.ini -if=temp/.deploy -g deploy").split()
+        if self.remote:
+            cmd += ["-r", self.remote]
+        self.run_command(cmd)
+        cmd = ["cp", "temp/.deploy/luajit/usr/bin/luajit", f"{conan_bin}"]
+        self.run_command(cmd)
+        cmd = ["cp", "temp/.deploy/luajit/usr/lib64/liblua.so", f"{conan_bin}"]
+        self.run_command(cmd)
+        cmd = ["cp", "-r", "temp/.deploy/luajit/usr/bin/jit", f"{conan_bin}"]
+        self.run_command(cmd)
+        os.chmod(luac_path, stat.S_IRWXU)
+        luajit2luac = shutil.which("luajit2luac.sh")
+        cmd = ["cp", luajit2luac, f"{conan_bin}/luac"]
+        self.run_command(cmd)
+        pathlib.Path(luajit_flag).touch(exist_ok=True)
+
+    def run(self):
+        self.install_luac()
+        if self.path == "" or self.version == "":
+            raise errors.BmcGoException(f"Path({self.path}) or version({self.version}) error")
+        os.chdir(self.bconfig.conan_index.folder)
+        os.chdir(self.path)
+        if self.stage != misc.StageEnum.STAGE_DEV.value:
+            # rc和stable模式
+            self.channel = f"@{tool.conan_user}/{self.stage}"
+        log.info("Start build package")
+        if self.build_type == "dt":
+            setting = "-s build_type=Dt"
+        else:
+            if self.build_type == "debug":
+                setting = "-s build_type=Debug"
+            else:
+                setting = "-s build_type=Release"
+        options = " "
+        packake_name = self.path.split("/")[0]
+        if self.asan:
+            options += f"-o {packake_name}:asan=True"
+        for option in self.options:
+            options += f" -o {option}"
+
+        dt_stat = os.environ.get("BMCGO_DT_RUN", "off")
+        show_log = True if dt_stat == "off" else False
+        pkg = packake_name + "/" + self.version + self.channel
+        append_cmd = f"-r {self.remote}" if self.remote else ""
+        cmd = "conan create . {} -pr={} -pr:b profile.dt.ini {} {} -tf None {}".format(
+            pkg, self.profile, setting, append_cmd, options
+        )
+        cmd += f" --build={packake_name}"
+        if self.from_source:
+            cmd += " --build=*"
+        else:
+            cmd += " --build=missing"
+        self.run_command(cmd, show_log=show_log)
+
+        if not self.upload:
+            return
+
+        cmd = "conan info {} -pr={} -j".format(pkg, self.profile)
+        info_json = self.run_command(cmd, shell=True, capture_output=True).stdout
+        for line in info_json.split("\n"):
+            if line.startswith("[{"):
+                info_json = line
+                break
+        info = json.loads(info_json)
+        for i in info:
+            if i["reference"] != pkg:
+                continue
+            cmd = "conan upload {}:{} -r {} -c --retry=3 --retry-wait 10 --all".format(pkg, i["id"], self.remote)
+            self.run_command(cmd)
+            log.info("===>>>Upload package successfully, pkg: {}:{}".format(pkg, i["id"]))
+
+    def tag_check(self):
+        yaml_file = f"{self.path}/conandata.yml"
+        if os.path.exists(yaml_file) is False:
+            log.warning("%s config not exists or does not need config file")
+        with open(yaml_file, mode="r") as fp:
+            config_data = yaml.safe_load(fp)
+        # 这个值是必定存在的
+        version_config = config_data['sources'][self.version]
+        tag_list = []
+        # 单个仓的处理
+        if version_config.get('url', None) is not None:
+            tag_list.append({"tag": version_config['branch'], "url": version_config['url']})
+        # 多个仓的处理
+        else:
+            for value in version_config.values():
+                tag_list.append({"tag": value['branch'], "url": value['url']})
+        # 这里的 git 命令的意思是检查远端是否有此 tag, 如果不加 grep, 可能导致分支被认为是 tag
+        Function.git_check(tag_list)