openubmc-bingo 0.5.240__py3-none-any.whl

bmcgo/functional/deploy.py

@@ -0,0 +1,263 @@
+#!/usr/bin/env python3
+# encoding=utf-8
+# 描述：组件维护工具
+# Copyright (c) 2024 Huawei Technologies Co., Ltd.
+# openUBMC is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#         http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+import os
+import argparse
+import json
+import time
+from multiprocessing import Process
+import pysftp
+import requests
+import yaml
+import urllib3
+from bmcgo.misc import CommandInfo
+from bmcgo.utils.tools import Tools
+from bmcgo.bmcgo_config import BmcgoConfig
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+tool = Tools("deploy")
+log = tool.log
+command_info: CommandInfo = CommandInfo(
+    group="Misc commands",
+    name="deploy",
+    description=["产品包部署"],
+    hidden=True
+)
+
+
+def if_available(bconfig: BmcgoConfig):
+    # 只在二级流水线有效
+    return bconfig.component is None
+
+
+class BmcHostException(Exception):
+    def __init__(self, *args, **kwargs):
+        super(BmcHostException, self).__init__(*args, **kwargs)
+
+    def __str__(self):
+        return super(BmcHostException, self).__str__()
+
+
+class RestApiException(Exception):
+    def __init__(self, *args, **kwargs):
+        super(RestApiException, self).__init__(*args, **kwargs)
+
+    def __str__(self):
+        return super(RestApiException, self).__str__()
+
+
+class UpgradeTask(Process):
+    def __init__(self, config, cfg_file, filename):
+        super().__init__()
+        self.host = config.get("ip")
+        self.port = config.get("port")
+        self.username = config.get("username")
+        self.password = config.get("password")
+        self.filename = filename
+        if self.host is None:
+            raise BmcHostException(f"配置文件{cfg_file}缺少ip配置项.")
+        if self.port is None:
+            self.port = 443
+        else:
+            self.port = int(self.port)
+        if self.username is None:
+            raise BmcHostException(f"配置文件{cfg_file}缺少username配置项.")
+        if self.host is None:
+            raise BmcHostException(f"配置文件{cfg_file}缺少password配置项.")
+        if self.port != 443:
+            self.base_url = "https://" + self.host + ":" + str(self.port)
+        else:
+            self.base_url = "https://" + self.host
+        self.session = None
+        self.request_header = None
+
+    def run(self):
+        if self.session is None:
+            self._login()
+        try:
+            self._upload(self.filename)
+            if self.filename.endswith(".hpm"):
+                self._upgrade(self.filename)
+            else:
+                log.warning(f"{self.filename}文件名不是以.hpm结尾, 不支持升级.")
+        except Exception as e:
+            self._logout()
+            log.error("部署失败，错误信息:" + str(e))
+            return -1
+        self._logout()
+        return 0
+
+    def _login(self):
+        url = f"{self.base_url}/UI/Rest/Login"
+        log.info(f"{self.host} >> 登录")
+        payload = {
+            "UserName": self.username,
+            "Password": self.password,
+            "Domain": "LocaliBMC",
+            "Type": "Local"
+        }
+        resp = requests.post(url, data=payload, verify=False, timeout=10)
+        if resp.status_code != 200:
+            raise RestApiException(f"登录失败，登录接口返回状态码 {resp.status_code}.")
+        cookies = resp.cookies
+        body = json.loads(resp.content)
+        scsrf_token = body.get("XCSRFToken")
+        self.session = body.get("Session")
+        self.request_header = {
+            "X-Csrf-Token": scsrf_token,
+            "Cookie": "SessionId=" + cookies.get("SessionId")
+        }
+
+    def _upload_by_sftp(self, filename):
+        log.info(f"{self.host} >> 尝试使用sftp接口上传文件")
+        with pysftp.Connection(self.host, username=self.username, password=self.password) as sftp:
+            sftp.makedirs('/tmp/web')
+            with sftp.cd('/tmp/web'):
+                sftp.put(filename)
+
+    def _upload(self, filename):
+        log.info(f"{self.host} >> 上传文件 {filename} 至 /tmp/web/{os.path.basename(filename)}")
+        url = f"{self.base_url}/UI/Rest/FirmwareInventory"
+        files = {
+            "imgfile": open(filename, 'rb')
+        }
+        resp = requests.post(url, files=files, headers=self.request_header, verify=False, timeout=300)
+        if resp.status_code == 200:
+            return
+        log.warning(f"{self.host} >> 使用Rest接口上传{filename}失败，返回状态码 {resp.status_code}.")
+        self._upload_by_sftp(filename)
+
+    def _upgrade(self, filename):
+        log.info(f"{self.host} >> 调用接口启动升级")
+        url = f"{self.base_url}/UI/Rest/BMCSettings/UpdateService/FirmwareUpdate"
+        payload = {
+            "FilePath": os.path.join("/tmp/web/", os.path.basename(filename))
+        }
+        resp = requests.post(url, data=payload, headers=self.request_header, verify=False, timeout=10)
+        if resp.status_code != 200:
+            raise BmcHostException(f"升级{self.host}失败，Rest接口返回状态码 {resp.status_code}, 响应: {resp.content.decode()}.")
+        body = json.loads(resp.content)
+        url = self.base_url + body.get("url")
+        log.info(f"{self.host} >> 启动升级成功，开始检测升级状态")
+        sleep_sec = 1
+        progress = 0
+        while True:
+            time.sleep(sleep_sec)
+            resp = requests.get(url, headers=self.request_header, verify=False)
+            if resp.status_code != 200:
+                log.warning(f"{self.host} >> 轮询升级状态失败，Rest接口返回状态码 {resp.status_code}，升级可能失败，请自检.")
+                break
+            body = json.loads(resp.content)
+            state = body.get("state")
+            if state in ["Exception", "Suspended", "Interrupted", "Pending", "Killed", "Cancelled"]:
+                raise BmcHostException(f"{self.host} >> 升级状态接口返回状态异常{state}，升级失败")
+            if state == "Completed":
+                log.info(f"{self.host} >> 升级状态接口返回Completed状态")
+                return
+            if state not in ["Starting", "New", "Running"]:
+                raise BmcHostException(f"{self.host} >> 升级状态接口返回未知的{state}状态，升级可能失败，请自检.")
+            new_progress = int(body.get("prepare_progress"))
+            if new_progress != progress:
+                log.info(f"{self.host} >> 升级进度：{new_progress}")
+                progress = new_progress
+            if sleep_sec <= 2:
+                sleep_sec += 0.1
+
+    def _logout(self):
+        log.info(f"{self.host} >> 退出登录")
+        session_id = self.session.get("SessionID")
+        url = f"{self.base_url}/UI/Rest/Sessions/" + session_id
+        resp = requests.delete(url, headers=self.request_header, verify=False, timeout=10)
+        if resp.status_code != 200:
+            log.error(f"{self.host} >> 退出登录失败, status code {resp.status_code}")
+
+
+class BmcgoCommand:
+    def __init__(self, bconfig: BmcgoConfig, *args):
+        self.bconfig = bconfig
+        parser = argparse.ArgumentParser(prog="bingo deploy", description="BMC产品包部署升级，待部署主机配置由bmcgo config命令管理",
+                                         add_help=True, formatter_class=argparse.RawTextHelpFormatter)
+        parser.add_argument("-f", "--filename", help="待部署的文件", required=True)
+        parsed_args = parser.parse_args(*args)
+        self.filename = os.path.realpath(parsed_args.filename)
+        if not os.path.isfile(self.filename):
+            raise argparse.ArgumentError(f"file {parsed_args.filename} not exist")
+
+    @staticmethod
+    def _calc_checksum():
+        pass
+
+    def run(self):
+        tasks = self._read_hosts()
+        if len(tasks) == 0:
+            log.warning("未找到可用的部署配置")
+            return
+        if len(tasks) == 1:
+            tasks[0].run()
+            return
+        for task in tasks:
+            task.start()
+        while len(tasks) > 0:
+            time.sleep(1)
+            for task in tasks:
+                if task.is_alive():
+                    continue
+                if task.exitcode is not None and task.exitcode != 0:
+                    log.error(f"{task.host} >> 升级失败")
+                else:
+                    log.success(f"{task.host} >> 升级完成")
+                tasks.remove(task)
+        log.warning("部署过程受账号、网络、会话、代理、存储、设备状态等多方面影响，失败问题需要使用者自己定位，构建工程不提供直接支持。")
+
+    def _read_hosts(self):
+        tasks: list[UpgradeTask] = []
+        tasks = self._read_yml_hosts()
+        if tasks:
+            return tasks
+        tasks = self._read_config_hosts()
+        return tasks
+
+    def _read_yml_hosts(self):
+        tasks: list[UpgradeTask] = []
+        cur_dir = os.getcwd()
+        log.info("从当前路径开始向上递归查找.openUBMC_config.yml文件")
+        while cur_dir != "/":
+            cfg_file = os.path.join(cur_dir, ".openUBMC_config.yml")
+            if not os.path.isfile(cfg_file):
+                cur_dir = os.path.dirname(cur_dir)
+                continue
+            log.info(f"读取配置文件 {cfg_file}")
+            with open(cfg_file, "r") as fp:
+                config = yaml.safe_load(fp)
+            hosts = config.get("hosts", None)
+            if hosts is None:
+                log.warning(f"配置文件 {cfg_file} 未配置hosts对象")
+                continue
+            if isinstance(hosts, list):
+                for conf in hosts:
+                    tasks.append(UpgradeTask(conf, cfg_file, self.filename))
+            elif isinstance(hosts, object):
+                tasks.append(UpgradeTask(hosts, cfg_file, self.filename))
+            return tasks
+        return []
+
+    def _read_config_hosts(self):
+        tasks: list[UpgradeTask] = []
+        for item in filter(lambda x: x.startswith("deploy-"), self.bconfig.bmcgo_config_list.keys()):
+            cfg_file = item
+            conf = {}
+            conf["ip"] = item.split('-')[1]
+            for k, v in self.bconfig.bmcgo_config_list[item].items():
+                conf[k] = v
+            tasks.append(UpgradeTask(conf, cfg_file, self.filename))
+        return tasks

bmcgo/functional/diff.py

@@ -0,0 +1,235 @@
+#!/usr/bin/env python3
+# coding: utf-8
+# Copyright (c) 2025 Huawei Technologies Co., Ltd.
+# openUBMC is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#         http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+import os
+import subprocess
+import re
+import shutil
+import json
+from tempfile import NamedTemporaryFile
+from git import Repo
+from bmcgo import misc
+from bmcgo.utils.tools import Tools
+from bmcgo.bmcgo_config import BmcgoConfig
+from bmcgo import errors
+
+tools = Tools()
+log = Tools().log
+
+command_info: misc.CommandInfo = misc.CommandInfo(
+    group=misc.GRP_MISC,
+    name="diff",
+    description=["获取两个产品版本间, 组件版本变更时的合并记录"],
+    help_info=["usage: bingo diff <commit id before> <commit id after>"],
+    hidden=False
+)
+
+
+def if_available(bconfig: BmcgoConfig):
+    if bconfig.manifest is None:
+        return False
+    return True
+
+CONAN_DATA = ".conan/data"
+VERSION_KEY = "revision"
+
+
+class BmcgoCommand:
+    def __init__(self, bconfig: BmcgoConfig, *args):
+        # 这里可以是两个版本号, 也可以是两个version id, 版本号功能不开放
+        self.bconfig = bconfig
+        if len(args) != 1 or len(args[0]) != 2:
+            log.info("\n".join(command_info.help_info))
+            raise errors.bingoException("参数格式错误, 请查看上述提示")
+        self.version_before = args[0][0]
+        self.version_after = args[0][1]
+
+    @staticmethod
+    def check_path(version):
+        # 获取组件名称, 并作为下载的目标目录
+        component_name = version.split('/')[0]
+        to_path = f"temp/cmp_diff/{component_name}"
+        # 清空下载的目标目录
+        if os.path.exists(to_path):
+            shutil.rmtree(to_path)
+        os.makedirs(to_path)
+        return to_path, component_name
+
+    @staticmethod
+    def log_print(version_msg: dict, log_list: list):
+        description = ""
+        commit_index = 0
+        new_commit = False
+        edit_description = False
+
+        def printall(current_commit: list, description: str, version_msg: dict):
+            description = description.strip()
+            description = "当前MR未填写修改描述" if description == '' else description
+            cur_version = version_msg.get('url')[:-4]
+            for _log in current_commit[::-1]:
+                if "See merge request" in _log:
+                    line = re.search("(?<=!)\d+", _log)[0]
+                    log.info("%s/merge_requests/%s: %s", cur_version, line, description)
+                    break
+        #将数据分割为一个一个commit
+        for i, _log in enumerate(log_list):
+            if re.match('^\s*Created-by:', _log):
+                new_commit = False
+                edit_description = False
+            #都为真时记录描述信息
+            if new_commit and edit_description:
+                description += _log
+            if re.match('^\s*merge', _log):
+                edit_description = True
+            if re.match('^commit', _log):
+                new_commit = True
+                if i != 0:
+                    current_commit = log_list[commit_index: i - 1]
+                    printall(current_commit, description, version_msg)
+                    description = ''
+                    commit_index = i
+            if i == len(log_list) - 1:
+                current_commit = log_list[commit_index:]
+                printall(current_commit, description, version_msg)
+
+    @staticmethod
+    def get_version_msg(version):
+        tempfile = NamedTemporaryFile()
+        # 过滤只包含scm的信息, 并将其生成为字典对象
+        ret = tools.run_command(f"conan info {version} --json {tempfile.name} \
+              -r {misc.CONAN_REPO}", ignore_error=True, command_echo=False, capture_output=True)
+        file_handler = open(tempfile.name, "r")
+        conan_comps = json.load(file_handler)
+        version_msg = ""
+        for conan_comp in conan_comps:
+            comp_ref = conan_comp.get("reference", "")
+            if comp_ref == version:
+                version_msg = conan_comp.get("scm", "")
+                break
+        file_handler.close()
+        if ret.returncode != 0 or version_msg == "":
+            log.info(version)
+            log.info("仅自研软件支持版本对比功能!")
+            return 1, version_msg
+        # 由于两个组件版本之间的依赖可能冲突，所以清理一遍data
+        if os.path.exists(os.path.join(os.path.expanduser('~'), CONAN_DATA)):
+            shutil.rmtree(os.path.join(os.path.expanduser('~'), CONAN_DATA))
+        return ret.returncode, version_msg
+
+    def stage_match_parse(self, repo, stage):
+        merge_time_1 = repo.git.log("--pretty=format:%at", "-1", self.version_before)
+        merge_time_2 = repo.git.log("--pretty=format:%at", "-1", self.version_after)
+        # 先对比两个version id的subsys目录下的几个文件, 并获取到有差异的组件
+        if merge_time_1 > merge_time_2:
+            self.version_before, self.version_after = self.version_after, self.version_before
+
+        diff_list = repo.git.diff(self.version_before, self.version_after, "--", f"build/subsys/{stage}").split('\n')
+        log.info("====>>>>>> manifest 由 %s 演进至 %s <<<<<<====", self.version_before, self.version_after)
+        # 由于git diff会打印许多的无关内容, 这里对其过滤
+        cmp_bef_dict = {}
+        cmp_aft_dict = {}
+        for diff in diff_list:
+            # 时间旧的 version id
+            match_bef = None
+            match_bef = re.search("(?<=^-  - conan: ).*", diff)
+            if match_bef is not None:
+                match_bef = match_bef[0].replace('"', '')
+                cmp_bef_dict[match_bef.split('/')[0]] = match_bef
+
+            # 时间新的 version id
+            match_aft = None
+            match_aft = re.search("(?<=^\+  - conan: ).*", diff)
+            if match_aft is not None:
+                match_aft = match_aft[0].replace('"', '')
+                cmp_aft_dict[match_aft.split('/')[0]] = match_aft
+
+        # 差集列表
+        bef_aft_diff_set_list = []
+        aft_bef_diff_set_list = []
+        for match_name, match in cmp_bef_dict.items():
+            if match_name not in cmp_aft_dict.keys():
+                bef_aft_diff_set_list.append(match)
+            else:
+                self.parse_component_merge(match, cmp_aft_dict.get(match_name))
+                cmp_aft_dict.pop(match_name)
+
+        aft_bef_diff_set_list = cmp_aft_dict.values()
+        for del_cmp in bef_aft_diff_set_list:
+            log.info("---- ---- 由 %s 演进至 %s 删除组件 ---- ----", self.version_before, self.version_after)
+            self.parse_component_merge(del_cmp)
+        for del_cmp in aft_bef_diff_set_list:
+            log.info("++++ ++++ 由 %s 演进至 %s 新增组件 ++++ ++++", self.version_before, self.version_after)
+            self.parse_component_merge(del_cmp)
+
+    def parse_component_merge(self, version_before=None, version_after=None, sigle_commit_mode=False):
+        """如果传入的是两个组件的版本，使用此接口分析
+        """
+        # 单个提交模式下，执行这个if会自动退出
+        to_path, component_name = self.check_path(version_before)
+
+        ret, version_msg1 = self.get_version_msg(version_before)
+        if ret != 0:
+            return
+        if version_after:
+            ret, version_msg2 = self.get_version_msg(version_after)
+            if ret != 0:
+                return
+
+        # 由于组件在版本变更时，可能存储地址发生了改变，旧地址不存在，导致下载失败,同时输入的新地址可能也不存在
+        # 故用try进行确认，确保创建目标目录和下载正确执行
+        try:
+            Repo.clone_from(version_msg1.get("url"), to_path=to_path, branch='main')
+        except Exception as e1:
+            if version_after:
+                try:
+                    Repo.clone_from(version_msg2.get("url"), to_path=to_path, branch='main')
+                except Exception as e2:
+                    log.error(f"远程软件仓库地址不存在：{version_msg2.get('url')}")
+                    raise e2
+            else:
+                log.error(f"远程软件仓库地址不存在：{version_msg1.get('url')}")
+                raise e1
+        repo = Repo(to_path)
+
+        if version_after is None:
+            log.info("== %s: %s", component_name, version_before)
+            merge_log = repo.git.log('--merges', version_msg1.get(VERSION_KEY), '--remotes=main', "-3")
+            self.log_print(version_msg1, merge_log.split('\n'))
+            return
+
+        # 获取两个节点的提交时间, 使用 ‘old..new' 格式进行对比
+        merge_time_1 = repo.git.log("--pretty=format:%at", "-1", version_msg1.get(VERSION_KEY))
+        merge_time_2 = repo.git.log("--pretty=format:%at", "-1", version_msg2.get(VERSION_KEY))
+        if merge_time_1 > merge_time_2:
+            merge_time_1, merge_time_2 = merge_time_2, merge_time_1
+            version_before, version_after = version_after, version_before
+        log.info("==== ==== %s 由 %s 演进至 %s ==== ====", component_name, version_before, version_after)
+        merge_log = repo.git.log('--merges', f"{version_msg1.get(VERSION_KEY)}..{version_msg2.get(VERSION_KEY)}",
+                                '--remotes=main')
+        self.log_print(version_msg1, merge_log.split('\n'))
+
+    def parse_commit_merge(self):
+        """如果是两个commit id, 那么使用此接口进行分析
+        """
+        log.info(f"合并记录为时间排序，并非编号排序")
+        log.info(f"版本演进均为旧版本演进为新版本, 回滚提交也视为演进")
+        repo = Repo(self.bconfig.manifest.folder)
+        self.stage_match_parse(repo, "stable")
+        self.stage_match_parse(repo, "rc")
+
+    def run(self):
+        if "@" in self.version_before and "@" in self.version_after:
+            raise errors.bingoException(f"不支持两个组件版本对比")
+        elif "@" not in self.version_before and "@" not in self.version_after:
+            self.parse_commit_merge()
+        else:
+            raise errors.bingoException("输入的版本号无法解析")
+        return 0