openubmc-bingo 0.5.240__py3-none-any.whl

bmcgo/component/analysis/dep_node.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# coding: utf-8
+# Copyright (c) 2024 Huawei Technologies Co., Ltd.
+# openUBMC is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#         http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+
+import json
+import os
+import sys
+
+from typing import List, Set
+from bmcgo.logger import Logger
+
+global log
+log = Logger()
+
+TABLE_CONFLICT_WHITE_LIST = {"nsm": "t_snmp_config", "event_policy": "t_snmp_config"}
+
+
+class DepNode():
+    ref: str
+    options: str
+    package_id: str
+    revision: str
+
+    def __init__(self, node: dict, index):
+        self.ref = node.get("ref")
+        self.options = node.get("options")
+        self.package_id = node.get("package_id", "")
+        self.revision = node.get("revision")
+        self.requires: List[DepNode] = []
+        self.name = self.ref.split("@", 1)[0]
+        self.package_name = self.name.split("/", 1)[0]
+        self.package_version = self.name.split("/", 1)[-1]
+        self.index = int(index)
+        self.model_path = "unknown"
+        self.subsys_name = "unknown"
+        self.subsys_level = 1000
+        self.package_type = []      # 可选library, app, tool, configuration, command
+        self.intf_impl: Set[str] = set()
+        self.intf_deps: Set[str] = set()
+        self.local_tables = {}
+        self.remote_tables = {}
+        self.table_conflict = False
+        self._parse_mds()
+
+    @staticmethod
+    def _get_class_properties(class_data):
+        properties = set()
+        for _, intf_data in class_data.get("interfaces", {}).items():
+            for prop, prop_data in intf_data.get("properties", {}).items():
+                properties.add(prop_data.get("alias", prop))
+        for prop, prop_data in class_data.get("properties", {}).items():
+            properties.add(prop_data.get("alias", prop))
+        return properties
+
+    def set_subsys(self, subsys_name, subsys_level):
+        self.subsys_name = subsys_name
+        self.subsys_level = subsys_level
+
+    def set_package_type(self, package_type):
+        if package_type not in ["Library", "App", "Tool", "Configuration", "Command"]:
+            raise Exception("包类型错误")
+        self.package_type.append(package_type)
+
+    def _collect_table_info(self, class_data):
+        if "tableName" not in class_data:
+            return
+
+        table_name = class_data["tableName"]
+        if class_data.get("tableLocation", "") == "Local":
+            if table_name in self.local_tables:
+                log.error("%s中本地持久化表名冲突: %s", self.package_name, table_name)
+                self.table_conflict = True
+            self.local_tables[table_name] = True
+        else:
+            if table_name in self.remote_tables:
+                log.error("%s中远程持久化表名冲突: %s", self.package_name, table_name)
+                self.table_conflict = True
+            if self.package_name in TABLE_CONFLICT_WHITE_LIST \
+                and table_name == TABLE_CONFLICT_WHITE_LIST[self.package_name]:
+                self.remote_tables[table_name] = DepNode._get_class_properties(class_data)
+            else:
+                self.remote_tables[table_name] = True
+
+    def _parse_mds(self):
+        pkg_dir = self.ref.split("#")[0].replace("@", "/")
+        package_source = os.path.join(os.environ["HOME"], ".conan/data", pkg_dir, "package", self.package_id)
+        mds_path = os.path.join(package_source, "include/mds")
+        self.model_path = os.path.join(mds_path, "model.json")
+        if os.path.exists(self.model_path):
+            with open(self.model_path, "r") as file_descriptor:
+                model_data = json.load(file_descriptor)
+            for class_name in model_data:
+                self._collect_table_info(model_data[class_name])
+                for intf in model_data[class_name].get("interfaces", {}):
+                    self.intf_impl.add(intf)
+        service_path = os.path.join(mds_path, "service.json")
+        if os.path.exists(service_path):
+            with open(service_path, "r") as file_descriptor:
+                service_data = json.load(file_descriptor)
+            for intf_entry in service_data.get("required", []):
+                if "interface" in intf_entry:
+                    self.intf_deps.add(intf_entry.get("interface"))

bmcgo/component/analysis/intf_deps.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python
+# coding: utf-8
+# Copyright (c) 2024 Huawei Technologies Co., Ltd.
+# openUBMC is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#         http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+
+from collections import defaultdict
+from typing import Dict, List
+import json
+import os
+import stat
+
+from pyecharts import options as opts
+from pyecharts.charts import Graph
+from pyecharts.options import GraphLink, GraphNode
+
+from bmcgo.component.analysis.dep_node import DepNode
+from bmcgo.component.analysis.rule import Rules
+from bmcgo.logger import Logger
+from bmcgo import misc
+
+global log
+log = Logger()
+INTERFACE_DEPENDENCIES = "interfaceDependencies"
+
+
+class InterfaceDependenciesAnalysis():
+    def __init__(self, nodes: List[DepNode], rules: List[Rules]):
+        # All packages in the conan build data
+        self._nodes: List[DepNode] = list()
+        # Select packages that have dependencies on d-bus
+        for node in nodes:
+            if len(node.intf_impl) != 0 or len(node.intf_deps) != 0:
+                self._nodes.append(node)
+        self._rules = rules
+
+        # All d-bus interfaces that the package implements
+        self.node_impl_intfs: Dict[str, List[DepNode]] = defaultdict(list)
+
+        # All d-bus interfaces the package depends on
+        self.node_deps_intfs: Dict[str, List[DepNode]] = defaultdict(list)
+
+        # The package dependency graph: vertex are conan packages and edges are interfaces
+        self.intf_deps_graph: Dict[str, Dict[str, List[str]]] = defaultdict(dict)
+        self.intf_deps_unallowed_graph: Dict[str, Dict[str, List[str]]] = defaultdict(dict)
+
+        self._node_intf_relation()
+
+    def is_dependency_allowed(self):
+        return not bool(self.intf_deps_unallowed_graph)
+
+    def assemble_deps_json_desc(self, path):
+        node_name_map = dict()
+        for node in self._nodes:
+            node_name_map[node.package_name] = {
+                misc.NAME: node.package_name,
+                "version": node.package_version,
+                "subsystem": node.subsys_name,
+                "type": node.package_type
+            }
+
+        json_dict = {"InterfaceDependencyGraph": [], "IllegalInterfaceDependencyGraph": []}
+        for src in self.intf_deps_graph:
+            entry = {"sourceComponent": node_name_map.get(src), INTERFACE_DEPENDENCIES: []}
+            for tgt, intfs in self.intf_deps_graph[src].items():
+                entry[INTERFACE_DEPENDENCIES].append({"targetComponent": node_name_map.get(tgt), "interfaces": intfs})
+            json_dict["InterfaceDependencyGraph"].append(entry)
+
+        for src in self.intf_deps_unallowed_graph:
+            entry = {"sourceComponent": node_name_map.get(src), INTERFACE_DEPENDENCIES: []}
+            for tgt, intfs in self.intf_deps_unallowed_graph[src].items():
+                entry[INTERFACE_DEPENDENCIES].append({"targetComponent": node_name_map.get(tgt), "interfaces": intfs})
+            json_dict["IllegalInterfaceDependencyGraph"].append(entry)
+
+        log.info("保存 BMC 组件 d-bus 接口依赖信息到: %s", path)
+        flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
+        modes = stat.S_IWUSR | stat.S_IRUSR
+        with os.fdopen(os.open(path, flags, modes), 'w') as file_descriptor:
+            file_descriptor.write(json.dumps(json_dict, indent=4))
+
+    def visualize_graph(self, path):
+        subsystems = set()
+        for pkg in self._nodes:
+            subsystems.add(pkg.subsys_name)
+        categories = []
+        for subsys in subsystems:
+            categories.append({misc.NAME: subsys, "symbolSize": 100})
+        categories.append({misc.NAME: "unknown", "symbolSize": 100})
+        category_id_map: Dict[str, int] = dict()
+        for index, category in enumerate(categories):
+            category_id_map[category[misc.NAME]] = index
+
+        graph_nodes: List[GraphNode] = []
+        for pkg in self._nodes:
+            graph_nodes.append(GraphNode(name=pkg.package_name, symbol_size=5, \
+                    category=category_id_map.get(pkg.subsys_name, len(categories))))
+
+        links: list[GraphLink] = []
+        for src in self.intf_deps_graph:
+            for dest, intfs in self.intf_deps_graph[src].items():
+                if src in self.intf_deps_unallowed_graph and dest in self.intf_deps_unallowed_graph[src]:
+                    links.append(GraphLink(
+                        source=src,
+                        target=dest,
+                        linestyle_opts=opts.LineStyleOpts(color=misc.COLOR_RED, curve=0.2, width=1.5),
+                        value=self.intf_deps_unallowed_graph[src][dest]))
+                else:
+                    links.append(GraphLink(source=src, target=dest, value=intfs))
+
+        log.info("保存 BMC 组件 d-bus 接口依赖信息到: %s", path)
+        (Graph(init_opts=opts.InitOpts(width="100%", height="1200px", page_title="BMC-Interface-Dependency-Graph"))
+        .add("", graph_nodes, links, categories,
+             repulsion=100,
+             is_draggable=False,
+             edge_symbol="arrow",
+             edge_symbol_size=[0, 5],
+             is_layout_animation=False,
+             label_opts=opts.LabelOpts(position="right", formatter="{b}"))
+             .set_global_opts(
+            title_opts=opts.TitleOpts(title="Graph-BMC-Interface-Dependency-Graph"),
+            legend_opts=opts.LegendOpts(orient="vertical", pos_left="2%", pos_top="20%"))
+            .render(path))
+
+    def _node_intf_relation(self):
+        self._construct_node_intf_detail()
+
+        for intf, nodes in self.node_deps_intfs.items():
+            if intf not in self.node_impl_intfs:
+                continue
+            for src in nodes:
+                for tgt in self.node_impl_intfs[intf]:
+                    self._add_two_node_relation(src, tgt, intf)
+
+    def _construct_node_intf_detail(self):
+        for node in self._nodes:
+            for intfi in node.intf_impl:
+                self.node_impl_intfs[intfi].append(node)
+            for intfd in node.intf_deps:
+                self.node_deps_intfs[intfd].append(node)
+
+    def _add_two_node_relation(self, src: DepNode, tgt: DepNode, intf: str):
+        interface_relation = self.intf_deps_graph[src.package_name].get(tgt.package_name, [])
+        interface_relation.append(intf)
+        self.intf_deps_graph[src.package_name][tgt.package_name] = interface_relation
+        if self._check_intf_dependency(src, tgt, intf):
+            return
+        unallowed_interface_relation = self.intf_deps_unallowed_graph[src.package_name].get(tgt.package_name, [])
+        unallowed_interface_relation.append(intf)
+        self.intf_deps_unallowed_graph[src.package_name][tgt.package_name] = unallowed_interface_relation
+
+    def _check_intf_dependency(self, src: DepNode, tgt: DepNode, intf: str):
+        for rules in self._rules:
+            if rules.intf_dep_check(src, tgt, intf):
+                return True
+        log.error("发现不合规的总线接口依赖, 源包: %s, 目标包: %s",
+                src.name, tgt.name)
+        return False

bmcgo/component/analysis/intf_validation.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python
+# coding: utf-8
+# Copyright (c) 2024 Huawei Technologies Co., Ltd.
+# openUBMC is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#         http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+
+from collections import defaultdict
+from typing import Dict, Set
+import json
+import os
+import re
+import shutil
+import subprocess
+
+from bmcgo.logger import Logger
+from bmcgo.codegen.c.helper import Helper
+from bmcgo import misc
+
+global log
+log = Logger()
+
+
+class InterfaceValidation:
+    def __init__(self, remote):
+        self.intf_impl: Set[str] = set()
+        self.intf_deps: Set[str] = set()
+        self.intf_hardcoded: Dict[str, Set[str]] = defaultdict(set)
+        self.all_predefined_intfs: Set[str] = set()
+        self.remote = remote
+        self._parse_mds_interfaces()
+        self._parse_hardcoded_interfaces()
+        self._pull_all_interfaces()
+
+    @staticmethod
+    def extract_dbus_intf_in_file(file_path):
+        all_intfs = list()
+        with open(file_path, "r") as file_descriptor:
+            # iBMC d-bus interface pattern: bmc.(kepler|dev).[A-Z]
+            pattern = "[\"\']bmc\.(?:kepler|dev)(?:\.[A-Z].*?)?[\"\']"
+            intf_match = re.findall(pattern, file_descriptor.read())
+            for intf in intf_match:
+                all_intfs.append(intf[1:-1])
+        return all_intfs
+
+    @staticmethod
+    def _check_bus_call_in_file(file_path):
+        patterns = ["bus:p?call\(.+\n?"]
+        has_pattern = False
+        with open(file_path, "r") as file_descriptor:
+            src_lua = file_descriptor.read()
+            for pattern in patterns:
+                bus_call = re.findall(pattern, src_lua)
+                if len(bus_call) != 0:
+                    log.error("在 %s 发现非自动生成的代码调用, 匹配到: %s. 请使用代码自动生成(generation)", file_path, pattern)
+                    has_pattern = True
+        return has_pattern
+
+    @staticmethod
+    def _parse_mds_package_deps():
+        channel = f"@{misc.ConanUserEnum.CONAN_USER_RELEASE.value}/{misc.StageEnum.STAGE_RC.value}"
+        package = f"mdb_interface/[>=0.0.1]{channel}"
+        service_path = os.path.join(os.getcwd(), "mds/service.json")
+        if not os.path.exists(service_path):
+            return package
+        with open(service_path, "r") as service_fp:
+            try:
+                content = json.load(service_fp)
+            except json.decoder.JSONDecodeError as error:
+                return package
+        dependencies = content.get("dependencies")
+        if not dependencies:
+            return package
+        dep_list = dependencies.get("test", [])
+        dep_list.extend(dependencies.get("build", []))
+        for dep in dep_list:
+            conan_package = dep.get(misc.CONAN, "")
+            if not conan_package.startswith("mdb_interface"):
+                continue
+            if "@" in conan_package:
+                package = conan_package
+            else:
+                package = f"{conan_package}{channel}"
+        return package
+
+    def run(self):
+        intf_validation_passed = self._validate_mds_intf() and self._validate_hardcoded_intf() and \
+                self._check_dbus_call()
+        if intf_validation_passed:
+            log.info("接口检验通过")
+        else:
+            log.error("接口校验未通过")
+        return intf_validation_passed
+
+    def _parse_mds_interfaces(self):
+        mds_model_path = os.path.join(os.getcwd(), "mds/model.json")
+        if not os.path.exists(mds_model_path):
+            return
+        with open(mds_model_path, "r") as file_descriptor:
+            try:
+                model_data = json.load(file_descriptor)
+            except json.decoder.JSONDecodeError as error:
+                raise OSError(f"无法加载文件 {mds_model_path}, 错误消息: {error.msg}") from error
+            else:
+                for class_name in model_data:
+                    self.intf_impl.update(model_data[class_name].get("interfaces", {}).keys())
+
+        mds_service_path = os.path.join(os.getcwd(), "mds/service.json")
+        if not os.path.exists(mds_service_path):
+            return
+        with open(mds_service_path, "r") as file_descriptor:
+            try:
+                service_data = json.load(file_descriptor)
+            except json.decoder.JSONDecodeError as error:
+                raise OSError(f"无法加载文件 {mds_model_path}, 错误消息: {error.msg}") from error
+            else:
+                for require_object in service_data.get("required", []):
+                    self.intf_deps.add(require_object.get("interface"))
+
+    """
+    @brief: parse the d-bus interface claimed in the source code
+    """
+    def _parse_hardcoded_interfaces(self):
+        for root, _, files in os.walk(os.path.join(os.getcwd(), "", "src")):
+            for file_name in files:
+                if not file_name.endswith(".lua"):
+                    continue
+                file_path = os.path.join(root, file_name)
+                for intf in self.extract_dbus_intf_in_file(file_path):
+                    self.intf_hardcoded[intf].update([file_path])
+
+    def _pull_interfaces_from_repo(self):
+        mdb_interface_repo_url = "https://gitcode.com/openUBMC/mdb_interface.git"
+        mktemp_cmd = subprocess.run(["/usr/bin/mktemp", "-d", "--suffix", "_mdb_interface"], capture_output=True)
+        if mktemp_cmd.returncode != 0:
+            raise OSError(f"创建文件夹失败, 错误消息: {mktemp_cmd.stderr}")
+        temp_dir = mktemp_cmd.stdout.decode().strip('\n')
+
+        git_cmd = Helper.get_git_path()
+        repo_fetch = subprocess.run(
+            [git_cmd, "clone", mdb_interface_repo_url, temp_dir, "--depth=1"],
+            stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+        if repo_fetch.returncode != 0:
+            log.error("克隆远端仓库 %s 失败, 错误信息: %s", mdb_interface_repo_url, repo_fetch.stderr)
+        intf_dir = os.path.join(temp_dir, "json/intf/mdb/bmc")
+
+        for root, _, files in os.walk(intf_dir):
+            for file_name in files:
+                if not file_name.endswith(".json"):
+                    continue
+                file_path = os.path.join(root, file_name)
+                self.all_predefined_intfs.update(intf for intf in self.extract_dbus_intf_in_file(file_path))
+
+        shutil.rmtree(temp_dir, ignore_errors=True)
+
+    def _pull_interfaces_from_conan(self):
+        pkg = self._parse_mds_package_deps()
+        mktemp_cmd = subprocess.run(["/usr/bin/mktemp", "-d", "--suffix", "_mdb_interface"], capture_output=True)
+        if mktemp_cmd.returncode != 0:
+            raise OSError(f"创建文件夹失败, 错误消息: {mktemp_cmd.stderr}")
+        temp_dir = mktemp_cmd.stdout.decode().strip('\n')
+        cmd = [misc.CONAN, "install", pkg, f"-if={temp_dir}", "--build=missing", "-g", "deploy", "-s", "build_type=Dt"]
+        if self.remote:
+            cmd += ["-r", self.remote]
+        subprocess.call(cmd)
+        intf_dir = os.path.join(temp_dir, "mdb_interface/opt/bmc/apps/mdb_interface/intf/mdb/bmc")
+
+        for root, _, files in os.walk(intf_dir):
+            for file_name in files:
+                if not file_name.endswith(".json"):
+                    continue
+                file_path = os.path.join(root, file_name)
+                self.all_predefined_intfs.update(intf for intf in self.extract_dbus_intf_in_file(file_path))
+        shutil.rmtree(temp_dir, ignore_errors=True)
+
+    def _pull_interfaces_from_codegen(self):
+        service_path = os.path.join(os.getcwd(), "mds/service.json")
+        try:
+            with open(service_path, 'r') as file_descriptor:
+                repo_name = json.load(file_descriptor).get('name')
+        except Exception as e:
+            log.error('mds/service.json 文件解析失败: %s', e)
+            return
+        generated_intf_dir = os.path.join(os.getcwd(), "gen", repo_name, "json_types")
+        for root, _, files in os.walk(generated_intf_dir):
+            for file_name in files:
+                if not file_name.endswith(".lua"):
+                    continue
+                file_path = os.path.join(root, file_name)
+                self.all_predefined_intfs.update(intf for intf in self.extract_dbus_intf_in_file(file_path))
+        client_intf_path = os.path.join(os.getcwd(), "gen", repo_name, "client.lua")
+        if not os.path.exists(client_intf_path):
+            return
+        self.all_predefined_intfs.update(intf for intf in self.extract_dbus_intf_in_file(client_intf_path))
+
+    """
+    @brief: fetch the mdb_interface repository and parse all predefined d-bus interfaces
+    """
+    def _pull_all_interfaces(self):
+        self._pull_interfaces_from_repo()
+        self._pull_interfaces_from_conan()
+        self._pull_interfaces_from_codegen()
+
+    """
+    @brief: check the d-bus interfaces claimed in mds are defined in mdb_interface
+    """
+    def _validate_mds_intf(self):
+        intf_impl_in_mdb = True
+        intf_deps_in_mdb = True
+        for intf in self.intf_impl:
+            if intf not in self.all_predefined_intfs:
+                log.error("在 mds/model.json 中找到未定义的接口: %s", intf)
+                intf_impl_in_mdb = False
+
+        for intf in self.intf_deps:
+            if intf not in self.all_predefined_intfs:
+                log.error("在 mds/service.json 找到未定义的接口: %s", intf)
+                intf_deps_in_mdb = False
+
+        return intf_impl_in_mdb and intf_deps_in_mdb
+
+    """
+    @brief: check the hardcoded d-bus interfaces in the source lua code are claimed in mds and mdb_interface
+    """
+    def _validate_hardcoded_intf(self):
+        hardcoded_intf_in_mdb = True
+        hardcode_intf_in_mds = True
+        for intf, paths in self.intf_hardcoded.items():
+            if intf not in self.all_predefined_intfs:
+                log.error("在 %s 找到 未定义的接口 %s. 请检查你的源码", paths, intf)
+                hardcoded_intf_in_mdb = False
+            if intf not in self.intf_impl and intf not in self.intf_deps:
+                log.warning("在 %s 找到 未定义的接口 %s. 请使用 mds 模板", paths, intf)
+                hardcoded_intf_in_mds = False
+        return hardcoded_intf_in_mdb
+
+    """
+    @brief: check d-bus call snippet in source code. It is recommend to use generated templates for d-bus call.
+    """
+    def _check_dbus_call(self):
+        has_pattern = False
+        for root, _, files in os.walk(os.path.join(os.getcwd(), "src")):
+            for file_name in files:
+                if not file_name.endswith(".lua"):
+                    continue
+                file_path = os.path.join(root, file_name)
+                if self._check_bus_call_in_file(file_path):
+                    has_pattern = True
+        return not has_pattern