mindstudio-probe 1.3.0__py3-none-any.whl → 8.1.1__py3-none-any.whl

msprobe/core/config_check/checkers/dataset_checker.py

@@ -0,0 +1,138 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import json
+import pandas as pd
+from msprobe.core.common.file_utils import create_file_in_zip, load_json
+from msprobe.core.config_check.checkers.base_checker import BaseChecker
+from msprobe.core.config_check.config_checker import register_checker_item, register_pre_forward_fun_list
+from msprobe.core.config_check.utils.utils import config_checking_print, get_tensor_features
+from msprobe.core.common.decorator import recursion_depth_decorator
+from msprobe.core.common.framework_adapter import FmkAdp
+
+
+@recursion_depth_decorator("config_check: process_obj")
+def process_obj(obj):
+    if FmkAdp.is_tensor(obj):
+        return get_tensor_features(obj)
+    elif isinstance(obj, (tuple, list)):
+        return {i: process_obj(x) for i, x in enumerate(obj)}
+    elif isinstance(obj, dict):
+        return {k: process_obj(v) for k, v in obj.items()}
+    else:
+        return ""
+
+
+def parse_args_and_kargs(args, kwargs):
+    processed_args = process_obj(args)
+    processed_kargs = process_obj(kwargs)
+
+    return {
+        'args': processed_args,
+        'kwargs': processed_kargs
+    }
+
+
+@recursion_depth_decorator("config_check: compare_dataset_dicts")
+def compare_dataset_dicts(dict1, dict2, tag=''):
+    results = []
+    # 处理 dict1 中的键
+    for key in dict1:
+        new_tag = f"{tag}.{key}" if tag else key
+        if key not in dict2:
+            result = {'tag': new_tag, 'equal': False, 'status': 'delete'}
+            results.append(result)
+            continue
+        value1 = dict1[key]
+        value2 = dict2[key]
+        if not isinstance(value1, dict):
+            continue
+        if set(value1.keys()) == {'max', 'min', 'mean', 'norm'}:
+            equal = value1 == value2
+            relative_diffs = {
+                f"{k}_relative_diff": (abs(value1[k] - value2[k]) / value1[k]) if value1[k] != 0 else None
+                for k in ['max', 'min', 'mean', 'norm']
+            }
+            result = {'tag': new_tag, 'equal': equal, 'status': 'unchanged'}
+            result.update(relative_diffs)
+            results.append(result)
+        else:
+            results.extend(compare_dataset_dicts(value1, value2, new_tag))
+    # 处理 dict2 中独有的键
+    for key in dict2:
+        if key not in dict1:
+            new_tag = f"{tag}.{key}" if tag else key
+            result = {'tag': new_tag, 'equal': False, 'status': 'added'}
+            results.append(result)
+    return results
+
+
+def compare_dataset(bench_dir, cmp_dir):
+    all_results = []
+    for step in os.listdir(bench_dir):
+        step_path_bench = os.path.join(bench_dir, step)
+        if not os.path.isdir(step_path_bench):
+            continue
+        step_path_cmp = os.path.join(cmp_dir, step)
+        for rank in os.listdir(step_path_bench):
+            rank_path_bench = os.path.join(step_path_bench, rank, 'dataset.json')
+            rank_path_cmp = os.path.join(step_path_cmp, rank, 'dataset.json')
+            if not os.path.isfile(rank_path_bench) or not os.path.isfile(rank_path_cmp):
+                continue
+
+            dict1 = load_json(rank_path_bench)
+            dict2 = load_json(rank_path_cmp)
+            results = compare_dataset_dicts(dict1, dict2)
+            for result in results:
+                result['step'] = int(step.replace("step", ""))
+                result['rank'] = int(rank.replace("rank", ""))
+            all_results.extend(results)
+
+    df = pd.DataFrame(all_results, columns=DatasetChecker.result_header)
+    df = df.sort_values(by=['step', 'rank'], ascending=[True, True])
+    return df
+
+
+@register_checker_item("dataset")
+class DatasetChecker(BaseChecker):
+    input_needed = "model"
+    multi_rank = True
+
+    target_name_in_zip = "dataset"
+    result_header = ['step', 'rank', 'tag', 'equal', 'max_relative_diff',
+                     'min_relative_diff', 'mean_relative_diff', 'norm_relative_diff']
+
+    @staticmethod
+    def pack(pack_input):
+        output_zip_path = pack_input.output_zip_path
+
+        def collect_input(model, args, kwargs, step):
+            features = parse_args_and_kargs(args, kwargs)
+            dataset_filepath = os.path.join(DatasetChecker.target_name_in_zip, 
+                                            f"step{step}", f"rank{FmkAdp.get_rank_id()}", "dataset.json")
+            create_file_in_zip(output_zip_path, dataset_filepath, json.dumps(features, indent=4))
+            config_checking_print(f"add first dataset input features to zip")
+            
+        register_pre_forward_fun_list(collect_input)
+
+    @staticmethod
+    def compare(bench_dir, cmp_dir, output_path, fmk):
+        bench_dataset_pack_path = os.path.join(bench_dir, DatasetChecker.target_name_in_zip)
+        cmp_dataset_pack_path = os.path.join(cmp_dir, DatasetChecker.target_name_in_zip)
+
+        df = compare_dataset(bench_dataset_pack_path, cmp_dataset_pack_path)
+        pass_check = False not in df['equal'].values
+        return DatasetChecker.target_name_in_zip, pass_check, df

msprobe/core/config_check/checkers/env_args_checker.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import json
+
+import pandas as pd
+
+from msprobe.core.common.file_utils import load_json, load_yaml, create_file_with_content, create_file_in_zip
+from msprobe.core.config_check.checkers.base_checker import BaseChecker
+from msprobe.core.config_check.config_checker import register_checker_item
+from msprobe.core.config_check.utils.utils import config_checking_print
+from msprobe.core.common.const import Const
+
+
+dirpath = os.path.dirname(__file__)
+env_yaml_path = os.path.join(dirpath, "../resource/env.yaml")
+
+
+def collect_env_data():
+    result = {}
+    for key, value in os.environ.items():
+        result[key] = value
+    return result
+
+
+def get_device_type(env_json):
+    for key in env_json.keys():
+        if Const.ASCEND in key:
+            return Const.NPU_LOWERCASE
+    return Const.GPU_LOWERCASE
+
+
+def compare_env_data(npu_path, bench_path):
+    necessary_env = load_yaml(env_yaml_path)
+    cmp_data = load_json(npu_path)
+    cmp_type = get_device_type(cmp_data)
+    bench_data = load_json(bench_path)
+    bench_type = get_device_type(bench_data)
+    data = []
+    for _, value in necessary_env.items():
+        cmp_env = value.get(cmp_type)
+        bench_env = value.get(bench_type)
+        if not bench_env and not cmp_env:
+            continue
+        elif cmp_env:
+            cmp_env_name = cmp_env["name"]
+            cmp_value = cmp_data.get(cmp_env_name, value[cmp_type]["default_value"])
+            if not bench_env:
+                data.append(["only cmp has this env", cmp_env["name"], "", cmp_value, "warning"])
+                continue
+            bench_env_name = bench_env["name"]
+            bench_value = bench_data.get(bench_env_name, value[bench_type]["default_value"])
+            if cmp_value != bench_value:
+                data.append([bench_env_name, cmp_env_name, bench_value, cmp_value, "error"])
+        else:
+            bench_env_name = bench_env["name"]
+            bench_value = bench_data.get(bench_env_name) if bench_data.get(bench_env_name) else value[bench_type][
+                "default_value"]
+            data.append([bench_env_name, "only bench has this env", bench_value, "", "warning"])
+    df = pd.DataFrame(data, columns=EnvArgsChecker.result_header)
+    return df
+
+
+@register_checker_item("env")
+class EnvArgsChecker(BaseChecker):
+
+    target_name_in_zip = "env"
+    result_header = ["bench_env_name", "cmp_env_name", "bench_value", "cmp_value", "level"]
+
+    @staticmethod
+    def pack(pack_input):
+        output_zip_path = pack_input.output_zip_path
+        env_args_dict = collect_env_data()
+        create_file_in_zip(output_zip_path, EnvArgsChecker.target_name_in_zip, json.dumps(env_args_dict, indent=4))
+        config_checking_print(f"add env args to zip")
+
+    @staticmethod
+    def compare(bench_dir, cmp_dir, output_path, fmk):
+        bench_env_data = os.path.join(bench_dir, EnvArgsChecker.target_name_in_zip)
+        cmp_env_data = os.path.join(cmp_dir, EnvArgsChecker.target_name_in_zip)
+        df = compare_env_data(bench_env_data, cmp_env_data)
+        pass_check = "error" not in df['level'].values
+        return EnvArgsChecker.target_name_in_zip, pass_check, df

msprobe/core/config_check/checkers/hyperparameter_checker.py

@@ -0,0 +1,170 @@
+# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import json
+from difflib import SequenceMatcher
+
+from typing import Union, List, Dict, Any
+import pandas as pd
+
+from msprobe.core.config_check.checkers.base_checker import BaseChecker
+from msprobe.core.config_check.config_checker import register_checker_item
+from msprobe.core.config_check.utils.utils import compare_dict, config_checking_print, update_dict
+from msprobe.core.config_check.utils.hyperparameter_parser import ParserFactory
+from msprobe.core.common.file_utils import (os_walk_for_files, create_file_in_zip, load_json, create_file_with_list,
+                                            FileOpen, load_yaml)
+from msprobe.core.common.const import Const
+
+
+dirpath = os.path.dirname(__file__)
+hyperparameters_path = os.path.join(dirpath, "../resource/hyperparameter.yaml")
+parameter_name_mapping = load_yaml(os.path.realpath(hyperparameters_path))
+hyperparameters_dict = {}
+
+
+@register_checker_item("hyperparameter")
+class HyperparameterChecker(BaseChecker):
+    target_name_in_zip = "hyperparameters"
+    result_header = ["file_name", "bench_para", "cmp_para", "bench_value", "cmp_value", "matched_with", "level"]
+    hyperparameters_file_list = ["hyperparameters_static.json", "hyperparameters_dynamic.json"]
+
+    @staticmethod
+    def pack(pack_input):
+        shell_path = pack_input.shell_path
+        output_zip_path = pack_input.output_zip_path
+
+        if shell_path:
+            if not isinstance(shell_path, list):
+                raise TypeError("shell_path should be a list of file paths.")
+
+            hyperparameters = {}
+            parser_factory = ParserFactory()
+            for script_path in shell_path:
+                if os.path.isfile(script_path):
+                    parser = parser_factory.get_parser(os.path.splitext(script_path)[1])
+                    update_dict(hyperparameters, parser.run(os.path.realpath(script_path)))
+                else:
+                    config_checking_print(f"Warning: Script path {script_path} is not a file.")
+            if hyperparameters:
+                create_file_in_zip(output_zip_path,
+                                   os.path.join(HyperparameterChecker.target_name_in_zip,
+                                                HyperparameterChecker.hyperparameters_file_list[0]),
+                                   json.dumps(hyperparameters, indent=4))
+                config_checking_print(f"add static hyperparameters args to zip")
+            else:
+                config_checking_print(f"Warning: Failed to extract hyperparameters from script {shell_path}")
+        if hyperparameters_dict:
+            create_file_in_zip(output_zip_path,
+                               os.path.join(HyperparameterChecker.target_name_in_zip,
+                                            HyperparameterChecker.hyperparameters_file_list[1]),
+                               json.dumps(vars(hyperparameters_dict), default=lambda x: None, indent=4))
+            config_checking_print(f"add dynamic hyperparameters args to zip")
+
+    @staticmethod
+    def compare(bench_dir, cmp_dir, output_path, fmk):
+        all_diffs = []
+        for file_name in HyperparameterChecker.hyperparameters_file_list:
+            bench_model_dir = os.path.join(bench_dir, HyperparameterChecker.target_name_in_zip, file_name)
+            cmp_model_dir = os.path.join(cmp_dir, HyperparameterChecker.target_name_in_zip, file_name)
+            if os.path.isfile(bench_model_dir) and os.path.isfile(cmp_model_dir):
+                bench_hyperparameters = load_json(bench_model_dir)
+                cmp_hyperparameters = load_json(cmp_model_dir)
+                all_diffs.extend(
+                    HyperparameterChecker.compare_param(bench_hyperparameters, cmp_hyperparameters, file_name))
+        df = pd.DataFrame(all_diffs, columns=HyperparameterChecker.result_header)
+        pass_check = "error" not in df["level"].values
+        return HyperparameterChecker.target_name_in_zip, pass_check, df
+
+    @staticmethod
+    def compare_param(bench_params, cmp_params, file_name):
+        all_diffs = []
+        bench_param_names = bench_params.keys()
+        for bench_param_name in bench_param_names:
+            matched_cmp_param_name, matched_with = HyperparameterChecker._fuzzy_match_parameter(bench_param_name,
+                                                                                                cmp_params)
+            bench_param_value = bench_params[bench_param_name]
+            if matched_cmp_param_name:
+                cmp_param_value = cmp_params[matched_cmp_param_name]
+                if bench_param_value != cmp_param_value:
+                    all_diffs.append(
+                        [file_name, bench_param_name, matched_cmp_param_name, bench_param_value, cmp_param_value,
+                         matched_with, "error"])
+                del cmp_params[matched_cmp_param_name]
+            else:
+                all_diffs.append(
+                    [file_name, bench_param_name, "Only in benchmark", bench_param_value, "", "", "warning"])
+        for cmp_param_name, cmp_param_value in cmp_params.items():
+            all_diffs.append([file_name, "Only in comparison", cmp_param_name, "", cmp_param_value, "", "warning"])
+        all_diffs.sort()
+        return all_diffs
+
+    @staticmethod
+    def apply_patches(fmk):
+        try:
+            from megatron import training
+
+            def collect_hyperparameter_wrapper(func):
+                def wrapper(*args, **kwargs):
+                    global hyperparameters_dict
+                    result = func(*args, **kwargs)
+                    if not hyperparameters_dict:
+                        hyperparameters_dict = result
+                    return result
+                return wrapper
+            training.get_args = collect_hyperparameter_wrapper(training.get_args)
+        except ImportError:
+            config_checking_print("No megatron find.")
+        except Exception as e:
+            config_checking_print(f"Patch megatron method failed, detail:{str(e)}")
+
+    @staticmethod
+    def _fuzzy_match_parameter(param_name: str, available_params: Dict[str, Any]):
+        """
+        Fuzzy matches a parameter name against available parameter names using predefined
+        mappings and string similarity.
+        """
+        if param_name in available_params:
+            return param_name, Const.MATCH_MODE_NAME
+
+        canonical_name = None
+        for standard_name, aliases in parameter_name_mapping.items():
+            if param_name == standard_name or param_name in aliases:
+                canonical_name = standard_name
+                break
+
+        if canonical_name:
+            if canonical_name in available_params:
+                return canonical_name, Const.MATCH_MODE_MAPPING
+            for alias in parameter_name_mapping[canonical_name]:
+                if alias in available_params:
+                    config_checking_print(
+                        f"Matched '{param_name}' to alias '{alias}' via canonical name '{canonical_name}'")
+                    return alias, Const.MATCH_MODE_MAPPING
+
+        best_match_name = None
+        best_match_ratio = 0.8
+        for available_param_name in available_params:
+            ratio = SequenceMatcher(None, param_name.lower(), available_param_name.lower()).ratio()
+            if ratio > best_match_ratio:
+                best_match_ratio = ratio
+                best_match_name = available_param_name
+
+        if best_match_name:
+            config_checking_print(
+                f"Fuzzy matched parameter '{param_name}' to '{best_match_name}' (similarity: {best_match_ratio:.2f})")
+            return best_match_name, f"{Const.MATCH_MODE_SIMILARITY}:{best_match_ratio}"
+
+        return None, None

msprobe/core/config_check/checkers/pip_checker.py

@@ -0,0 +1,90 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import pandas as pd
+try:
+    import importlib.metadata as metadata
+except ImportError:
+    import importlib_metadata as metadata
+
+from msprobe.core.common.file_utils import load_yaml, create_file_in_zip
+from msprobe.core.config_check.checkers.base_checker import BaseChecker
+from msprobe.core.config_check.config_checker import register_checker_item
+from msprobe.core.config_check.utils.utils import config_checking_print
+from msprobe.core.common.file_utils import FileOpen, save_excel
+
+dirpath = os.path.dirname(__file__)
+depend_path = os.path.join(dirpath, "../resource/dependency.yaml")
+
+
+def load_pip_txt(file_path):
+    output_dir = {}
+    with FileOpen(file_path, 'r', encoding='utf-8') as file:
+        lines = file.readlines()
+        for line in lines:
+            info_list = line.strip().split("=")
+            output_dir[info_list[0]] = "" if len(info_list) != 2 else info_list[1]
+    return output_dir
+
+
+def collect_pip_data():
+    result = ""
+    packages = metadata.distributions()
+    for pkg in packages:
+        if pkg.metadata:
+            result += f"{pkg.metadata.get('Name')}={pkg.version}\n"
+    return result
+
+
+def compare_pip_data(bench_pip_path, cmp_pip_path, fmk):
+    necessary_dependency = load_yaml(depend_path)["dependency"]
+    necessary_dependency.append(fmk)
+    bench_data = load_pip_txt(bench_pip_path)
+    cmp_data = load_pip_txt(cmp_pip_path)
+    data = []
+    for package in necessary_dependency:
+        bench_version = bench_data.get(package)
+        cmp_version = cmp_data.get(package)
+
+        if bench_version != cmp_version:
+            data.append([package, bench_version if bench_version else 'None',
+                         cmp_version if cmp_version else 'None',
+                         "error"])
+
+    df = pd.DataFrame(data, columns=PipPackageChecker.result_header)
+    return df
+
+
+@register_checker_item("pip")
+class PipPackageChecker(BaseChecker):
+
+    target_name_in_zip = "pip"
+    result_header = ['package', 'bench version', 'cmp version', 'level']
+
+    @staticmethod
+    def pack(pack_input):
+        output_zip_path = pack_input.output_zip_path
+        pip_data = collect_pip_data()
+        create_file_in_zip(output_zip_path, PipPackageChecker.target_name_in_zip, pip_data)
+        config_checking_print(f"add pip info to zip")
+
+    @staticmethod
+    def compare(bench_dir, cmp_dir, output_path, fmk):
+        bench_pip_path = os.path.join(bench_dir, PipPackageChecker.target_name_in_zip)
+        cmp_pip_path = os.path.join(cmp_dir, PipPackageChecker.target_name_in_zip)
+        df = compare_pip_data(bench_pip_path, cmp_pip_path, fmk)
+        pass_check = "error" not in df['level'].values
+        return PipPackageChecker.target_name_in_zip, pass_check, df