mindstudio-probe 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

msprobe/mindspore/api_accuracy_checker/type_mapping.py

@@ -0,0 +1,114 @@
+from mindspore.common import dtype as mstype
+import numpy as np
+import mindspore
+import torch
+
+INT8 = "Int8"
+UINT8 = "UInt8"
+INT16 = "Int16"
+UINT16 = "UInt16"
+INT32 = "Int32"
+UINT32 = "UInt32"
+INT64 = "Int64"
+UINT64 = "UInt64"
+FLOAT16 = "Float16"
+FLOAT32 = "Float32"
+FLOAT64 = "Float64"
+BOOL = "Bool"
+BFLOAT16 = "BFloat16"
+INT4 = "Int4"
+
+
+dtype_str_to_ms_dtype = {
+    INT8: mstype.int8,
+    UINT8: mstype.uint8,
+    INT16: mstype.int16,
+    UINT16: mstype.uint16,
+    INT32: mstype.int32,
+    UINT32: mstype.uint32,
+    INT64: mstype.int64,
+    UINT64: mstype.uint64,
+    FLOAT16: mstype.float16,
+    FLOAT32: mstype.float32,
+    FLOAT64: mstype.float64,
+    BOOL: mstype.bool_,
+    BFLOAT16: mstype.bfloat16,
+    INT4: mstype.qint4x2
+}
+ms_dtype_to_dtype_str = {value: key for key, value in dtype_str_to_ms_dtype.items()}
+
+
+dtype_str_to_np_dtype = {
+    INT8: np.int8,
+    UINT8: np.uint8,
+    INT16: np.int16,
+    UINT16: np.uint16,
+    INT32: np.int32,
+    UINT32: np.uint32,
+    INT64: np.int64,
+    UINT64: np.uint64,
+    FLOAT16: np.float16,
+    FLOAT32: np.float32,
+    FLOAT64: np.float64,
+    BOOL: np.bool_
+}
+np_dtype_to_dtype_str = {value: key for key, value in dtype_str_to_np_dtype.items()}
+
+dtype_str_to_torch_dtype = {
+    INT8: torch.int8,
+    UINT8: torch.uint8,
+    INT16: torch.int16,
+    INT32: torch.int32,
+    INT64: torch.int64,
+    FLOAT16: torch.float16,
+    FLOAT32: torch.float32,
+    FLOAT64: torch.float64,
+    BOOL: torch.bool,
+    BFLOAT16: torch.bfloat16,
+}
+torch_dtype_to_dtype_str = {value: key for key, value in dtype_str_to_torch_dtype.items()}
+
+MINDSPORE_TENSOR_TYPE_STR = "mindspore.Tensor"
+BOOL_TYPE_STR = "bool"
+INT_TYPE_STR = "int"
+FLOAT_TYPE_STR = "float"
+SLICE_TYPE_STR = "slice"
+TUPLE_TYPE_STR = "tuple"
+STR_TYPE_STR = "str"
+
+api_info_type_str_to_type = {
+    MINDSPORE_TENSOR_TYPE_STR: mindspore.Tensor,
+    BOOL_TYPE_STR: bool,
+    INT_TYPE_STR: int,
+    FLOAT_TYPE_STR: float,
+    SLICE_TYPE_STR: slice,
+    STR_TYPE_STR: str,
+}
+type_to_api_info_type_str = {value: key for key, value in api_info_type_str_to_type.items()}
+
+DEFAULT_CONSTRUCT_NP_FLOAT_DTYPE = np.float64
+DEFAULT_CONSTRUCT_NP_INT_DTYPE = np.float64
+DEFAULT_CONSTRUCT_NP_UINT_DTYPE = np.float64
+
+float_dtype_str_list = [
+    FLOAT16,
+    FLOAT32,
+    FLOAT64,
+    BFLOAT16,
+]
+
+int_dtype_str_list = [
+    INT8,
+    INT16,
+    INT32,
+    INT64,
+    BOOL,
+    INT4,
+]
+
+uint_dtype_str_list = [
+    UINT8,
+    UINT16,
+    UINT32,
+    UINT64,
+]

msprobe/mindspore/api_accuracy_checker/utils.py

@@ -0,0 +1,63 @@
+from msprobe.core.common.exceptions import ApiAccuracyCheckerException
+from msprobe.core.common.log import logger
+
+def check_and_get_from_json_dict(dict_instance, key, key_description, accepted_type=None, accepted_value=None):
+    '''
+    Args:
+        dict_instance: dict, dict parsed from input json
+        key: str
+        key_description: str
+        accepted_type: tuple
+        accepted_value: Union[tuple, list]
+
+    Return:
+        value, the corresponding value of "key" in "dict_instance"
+
+    Exception:
+        raise ApiAccuracyCheckerException.ParseJsonFailed error when
+        1. dict_instance is not a dict
+        2. value is None
+        3. value is not accepted type
+        4. value is not accepted value
+    '''
+    parse_failed_exception = ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed)
+    if not isinstance(dict_instance, dict):
+        logger.error_log_with_exp("check_and_get_from_json_dict failed: input is not a dict", parse_failed_exception)
+    value = dict_instance.get(key)
+    if value is None:
+        logger.error_log_with_exp(f"check_and_get_from_json_dict failed: {key_description} is missing",
+                                  parse_failed_exception)
+    elif accepted_type is not None and not isinstance(value, accepted_type):
+        logger.error_log_with_exp(
+            f"check_and_get_from_json_dict failed: {key_description} is not accepted type: {accepted_type}",
+            parse_failed_exception)
+    elif accepted_value is not None and value not in accepted_value:
+        logger.error_log_with_exp(
+            f"check_and_get_from_json_dict failed: {key_description} is not accepted value: {accepted_value}",
+            parse_failed_exception)
+    return value
+
+def convert_to_tuple(input):
+    if isinstance(input, (tuple, list)):
+        return tuple(input)
+    else:
+        input_list = [input]
+        return tuple(input_list)
+
+class GlobalContext:
+    def __init__(self):
+        self.is_constructed = True
+        self.dump_data_dir = ""
+
+    def init(self, is_constructed, dump_data_dir):
+        self.is_constructed = is_constructed
+        self.dump_data_dir = dump_data_dir
+
+    def get_dump_data_dir(self):
+        return self.dump_data_dir
+
+    def get_is_constructed(self):
+        return self.is_constructed
+
+
+global_context = GlobalContext()

msprobe/mindspore/cell_processor.py

@@ -0,0 +1,34 @@
+from msprobe.core.data_dump.scope import ModuleRangeScope
+from msprobe.core.common.const import Const
+from msprobe.mindspore.common.log import logger
+
+
+class CellProcessor:
+    cell_count = {}
+
+    def __init__(self, scope):
+        if isinstance(scope, ModuleRangeScope):
+            self.scope = scope
+        else:
+            self.scope = None
+        
+    @staticmethod
+    def set_cell_count(cell_name):
+        if cell_name not in CellProcessor.cell_count:
+            CellProcessor.cell_count[cell_name] = 0
+        else:
+            CellProcessor.cell_count[cell_name] += 1
+        return CellProcessor.cell_count[cell_name]
+    
+    def node_hook(self, name_prefix, start_or_stop, **kwargs):
+        def begin_hook(cell, input):
+            index = self.set_cell_count(name_prefix)
+            cell.mindstudio_reserved_name = full_name = name_prefix + Const.SEP + str(index)
+            if self.scope:
+                self.scope.begin_module(full_name)
+    
+        def end_hook(cell, input, output):
+            if self.scope:
+                self.scope.end_module(cell.mindstudio_reserved_name)
+
+        return begin_hook if Const.START == start_or_stop else end_hook

msprobe/mindspore/common/const.py

@@ -0,0 +1,87 @@
+import numpy as np
+import mindspore as ms
+
+from msprobe.core.common.const import Const as CoreCost
+
+
+class Const:
+    CELL = "cell"
+    API = "api"
+    KERNEL = "kernel"
+    TOOL_LEVEL_DICT = {
+        CoreCost.LEVEL_L0: CELL,
+        CoreCost.LEVEL_L1: API,
+        CoreCost.LEVEL_L2: KERNEL
+    }
+    PYNATIVE_MODE = "pynative"
+    GRAPH_GE_MODE = "graph_ge"
+    GRAPH_KBYK_MODE = "graph_kbyk"
+
+
+class FreeBenchmarkConst:
+    DEFAULT_DEVICE = "npu"
+    DEFAULT_STAGE = "forward"
+    DEFAULT_DUMP_LEVEL = CoreCost.LEVEL_L1
+    DEFAULT_PERT_TYPE = "improve_precision"
+    DEFAULT_HANDLER_TYPE = "check"
+    FIX_HANDLER_MODE = "fix"
+    ADD_NOISE = "add_noise"
+    BIT_NOISE = "bit_noise"
+    NO_CHANGE = "no_change"
+    IMPROVE_PRECISION = "improve_precision"
+    CHECK = "check"
+    FIX = "fix"
+    DEVICE_LIST = ["npu"]
+    STAGE_LIST = ["forward"]
+    DUMP_LEVEL_LIST = [CoreCost.LEVEL_L1]
+    PERT_TYPE_LIST = [IMPROVE_PRECISION, ADD_NOISE, BIT_NOISE, NO_CHANGE]
+    HANDLER_TYPE_LIST = [CHECK, FIX]
+    COMMUNICATION_API_LIST = [
+        "mindspore.communication.comm_func.all_gather_into_tensor",
+        "mindspore.communication.comm_func.gather_into_tensor",
+        "mindspore.communication.comm_func.all_reduce",
+        "mindspore.communication.comm_func.reduce",
+        "mindspore.communication.comm_func.reduce_scatter_tensor"
+        ]
+    NO_CHANGE_ERROR_THRESHOLD = 1.0
+    SYMBOL_FLIPPING_RATIO = 8.0
+    OPS_PREFIX = "mindspore.ops."
+    Tensor_PREFIX = "mindspore.Tensor."
+    MINT_PREFIX = "mindspore.mint."
+    MINT_NN_FUNC_PREFIX = "mindspore.mint.nn.functional."
+    COMM_PREFIX = "mindspore.communication.comm_func."
+
+    API_PREFIX_DICT = {
+        "ops": OPS_PREFIX,
+        "Tensor": Tensor_PREFIX,
+        "mint": MINT_PREFIX,
+        "mint.nn.functional": MINT_NN_FUNC_PREFIX,
+        "communication": COMM_PREFIX
+    }
+
+    PERT_VALUE_DICT = {
+        ms.bfloat16: 1e-4,
+        ms.float16: 1e-6,
+        ms.float32: 1e-8,
+        ms.float64: 1e-16
+    }
+
+    ERROR_THRESHOLD = {
+        ms.float16: 1.002,
+        ms.float32: 1.0002
+    }
+
+    PERT_BIT_DICT = {
+        ms.float16: np.int16,
+        ms.float32: np.int32,
+        ms.float64: np.int64
+    }
+
+    MS_NUMPY_DTYPE_DICT = {
+        ms.int16: np.int16,
+        ms.int32: np.int32,
+        ms.int64: np.int64,
+        ms.float16: np.float16,
+        ms.float32: np.float32,
+        ms.float64: np.float64
+    }

msprobe/mindspore/common/log.py

@@ -0,0 +1,38 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+import os
+import time
+import sys
+
+from msprobe.mindspore.common.utils import get_rank_if_initialized
+from msprobe.core.common.log import BaseLogger
+from msprobe.core.common.exceptions import DistributedNotInitializedError
+
+
+class MindsporeLogger(BaseLogger):
+    def __init__(self):
+        super().__init__()
+
+    def get_rank(self):
+        try:
+            current_rank = get_rank_if_initialized()
+        except DistributedNotInitializedError:
+            current_rank = None
+
+        return current_rank
+
+
+logger = MindsporeLogger()

msprobe/mindspore/common/utils.py

@@ -0,0 +1,57 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+import mindspore as ms
+
+from msprobe.core.common.exceptions import DistributedNotInitializedError
+from msprobe.core.common.file_check import path_len_exceeds_limit
+from msprobe.core.common.utils import save_npy
+from msprobe.core.common.log import logger
+
+
+def get_rank_if_initialized():
+    if ms.communication.GlobalComm.INITED:
+        return ms.communication.get_rank()
+    else:
+        raise DistributedNotInitializedError("mindspore distributed environment is not initialized")
+
+
+def convert_bf16_to_fp32(tensor):
+    if tensor.dtype == ms.bfloat16:
+        tensor = tensor.to(ms.float32)
+    return tensor
+
+
+def save_tensor_as_npy(tensor, file_path):
+    if not path_len_exceeds_limit(file_path):
+        tensor = convert_bf16_to_fp32(tensor)
+        saved_tensor = tensor.asnumpy()
+        save_npy(saved_tensor, file_path)
+    else:
+        logger.warning(f'The file path {file_path} length exceeds limit.')
+
+
+class MsprobeStep(ms.train.Callback):
+
+    def __init__(self, debugger):
+        super(MsprobeStep, self).__init__()
+        self.debugger = debugger
+
+    def on_train_step_begin(self, run_context):
+        self.debugger.start()
+
+    def on_train_step_end(self, run_context):
+        self.debugger.stop()
+        self.debugger.step()

msprobe/mindspore/compare/distributed_compare.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# Copyright (C) 2019-2024. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+import os
+from msprobe.core.common.utils import CompareException, check_compare_param, \
+    check_configuration_param, task_dumppath_get
+from msprobe.core.common.file_check import create_directory
+from msprobe.core.common.exceptions import FileCheckException
+from msprobe.core.common.log import logger
+from msprobe.mindspore.compare.ms_compare import MSComparator
+from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json
+from msprobe.mindspore.compare.ms_graph_compare import GraphMSComparator
+
+def ms_compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs):
+    if kwargs.get('suffix'):
+        logger.error("Argument 'suffix' is not supported for compare_distributed.")
+        raise CompareException(CompareException.INVALID_PARAM_ERROR)
+    stack_mode = kwargs.get('stack_mode', False)
+    auto_analyze = kwargs.get('auto_analyze', True)
+    fuzzy_match = kwargs.get('fuzzy_match', False)
+    # get the ranks and match by order
+    npu_ranks = sorted(check_and_return_dir_contents(npu_dump_dir, 'rank'))
+    bench_ranks = sorted(check_and_return_dir_contents(bench_dump_dir, 'rank'))
+    if len(npu_ranks) != len(bench_ranks):
+        logger.error('The number of ranks in the two runs are different. '
+                        'Unable to match the ranks. Please use another folder to compare '
+                        'or use compare() api and manually match the ranks.')
+        raise CompareException(CompareException.INVALID_PATH_ERROR)
+    for nr, br in zip(npu_ranks, bench_ranks):
+        npu_data_dir = os.path.join(npu_dump_dir, nr)
+        bench_data_dir = os.path.join(bench_dump_dir, br)
+        npu_path = extract_json(npu_data_dir, stack_json=False)
+        bench_path = extract_json(bench_data_dir, stack_json=False)
+        stack_path = extract_json(npu_data_dir, stack_json=True)
+
+        dump_result_param = {
+            'npu_json_path': npu_path,
+            'bench_json_path': bench_path,
+            'stack_json_path': stack_path,
+            'is_print_compare_log': True
+        }
+        try:
+            summary_compare, md5_compare = task_dumppath_get(dump_result_param)
+            check_configuration_param(stack_mode, auto_analyze, fuzzy_match)
+            create_directory(output_path)
+            check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare)
+        except (CompareException, FileCheckException) as error:
+            logger.error('Compare failed. Please check the arguments and do it again!')
+            raise CompareException(error.code) from error
+        ms_comparator = MSComparator()
+        ms_comparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare,
+                     md5_compare=md5_compare, **kwargs)
+
+
+def ms_graph_compare(inputs, outputs):
+    try:
+        create_directory(outputs)
+    except (CompareException, FileCheckException) as error:
+        logger.error('Compare failed. Please check the arguments and do it again!')
+        return
+    msComparator = GraphMSComparator(inputs, outputs)
+    msComparator.compare_core()

msprobe/mindspore/compare/ms_compare.py

@@ -0,0 +1,117 @@
+import os.path
+from msprobe.core.common.utils import check_compare_param, CompareException, check_configuration_param, \
+    task_dumppath_get, load_yaml, load_npy
+from msprobe.core.common.file_check import create_directory
+from msprobe.core.common.const import Const
+from msprobe.core.common.log import logger
+from msprobe.core.common.exceptions import FileCheckException
+from msprobe.core.compare.acc_compare import Comparator
+from msprobe.core.compare.check import check_struct_match, fuzzy_check_op
+
+
+class MSComparator(Comparator):
+    def __init__(self, cell_mapping=None, api_mapping=None):
+        self.frame_name = MSComparator.__name__
+        self.cell_mapping = cell_mapping
+        self.api_mapping = api_mapping
+        self.cross_frame = cell_mapping is not None or api_mapping is not None
+        self.cell_mapping_dict = self.load_mapping_file(self.cell_mapping)
+        self.api_mapping_dict = {}
+        if api_mapping is not None:
+            self.ms_to_pt_mapping = self.load_internal_api()
+            
+    def load_internal_api(self):
+        cur_path = os.path.dirname(os.path.realpath(__file__))
+        yaml_path = os.path.join(cur_path,"ms_to_pt_api.yaml")
+        return load_yaml(yaml_path)
+
+    def load_mapping_file(self, mapping_file):
+        if isinstance(mapping_file, str):
+            mapping_dict = load_yaml(mapping_file)
+        else:
+            mapping_dict = {}
+        return mapping_dict
+
+    def process_cell_mapping(self, npu_op_name):
+        npu_op_name = [op_name.replace("Cell", "Module", 1) for op_name in npu_op_name]
+        if self.cell_mapping_dict:
+            for index, op_name in enumerate(npu_op_name):
+                # get cell name & class name from op_name
+                # Cell.fc1.Dense.forward.0.input.0
+                cell_name = op_name.split(Const.SEP, 1)[-1].rsplit(Const.SEP, 4)[0]
+                if cell_name in self.cell_mapping_dict:
+                    npu_op_name[index] = op_name.replace(cell_name, self.cell_mapping_dict[cell_name], 1)
+        return npu_op_name
+
+    def check_op(self, npu_dict, bench_dict, fuzzy_match):
+        npu_op_name = npu_dict["op_name"].copy()
+        bench_op_name = bench_dict["op_name"].copy()
+   
+        if self.api_mapping is not None:
+            npu_op_name = self.process_api_mapping(npu_op_name, bench_op_name)
+        if self.cell_mapping is not None:
+            npu_op_name = self.process_cell_mapping(npu_op_name)
+
+        struct_match = check_struct_match(npu_dict, bench_dict, cross_frame=self.cross_frame)
+        if not fuzzy_match:
+            return npu_op_name == bench_op_name and struct_match
+        is_match = True
+        try:
+            is_match = fuzzy_check_op(npu_op_name, bench_op_name)
+        except Exception as err:
+            logger.warning("%s and %s can not fuzzy match." % (npu_op_name, bench_op_name))
+            is_match = False
+        return is_match and struct_match
+    
+    def read_npy_data(self, dir_path, file_name, load_pt_file=False):
+        data_path = os.path.join(dir_path, file_name)
+        if load_pt_file:
+            import torch
+            from msprobe.pytorch.common.utils import load_pt
+            data_value = load_pt(data_path).detach()
+            if data_value.dtype == torch.bfloat16:
+                data_value = data_value.to(torch.float32)
+            data_value = data_value.numpy()
+        else:
+            data_value = load_npy(data_path) 
+        return data_value    
+
+    def api_replace(self, npu_op_name, target, para):
+        for idx, _ in enumerate(npu_op_name):
+            npu_op_name[idx] = npu_op_name[idx].replace(target, para)
+        return npu_op_name
+    
+    def process_api_mapping(self, npu_op_name, bench_op_name):
+        # get api name & class name from op_name
+        # Functional.addcmul.0.forward.input.0
+        ms_api_name = npu_op_name[0].rsplit(Const.SEP, 4)[0]
+        pt_api_name = bench_op_name[0].rsplit(Const.SEP, 4)[0]
+        class_name = ms_api_name.split(Const.SEP)[0]
+        if class_name == "Mint":
+            return self.api_replace(npu_op_name, "Mint", "Torch")
+        elif class_name == "MintFunctional":
+            return self.api_replace(npu_op_name, "MintFunctional", "Functional")
+        elif self.ms_to_pt_mapping.get(ms_api_name) == pt_api_name:
+            return self.api_replace(npu_op_name, ms_api_name, pt_api_name)
+        else:
+            return npu_op_name
+        
+
+def ms_compare(input_param, output_path, **kwargs):
+    try:
+        stack_mode = kwargs.get('stack_mode', False)
+        auto_analyze = kwargs.get('auto_analyze', True)
+        fuzzy_match = kwargs.get('fuzzy_match', False)
+        cell_mapping = kwargs.get('cell_mapping', None)
+        api_mapping = kwargs.get('api_mapping', None)
+        summary_compare, md5_compare = task_dumppath_get(input_param)
+        check_configuration_param(stack_mode, auto_analyze, fuzzy_match)
+        create_directory(output_path)
+        check_compare_param(input_param, output_path, summary_compare, md5_compare)
+    except (CompareException, FileCheckException) as error:
+        logger.error('Compare failed. Please check the arguments and do it again!')
+        raise CompareException(error.code) from error
+    ms_comparator = MSComparator(cell_mapping, api_mapping)
+    ms_comparator.compare_core(input_param, output_path, stack_mode=stack_mode,
+                 auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare,
+                 md5_compare=md5_compare)