mindstudio-probe 1.0.4__py3-none-any.whl → 1.1.1__py3-none-any.whl

msprobe/mindspore/api_accuracy_checker/utils.py

@@ -1,8 +1,24 @@
-from msprobe.core.common.exceptions import ApiAccuracyCheckerException
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from msprobe.core.common.const import Const
+from msprobe.core.common.exceptions import ApiAccuracyCheckerException
 from msprobe.mindspore.api_accuracy_checker.type_mapping import float_dtype_str_list
 from msprobe.mindspore.common.log import logger
 
+
 def check_and_get_from_json_dict(dict_instance, key, key_description, accepted_type=None, accepted_value=None):
     '''
     Args:
@@ -22,30 +38,30 @@ def check_and_get_from_json_dict(dict_instance, key, key_description, accepted_t
         3. value is not accepted type
         4. value is not accepted value
     '''
-    parse_failed_exception = ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed)
     if not isinstance(dict_instance, dict):
-        logger.error_log_with_exp("check_and_get_from_json_dict failed: input is not a dict", parse_failed_exception)
+        error_info = "check_and_get_from_json_dict failed: input is not a dict"
+        raise ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed, error_info)
     value = dict_instance.get(key)
     if value is None:
-        logger.error_log_with_exp(f"check_and_get_from_json_dict failed: {key_description} is missing",
-                                  parse_failed_exception)
+        error_info = f"check_and_get_from_json_dict failed: {key_description} is missing"
+        raise ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed, error_info)
     elif accepted_type is not None and not isinstance(value, accepted_type):
-        logger.error_log_with_exp(
-            f"check_and_get_from_json_dict failed: {key_description} is not accepted type: {accepted_type}",
-            parse_failed_exception)
+        error_info = f"check_and_get_from_json_dict failed: {key_description} is not accepted type: {accepted_type}"
+        raise ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed, error_info)
     elif accepted_value is not None and value not in accepted_value:
-        logger.error_log_with_exp(
-            f"check_and_get_from_json_dict failed: {key_description} is not accepted value: {accepted_value}",
-            parse_failed_exception)
+        error_info = f"check_and_get_from_json_dict failed: {key_description} is not accepted value: {accepted_value}"
+        raise ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed, error_info)
     return value
 
-def convert_to_tuple(input):
-    if isinstance(input, (tuple, list)):
-        return tuple(input)
+
+def convert_to_tuple(args):
+    if isinstance(args, (tuple, list)):
+        return tuple(args)
     else:
-        input_list = [input]
+        input_list = [args]
         return tuple(input_list)
 
+
 def trim_output_compute_element_list(compute_element_list, forward_or_backward):
     '''
     Args:
@@ -55,12 +71,13 @@ def trim_output_compute_element_list(compute_element_list, forward_or_backward):
     trimmed_list = []
     for compute_element in compute_element_list:
         if compute_element.get_parameter() is None or \
-            (forward_or_backward == Const.BACKWARD and compute_element.get_dtype() not in float_dtype_str_list):
+                (forward_or_backward == Const.BACKWARD and compute_element.get_dtype() not in float_dtype_str_list):
             # trim case: 1. parameter is None. 2. backward output has non float parameter
             continue
         trimmed_list.append(compute_element)
     return trimmed_list
 
+
 class GlobalContext:
     def __init__(self):
         self.is_constructed = True
@@ -77,4 +94,4 @@ class GlobalContext:
         return self.is_constructed
 
 
-global_context = GlobalContext()
+global_context = GlobalContext()

msprobe/mindspore/cell_processor.py

@@ -1,17 +1,31 @@
-from msprobe.core.data_dump.scope import ModuleRangeScope
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from msprobe.core.data_dump.scope import ModuleRangeScope, MixRangeScope
 from msprobe.core.common.const import Const
-from msprobe.mindspore.common.log import logger
 
 
 class CellProcessor:
     cell_count = {}
+    cell_stack = []
+    api_parent_node = ""
+    module_node = {}
 
     def __init__(self, scope):
-        if isinstance(scope, ModuleRangeScope):
-            self.scope = scope
-        else:
-            self.scope = None
-        
+        self.scope = scope if isinstance(scope, (ModuleRangeScope, MixRangeScope)) else None
+
     @staticmethod
     def set_cell_count(cell_name):
         if cell_name not in CellProcessor.cell_count:
@@ -19,16 +33,47 @@ class CellProcessor:
         else:
             CellProcessor.cell_count[cell_name] += 1
         return CellProcessor.cell_count[cell_name]
-    
+
+    @classmethod
+    def reset_cell_stats(cls):
+        cls.cell_count = {}
+        cls.cell_stack = []
+        cls.api_parent_node = ""
+        cls.module_node = {}
+
     def node_hook(self, name_prefix, start_or_stop, **kwargs):
-        def begin_hook(cell, input):
-            index = self.set_cell_count(name_prefix)
-            cell.mindstudio_reserved_name = full_name = name_prefix + Const.SEP + str(index)
+        def begin_hook(cell, input_data):
+            full_name = self.set_and_get_reserved_name(cell, name_prefix, is_called_by_pre_hook=True)
+            if CellProcessor.cell_stack:
+                CellProcessor.module_node[full_name] = CellProcessor.cell_stack[-1]
+            else:
+                CellProcessor.module_node[full_name] = None
+
+            CellProcessor.cell_stack.append(full_name)
+            CellProcessor.api_parent_node = full_name
+
             if self.scope:
                 self.scope.begin_module(full_name)
-    
-        def end_hook(cell, input, output):
+
+        def end_hook(cell, input_data, output_data):
+            if CellProcessor.cell_stack:
+                CellProcessor.cell_stack.pop()
+            if CellProcessor.cell_stack:
+                CellProcessor.api_parent_node = CellProcessor.cell_stack[-1]
+            else:
+                CellProcessor.api_parent_node = None
+
             if self.scope:
                 self.scope.end_module(cell.mindstudio_reserved_name)
 
         return begin_hook if Const.START == start_or_stop else end_hook
+
+    def set_and_get_reserved_name(self, cell, cell_name, is_called_by_pre_hook=False):
+        if not is_called_by_pre_hook and hasattr(cell, 'has_pre_hook_called') and cell.has_pre_hook_called:
+            cell.has_pre_hook_called = False
+        else:
+            if is_called_by_pre_hook:
+                cell.has_pre_hook_called = True
+            index = self.set_cell_count(cell_name)
+            cell.mindstudio_reserved_name = cell_name + Const.SEP + str(index)
+        return cell.mindstudio_reserved_name

msprobe/mindspore/common/const.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import numpy as np
 import mindspore as ms
 
@@ -23,31 +38,35 @@ class Const:
     ASCEND_910A = "ascend910"
 
     OPS_PREFIX = "mindspore.ops."
-    Tensor_PREFIX = "mindspore.Tensor."
+    TENSOR_PREFIX = "mindspore.Tensor."
     MINT_PREFIX = "mindspore.mint."
     MINT_NN_FUNC_PREFIX = "mindspore.mint.nn.functional."
-    COMM_PREFIX = "mindspore.communication.comm_func."
-    COMMUNICATION_API_LIST = [
-        "mindspore.communication.comm_func.all_gather_into_tensor",
-        "mindspore.communication.comm_func.gather_into_tensor",
-        "mindspore.communication.comm_func.all_reduce",
-        "mindspore.communication.comm_func.reduce",
-        "mindspore.communication.comm_func.reduce_scatter_tensor"
-        ]
+
     TENSOR_DATA_PREFIX = "Tensor."
     STUB_TENSOR_DATA_PREFIX = "Tensor."
     OPS_DATA_PREFIX = "Functional."
     MINT_DATA_PREFIX = "Mint."
     MINT_NN_FUNC_DATA_PREFIX = "MintFunctional."
+    DISTRIBUTED_DATA_PREFIX = "Distributed."
 
     SUPPORTED_API_LIST_FILE = "support_wrap_ops.yaml"
     SUPPORTED_TENSOR_LIST_KEY = "tensor"
     SUPPORTED_OPS_LIST_KEY = "ops"
     SUPPORTED_MINT_LIST_KEY = "mint.ops"
     SUPPORTED__MINT_NN_FUNC_LIST_KEY = "mint.nn.functional"
+    SUPPORTED_COMM_LIST_KEY = "communication.comm_func"
 
     DROPOUT_API_NAME_PREFIX = "dropout"
 
+    GRAPH_DATA_MODE_LIST = [CoreConst.ALL, CoreConst.INPUT, CoreConst.OUTPUT]
+
+    HOOK_MS_PREFIX_DICT = {
+        OPS_DATA_PREFIX: OPS_PREFIX,
+        TENSOR_DATA_PREFIX: TENSOR_PREFIX,
+        MINT_DATA_PREFIX: MINT_PREFIX,
+        MINT_NN_FUNC_DATA_PREFIX: MINT_NN_FUNC_PREFIX
+    }
+
 
 class FreeBenchmarkConst:
     ADD_NOISE = "add_noise"
@@ -63,19 +82,21 @@ class FreeBenchmarkConst:
     DEFAULT_PERT_TYPE = IMPROVE_PRECISION
     DEFAULT_HANDLER_TYPE = CHECK
     DEVICE_LIST = [DEFAULT_DEVICE]
-    STAGE_LIST = [CoreConst.FORWARD]
+    STAGE_LIST = [CoreConst.FORWARD, CoreConst.BACKWARD]
     DUMP_LEVEL_LIST = [DEFAULT_DUMP_LEVEL]
     PERT_TYPE_LIST = [IMPROVE_PRECISION, ADD_NOISE, BIT_NOISE, NO_CHANGE, EXCHANGE_VALUE]
     HANDLER_TYPE_LIST = [CHECK, FIX]
     NO_CHANGE_ERROR_THRESHOLD = 1.0
     SYMBOL_FLIPPING_RATIO = 8.0
 
+    SUPPORTED_CHECK_API_FILE = "support_wrap_ops.yaml"
+    CHECK_RESULT_FILE = "free_benchmark.csv"
+
     API_PREFIX_DICT = {
         "ops": Const.OPS_PREFIX,
-        "Tensor": Const.Tensor_PREFIX,
+        "Tensor": Const.TENSOR_PREFIX,
         "mint": Const.MINT_PREFIX,
-        "mint.nn.functional": Const.MINT_NN_FUNC_PREFIX,
-        "communication": Const.COMM_PREFIX
+        "mint.nn.functional": Const.MINT_NN_FUNC_PREFIX
     }
 
     PERT_VALUE_DICT = {
@@ -86,6 +107,7 @@ class FreeBenchmarkConst:
     }
 
     ERROR_THRESHOLD = {
+        ms.bfloat16: 1.004,
         ms.float16: 1.002,
         ms.float32: 1.0002
     }

msprobe/mindspore/common/log.py

@@ -1,4 +1,5 @@
-# Copyright 2024 Huawei Technologies Co., Ltd
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,15 +12,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ============================================================================
 
-import os
-import time
-import sys
-
-from msprobe.mindspore.common.utils import get_rank_if_initialized
-from msprobe.core.common.log import BaseLogger
 from msprobe.core.common.exceptions import DistributedNotInitializedError
+from msprobe.core.common.log import BaseLogger
+from msprobe.mindspore.common.utils import get_rank_if_initialized
 
 
 class MindsporeLogger(BaseLogger):
@@ -35,4 +31,4 @@ class MindsporeLogger(BaseLogger):
         return current_rank
 
 
-logger = MindsporeLogger()
+logger = MindsporeLogger()

msprobe/mindspore/common/utils.py

@@ -1,4 +1,5 @@
-# Copyright 2024 Huawei Technologies Co., Ltd
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,13 +12,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ============================================================================
+
 import os
+import random
+
 import mindspore as ms
 
+from mindspore import ops
+from mindspore.mint import nn
 from msprobe.core.common.exceptions import DistributedNotInitializedError
 from msprobe.core.common.file_utils import path_len_exceeds_limit, check_path_exists, save_npy
 from msprobe.core.common.log import logger
+from msprobe.core.common.const import Const
+from msprobe.core.common.utils import CompareException, check_seed_all
 
 
 def get_rank_if_initialized():
@@ -36,7 +43,7 @@ def convert_bf16_to_fp32(tensor):
 def save_tensor_as_npy(tensor, file_path):
     if not path_len_exceeds_limit(file_path):
         tensor = convert_bf16_to_fp32(tensor)
-        saved_tensor = tensor.asnumpy()
+        saved_tensor = tensor.contiguous().asnumpy()
         save_npy(saved_tensor, file_path)
     else:
         logger.warning(f'The file path {file_path} length exceeds limit.')
@@ -53,12 +60,15 @@ def list_lowest_level_directories(root_dir):
     check_path_exists(root_dir)
     lowest_level_dirs = []
 
-    def recurse_dirs(current_dir):
+    def recurse_dirs(current_dir, depth=0):
+        if depth > Const.MAX_DEPTH:
+            logger.error(f'The directory {current_dir} has more than {Const.MAX_DEPTH} levels.')
+            raise CompareException(CompareException.RECURSION_LIMIT_ERROR)
         for entry in os.listdir(current_dir):
             full_path = os.path.join(current_dir, entry)
             if os.path.isdir(full_path):
                 if any(os.path.isdir(os.path.join(full_path, subentry)) for subentry in os.listdir(full_path)):
-                    recurse_dirs(full_path)
+                    recurse_dirs(full_path, depth=depth+1)
                 else:
                     lowest_level_dirs.append(full_path)
 
@@ -66,6 +76,16 @@ def list_lowest_level_directories(root_dir):
     return lowest_level_dirs
 
 
+def seed_all(seed=1234, mode=False, rm_dropout=True):
+    check_seed_all(seed, mode)
+    os.environ['PYTHONHASHSEED'] = str(seed)
+    ms.set_seed(seed)
+    random.seed(seed)
+    ms.set_context(deterministic="ON" if mode else "OFF")
+    os.environ['HCCL_DETERMINISTIC'] = str(mode)
+    if rm_dropout:
+        remove_dropout()
+
 
 class MsprobeStep(ms.train.Callback):
 
@@ -79,3 +99,38 @@ class MsprobeStep(ms.train.Callback):
     def on_train_step_end(self, run_context):
         self.debugger.stop()
         self.debugger.step()
+
+
+class Dropout(ops.Dropout):
+    def __init__(self, keep_prob=0.5, Seed0=0, Seed1=1):
+        super().__init__(1., Seed0, Seed1)
+
+
+class Dropout2D(ops.Dropout2D):
+    def __init__(self, keep_prob=0.5):
+        super().__init__(1.)
+
+
+class Dropout3D(ops.Dropout3D):
+    def __init__(self, keep_prob=0.5):
+        super().__init__(1.)
+
+
+class DropoutExt(nn.Dropout):
+    def __init__(self, p=0.5):
+        super().__init__(0)
+
+
+def dropout_ext(input_tensor, p=0.5, training=True):
+    return input_tensor
+
+
+def remove_dropout():
+    ops.Dropout = Dropout
+    ops.operations.Dropout = Dropout
+    ops.Dropout2D = Dropout2D
+    ops.operations.Dropout2D = Dropout2D
+    ops.Dropout3D = Dropout3D
+    ops.operations.Dropout3D = Dropout3D
+    nn.Dropout = DropoutExt
+    nn.functional.dropout = dropout_ext

msprobe/mindspore/compare/distributed_compare.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2019-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,31 +12,29 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
+
 import os
-from msprobe.core.common.utils import CompareException, check_compare_param, \
-    check_configuration_param, task_dumppath_get
+from msprobe.core.common.utils import CompareException
 from msprobe.core.common.file_utils import create_directory
 from msprobe.core.common.exceptions import FileCheckException
 from msprobe.mindspore.common.log import logger
-from msprobe.mindspore.compare.ms_compare import MSComparator
+from msprobe.mindspore.compare.ms_compare import ms_compare
 from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json
 from msprobe.mindspore.compare.ms_graph_compare import GraphMSComparator
 
+
 def ms_compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs):
     if kwargs.get('suffix'):
         logger.error("Argument 'suffix' is not supported for compare_distributed.")
         raise CompareException(CompareException.INVALID_PARAM_ERROR)
-    stack_mode = kwargs.get('stack_mode', False)
-    auto_analyze = kwargs.get('auto_analyze', True)
-    fuzzy_match = kwargs.get('fuzzy_match', False)
+    is_print_compare_log = kwargs.get('is_print_compare_log', True)
     # get the ranks and match by order
     npu_ranks = sorted(check_and_return_dir_contents(npu_dump_dir, 'rank'))
     bench_ranks = sorted(check_and_return_dir_contents(bench_dump_dir, 'rank'))
     if len(npu_ranks) != len(bench_ranks):
         logger.error('The number of ranks in the two runs are different. '
-                        'Unable to match the ranks. Please use another folder to compare '
-                        'or use compare() api and manually match the ranks.')
+                     'Unable to match the ranks. Please use another folder to compare '
+                     'or use compare() api and manually match the ranks.')
         raise CompareException(CompareException.INVALID_PATH_ERROR)
     for nr, br in zip(npu_ranks, bench_ranks):
         npu_data_dir = os.path.join(npu_dump_dir, nr)
@@ -50,19 +47,9 @@ def ms_compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs):
             'npu_json_path': npu_path,
             'bench_json_path': bench_path,
             'stack_json_path': stack_path,
-            'is_print_compare_log': True
+            'is_print_compare_log': is_print_compare_log
         }
-        try:
-            summary_compare, md5_compare = task_dumppath_get(dump_result_param)
-            check_configuration_param(stack_mode, auto_analyze, fuzzy_match)
-            create_directory(output_path)
-            check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare)
-        except (CompareException, FileCheckException) as error:
-            logger.error('Compare failed. Please check the arguments and do it again!')
-            raise CompareException(error.code) from error
-        ms_comparator = MSComparator()
-        ms_comparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare,
-                     md5_compare=md5_compare, **kwargs)
+        ms_compare(input_param=dump_result_param, output_path=output_path, suffix=f'_{nr}-{br}', **kwargs)
 
 
 def ms_graph_compare(inputs, outputs):
@@ -71,5 +58,5 @@ def ms_graph_compare(inputs, outputs):
     except (CompareException, FileCheckException) as error:
         logger.error('Compare failed. Please check the arguments and do it again!')
         return
-    msComparator = GraphMSComparator(inputs, outputs)
-    msComparator.compare_core()
+    ms_comparator = GraphMSComparator(inputs, outputs)
+    ms_comparator.compare_core()