mindstudio-probe 1.0.4__py3-none-any.whl → 1.1.1__py3-none-any.whl

msprobe/core/advisor/advisor.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2022-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
 
 import os
 
@@ -24,15 +22,17 @@ from msprobe.core.common.utils import CompareException
 from msprobe.core.common.file_utils import FileChecker
 from msprobe.core.common.const import Const, CompareConst, FileCheckConst
 
+
 class Advisor:
     """
     Class for generate advisor
     """
 
-    def __init__(self, input_data, out_path=""):
+    def __init__(self, input_data, out_path="", suffix=""):
         self.input_data = input_data
         self.out_path = os.path.realpath(out_path)
         self.file_type = None
+        self.suffix = suffix
 
     @staticmethod
     def deterministic_advisor(message, node_name):
@@ -62,7 +62,12 @@ class Advisor:
                             .format(item[CompareConst.NPU_NAME]))
 
     def gen_advisor_result(self, pd_data):
-        first_failing_data = pd_data.iloc[0]
+        try:
+            first_failing_data = pd_data.iloc[0]
+        except IndexError as e:
+            err_msg = "index out of bounds error occurs, pd_data is empty, please check!"
+            logger.error(err_msg)
+            raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e
         node_name = first_failing_data[CompareConst.NPU_NAME]
         index = first_failing_data['index']
         message = self.gen_advisor_message(node_name)
@@ -87,7 +92,7 @@ class Advisor:
         return message
 
     def analysis(self):
-        self._check_path_vaild()
+        self._check_path_valid()
         analyze_data = self._parse_input_data()
         logger.info("Start analyzing the comparison result: %s" % self.file_type)
         self.analyze_unmatched(analyze_data)
@@ -103,7 +108,7 @@ class Advisor:
         else:
             result = self.gen_advisor_result(failing_data)
         message_list = result.print_advisor_log()
-        result.gen_summary_file(self.out_path, message_list)
+        result.gen_summary_file(self.out_path, message_list, suffix=self.suffix)
 
     def _parse_input_data(self):
         data_columns = self.input_data.columns.values
@@ -119,6 +124,6 @@ class Advisor:
         df = self.input_data.reset_index()
         return df
 
-    def _check_path_vaild(self):
+    def _check_path_valid(self):
         out_path_checker = FileChecker(self.out_path, FileCheckConst.DIR, FileCheckConst.WRITE_ABLE)
         out_path_checker.common_check()

msprobe/core/advisor/advisor_const.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2022-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
 
 
 class AdvisorConst:
@@ -35,7 +33,8 @@ class AdvisorConst:
     # advisor message
     NO_ERR_SUGGEST = "All data in comparison result meets the accuracy requirements."
     FORWARD_INPUT_SUGGEST = "1. Analyze the model to view the input source.\n" \
-                            "2. Check whether an inplace API causes the output result to overwrite the input result. That is, the fault is actually caused by a computation error.\n" \
+                            "2. Check whether an inplace API causes the output result to overwrite the input result. "\
+                            "That is, the fault is actually caused by a computation error.\n" \
                             "3. The fault may be caused by memory corruption and further analysis is required."
     FORWARD_OUTPUT_SUGGEST = "This is a forward API computation error. Check the computation implementation."
     BACKWARD_INPUT_SUGGEST = "Check whether the forward computation result is affected."

msprobe/core/advisor/advisor_result.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2022-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,13 +12,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
 import os
 import time
 
 from msprobe.core.advisor.advisor_const import AdvisorConst
 from msprobe.core.common.log import logger
-from msprobe.core.common.const import Const, FileCheckConst
+from msprobe.core.common.const import FileCheckConst
 from msprobe.core.common.file_utils import change_mode, FileOpen
 
 
@@ -34,8 +32,8 @@ class AdvisorResult:
         self.advisor_message = message
 
     @staticmethod
-    def gen_summary_file(out_path, message_list):
-        file_name = 'advisor_{}.txt'.format(time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())))
+    def gen_summary_file(out_path, message_list, suffix):
+        file_name = 'advisor{}_{}.txt'.format(suffix, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())))
         result_file = os.path.join(out_path, file_name)
         try:
             with FileOpen(result_file, 'w+') as output_file:
@@ -50,9 +48,11 @@ class AdvisorResult:
 
     def print_advisor_log(self):
         logger.info("The summary of the expert advice is as follows: ")
-        message_list = [AdvisorConst.LINE + AdvisorConst.COLON + str(self.line),
-                        AdvisorConst.SUSPECT_NODES + AdvisorConst.COLON + self.suspect_node,
-                        AdvisorConst.ADVISOR_SUGGEST + AdvisorConst.COLON + self.advisor_message]
+        message_list = [
+            AdvisorConst.LINE + AdvisorConst.COLON + str(self.line),
+            AdvisorConst.SUSPECT_NODES + AdvisorConst.COLON + self.suspect_node,
+            AdvisorConst.ADVISOR_SUGGEST + AdvisorConst.COLON + self.advisor_message
+        ]
         for message in message_list:
             logger.info(message)
         return message_list

msprobe/core/common/const.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
 import stat
 
@@ -10,15 +25,20 @@ class Const:
     """
     TOOL_NAME = "msprobe"
 
+    ipv4_pattern = "([1-9]?\d|1\d{2}|2[0-4]\d|25[0-5])(\.([1-9]?\d|1\d{2}|2[0-4]\d|25[0-5])){3}$"
     SEP = "."
     REGEX_PREFIX_MAX_LENGTH = 20
     REGEX_PREFIX_PATTERN = r"^[a-zA-Z0-9_-]+$"
     FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$'
+    STRING_BLACKLIST = r"^[＋－＝％＠\+\-=%@]|;[＋－＝％＠\+\-=%@]"
     COMMA = ","
     FLOAT_EPSILON = np.finfo(float).eps
     OFF = 'OFF'
     BACKWARD = 'backward'
     FORWARD = 'forward'
+    PROGRESS_TIMEOUT = 3000
+    EXCEPTION_NONE = None
+    JIT = 'Jit'
     PRIMITIVE_PREFIX = 'Primitive'
     DEFAULT_LIST = []
     DEFAULT_PATH = './'
@@ -30,6 +50,7 @@ class Const:
     FOUR_SEGMENT = 4
     SIX_SEGMENT = 6
     SEVEN_SEGMENT = 7
+    MAX_DEPTH = 10
 
     # dump mode
     ALL = "all"
@@ -78,6 +99,8 @@ class Const:
     RUN_UT = "run_ut"
     GRAD_PROBE = "grad_probe"
     TASK_LIST = [TENSOR, STATISTICS, OVERFLOW_CHECK, FREE_BENCHMARK, RUN_UT, GRAD_PROBE]
+    DUMP_DATA_COLLECTION_LIST = [STATISTICS, TENSOR]
+    DUMP_DATA_MODE_LIST = [ALL, INPUT, OUTPUT, FORWARD, BACKWARD]
     LEVEL_L0 = "L0"
     LEVEL_L1 = "L1"
     LEVEL_L2 = "L2"
@@ -89,6 +112,7 @@ class Const:
     DATA = "data"
     PT_FRAMEWORK = "pytorch"
     MS_FRAMEWORK = "mindspore"
+    UNKNOWN_FRAMEWORK = "unknown"
     DIRECTORY_LENGTH = 4096
     FILE_NAME_LENGTH = 255
     FLOAT_TYPE = [np.half, np.single, float, np.double, np.float64, np.longdouble, np.float32, np.float16]
@@ -99,7 +123,37 @@ class Const:
     CPU_LOWERCASE = 'cpu'
     CUDA_LOWERCASE = 'cuda'
     DISTRIBUTED = 'Distributed'
+    DUMP_PREFIX = ["Distributed", "Functional", "Torch", "Tensor", "Mint", "MintFunctional", "Primitive", 
+                   "Aten", "VF", "NPU", "Jit"]
+
+    # struct json param
+    ORIGIN_DATA = "origin_data"
+    SCOPE = "scope"
+    STACK = "stack"
 
+    ATEN = "Aten"
+    MODULE_WHITE_LIST = ["torch", "numpy"]
+
+    FUNC_SKIP_LIST = ["construct", "__call__"]
+    FILE_SKIP_LIST = ["msprobe", "MindSpeed"]
+    DATA_TYPE_SKIP_LIST = ["Primitive", "Jit"]
+
+    STACK_FILE_INDEX = 0
+    STACK_FUNC_INDEX = 2
+    STACK_FUNC_ELE_INDEX = 1
+
+    SCOPE_ID_INDEX = -1
+    SCOPE_DIRECTION_INDEX = -2
+    TYPE_NAME_INDEX = -3
+    LAYER_NAME_INDEX = -4
+    API_TYPE_INDEX = 0
+    LEFT_MOVE_INDEX = -1
+    RIGHT_MOVE_INDEX = 1
+
+    TOP_LAYER = "TopLayer"
+    CELL = "Cell"
+    MODULE = "Module"
+    FRAME_FILE_LIST = ["site-packages/torch", "package/torch", "site-packages/mindspore", "package/mindspore"]
     INPLACE_LIST = [
         "broadcast", "all_reduce", "reduce", "all_gather", "gather", "scatter", "reduce_scatter",
         "_reduce_scatter_base", "_all_gather_base", "send", "recv", "irecv", "isend", "all_to_all_single", "all_to_all",
@@ -114,6 +168,31 @@ class Const:
         "int32_to_int64": ["cross_entropy"]
     }
 
+    FILL_CHAR_NUMS = 50
+    TOOL_ENDS_SUCCESSFULLY = f"{TOOL_NAME} ends successfully."
+    WITHOUT_CALL_STACK = "The call stack retrieval failed."
+
+    STEP = "step"
+    RANK = "rank"
+    HYPHEN = "-"
+    STEP_RANK_MINIMUM_VALUE = 0
+    STEP_RANK_MAXIMUM_VALUE = int(1e6)
+
+    # data type const
+    FLOAT16 = "Float16"
+    FLOAT32 = "Float32"
+    BFLOAT16 = "BFloat16"
+    TORCH_FLOAT16 = "torch.float16"
+    TORCH_FLOAT32 = "torch.float32"
+    TORCH_BFLOAT16 = "torch.bfloat16"
+
+    DTYPE = 'dtype'
+    SHAPE = 'shape'
+    MAX = 'Max'
+    MIN = 'Min'
+    MEAN = 'Mean'
+    NORM = 'Norm'
+
 
 class CompareConst:
     """
@@ -156,9 +235,17 @@ class CompareConst:
     RESULT = "Result"
     MAGNITUDE = 0.5
     OP_NAME = "op_name"
+    STRUCT = "struct"
     INPUT_STRUCT = "input_struct"
+    KWARGS_STRUCT = "kwargs_struct"
     OUTPUT_STRUCT = "output_struct"
     SUMMARY = "summary"
+    MAX_EXCEL_LENGTH = 1048576
+    YES = "Yes"
+    NO = "No"
+    STATISTICS_INDICATOR_NUM = 4
+    EPSILON = 1e-10
+    COMPARE_ENDS_SUCCESSFULLY = "msprobe compare ends successfully."
 
     COMPARE_RESULT_HEADER = [
         NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, COSINE, MAX_ABS_ERR, MAX_RELATIVE_ERR,
@@ -176,6 +263,12 @@ class CompareConst:
         NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, NPU_MD5, BENCH_MD5, RESULT
     ]
 
+    HEAD_OF_COMPARE_MODE = {
+        Const.ALL: COMPARE_RESULT_HEADER,
+        Const.SUMMARY: SUMMARY_COMPARE_RESULT_HEADER,
+        Const.MD5: MD5_COMPARE_RESULT_HEADER
+    }
+
     # compare standard
     HUNDRED_RATIO_THRESHOLD = 0.01
     THOUSAND_RATIO_THRESHOLD = 0.001
@@ -195,8 +288,12 @@ class CompareConst:
     PASS = 'pass'
     WARNING = 'Warning'
     ERROR = 'error'
+    TRUE = 'TRUE'
+    FALSE = 'FALSE'
     SKIP = 'SKIP'
     N_A = 'N/A'
+    INF = 'inf'
+    NEG_INF = '-inf'
     BFLOAT16_MIN = -3.3895313892515355e+38
     BFLOAT16_MAX = 3.3895313892515355e+38
     BFLOAT16_EPS = 3.90625e-3  # 2 ** -8
@@ -250,6 +347,13 @@ class CompareConst:
         MAX_DIFF: None, MIN_DIFF: None, MEAN_DIFF: None, NORM_DIFF: None, MAX_RELATIVE_ERR: None,
         MIN_RELATIVE_ERR: None, MEAN_RELATIVE_ERR: None, NORM_RELATIVE_ERR: None
     }
+    INPUT_PATTERN = Const.SEP + Const.INPUT + Const.SEP
+    KWARGS_PATTERN = Const.SEP + Const.KWARGS + Const.SEP
+    OUTPUT_PATTERN = Const.SEP + Const.OUTPUT + Const.SEP
+    COMPARE_KEY = 'compare_key'
+    COMPARE_SHAPE = 'compare_shape'
+    INTERNAL_API_MAPPING_FILE = 'ms_to_pt_api.yaml'
+    UNREADABLE = 'unreadable data'
 
 
 class FileCheckConst:
@@ -274,7 +378,8 @@ class FileCheckConst:
     MAX_JSON_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     MAX_PT_SIZE = 10737418240  # 10 * 1024 * 1024 * 1024
     MAX_CSV_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
-    MAX_YAML_SIZE = 1048576  # 10 * 1024 * 1024
+    MAX_YAML_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
+    COMMOM_FILE_SIZE = 1048576  # 1 * 1024 * 1024
     DIR = "dir"
     FILE = "file"
     DATA_DIR_AUTHORITY = 0o750
@@ -287,6 +392,7 @@ class FileCheckConst:
         CSV_SUFFIX: MAX_CSV_SIZE,
         YAML_SUFFIX: MAX_YAML_SIZE
     }
+    CSV_BLACK_LIST = r'^[＋－＝％＠\+\-=%@]|;[＋－＝％＠\+\-=%@]'
 
 
 class OverflowConst:
@@ -301,6 +407,9 @@ class MsCompareConst:
     # api_info field
     MINT = "Mint"
     MINT_FUNCTIONAL = "MintFunctional"
+    TENSOR_API = "Tensor"
+
+    API_NAME_STR_LENGTH = 4
 
     TASK_FIELD = "task"
     STATISTICS_TASK = "statistics"
@@ -308,6 +417,10 @@ class MsCompareConst:
     DUMP_DATA_DIR_FIELD = "dump_data_dir"
     DATA_FIELD = "data"
 
+    # supported api yaml
+    SUPPORTED_API_LIST_FILE = "checker_support_api.yaml"
+    SUPPORTED_TENSOR_LIST_KEY = "tensor"
+
     # detail_csv
     DETAIL_CSV_API_NAME = "API Name"
     DETAIL_CSV_BENCH_DTYPE = "Bench Dtype"
@@ -329,13 +442,61 @@ class MsgConst:
     """
     Class for log messages const
     """
-    CLEAR_SYMBOL = "\033[K"
     MSPROBE_LOG_LEVEL = "MSPROBE_LOG_LEVEL"
-    LEVEL = ["INFO", "WARNING", "ERROR", "DEBUG"]
+    LOG_LEVEL_ENUM = ["0", "1", "2", "3", "4"]
+    LOG_LEVEL = ["DEBUG", "INFO", "WARNING", "ERROR"]
+
+    class LogLevel:
+        class DEBUG:
+            value = 0
+
+        class INFO:
+            value = 1
+
+        class WARNING:
+            value = 2
+
+        class ERROR:
+            value = 3
+
     SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"]
 
+    NOT_CREATED_INSTANCE = "PrecisionDebugger instance is not created."
+
 
 class GraphMode:
     NPY_MODE = "NPY_MODE"
     STATISTIC_MODE = "STATISTIC_MODE"
     ERROR_MODE = "ERROR_MODE"
+
+
+class MonitorConst:
+    """
+    Class for monitor const
+    """
+    OP_LIST = ["min", "max", "norm", "zeros", "nans", "id", "mean"]
+    MONITOR_OUTPUT_DIR = "MONITOR_OUTPUT_DIR"
+    DEFAULT_MONITOR_OUTPUT_DIR = "./monitor_output"
+    DATABASE = "database"
+    EMAIL = "email"
+    OPT_TY = ['Megatron_DistributedOptimizer', 'Megatron_Float16OptimizerWithFloat16Params']
+    DEEPSPEED_OPT_TY = ("DeepSpeedZeroOptimizer_Stage0", "DeepSpeedZeroOptimizer_Stage1_or_2", "DeepSpeedZeroOptimizer_Stage3")
+    RULE_NAME = ['AnomalyTurbulence']
+
+    DOT = "."
+    VPP_SEP = ":"
+    ACTV_IN = "input"
+    ACTV_OUT = "output"
+    ACTVGRAD_IN = "input_grad"
+    ACTVGRAD_OUT = "output_grad"
+    POST_GRAD = "post_grad"
+    PRE_GRAD = "pre_grad"
+    PREFIX_POST = "post"
+    PREFIX_PRE = "pre"
+
+
+    ANOMALY_JSON = "anomaly.json"
+    ANALYSE_JSON = "anomaly_analyse.json"
+    TENSORBOARD = "tensorboard"
+    CSV = "csv"
+    API = "api"

msprobe/core/common/exceptions.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 class CodedException(Exception):
     def __init__(self, code, error_info=''):
         super().__init__()
@@ -11,10 +26,12 @@ class CodedException(Exception):
 class MsprobeException(CodedException):
     INVALID_PARAM_ERROR = 0
     OVERFLOW_NUMS_ERROR = 1
+    RECURSION_LIMIT_ERROR = 2
 
     err_strs = {
-        INVALID_PARAM_ERROR: "[msprobe] 无效参数： ",
-        OVERFLOW_NUMS_ERROR: "[msprobe] 超过预设溢出次数 当前溢出次数:"
+        INVALID_PARAM_ERROR: "[msprobe] 无效参数：",
+        OVERFLOW_NUMS_ERROR: "[msprobe] 超过预设溢出次数 当前溢出次数：",
+        RECURSION_LIMIT_ERROR: "[msprobe] 递归调用超过限制："
     }
 
 
@@ -41,7 +58,7 @@ class ParseJsonException(CodedException):
     InvalidDumpJson = 1
     err_strs = {
         UnexpectedNameStruct: "[msprobe] Unexpected name in json: ",
-        InvalidDumpJson: "[msprobe] json格式不正确: ",
+        InvalidDumpJson: "[msprobe] Invalid dump.json format: ",
     }
 
 
@@ -73,9 +90,13 @@ class StepException(CodedException):
 class FreeBenchmarkException(CodedException):
     UnsupportedType = 0
     InvalidGrad = 1
+    InvalidPerturbedOutput = 2
+    OutputIndexError = 3
     err_strs = {
         UnsupportedType: "[msprobe] Free benchmark get unsupported type: ",
         InvalidGrad: "[msprobe] Free benchmark gradient invalid: ",
+        InvalidPerturbedOutput: "[msprobe] Free benchmark invalid perturbed output: ",
+        OutputIndexError: "[msprobe] Free benchmark output index out of bounds: ",
     }
 
 
@@ -87,6 +108,7 @@ class DistributedNotInitializedError(Exception):
     def __str__(self):
         return self.msg
 
+
 class ApiAccuracyCheckerException(CodedException):
     ParseJsonFailed = 0
     UnsupportType = 1
@@ -97,4 +119,4 @@ class ApiAccuracyCheckerException(CodedException):
         UnsupportType: "[msprobe] Api Accuracy Checker get unsupported type: ",
         WrongValue: "[msprobe] Api Accuracy Checker get wrong value: ",
         ApiWrong: "[msprobe] Api Accuracy Checker something wrong with api: ",
-    }
+    }