mindstudio-probe 1.0.1__py3-none-any.whl → 1.0.4__py3-none-any.whl

msprobe/{config/config.json → config.json}

@@ -1,28 +1,50 @@
-{
-    "task": "statistics",
-    "dump_path": "./dump_path", 
-    "rank": [],  
-    "step": [],  
-    "level": "L1", 
-    "seed": 1234,
-    "is_deterministic": false, 
-    "enable_dataloader": false,
-    "acl_config": "",
-    "tensor": {
-        "scope": [], 
-        "list":[], 
-        "data_mode": ["all"], 
-        "backward_input": [],
-        "file_format": "npy" 
-    },
-    "statistics": {
-        "scope": [], 
-        "list":[], 
-        "data_mode": ["all"],
-        "summary_mode": "statistics"
-    },
-    "overflow_check": {
-        "overflow_nums": 1, 
-        "check_mode":"all" 
-    }
+{
+    "task": "statistics",
+    "dump_path": "./dump_path", 
+    "rank": [],  
+    "step": [],  
+    "level": "L1", 
+    "seed": 1234,
+    "is_deterministic": false, 
+    "enable_dataloader": false,
+    "acl_config": "",
+    "tensor": {
+        "scope": [], 
+        "list":[], 
+        "data_mode": ["all"], 
+        "backward_input": [],
+        "file_format": "npy" 
+    },
+    "statistics": {
+        "scope": [], 
+        "list":[], 
+        "data_mode": ["all"],
+        "summary_mode": "statistics"
+    },
+    "overflow_check": {
+        "overflow_nums": 1, 
+        "check_mode":"all" 
+    },
+    "run_ut": {
+        "white_list": [],
+        "black_list": [],
+        "error_data_path": "./"
+    },
+    "grad_probe": {
+        "grad_level": "L1",
+        "param_list": [],
+        "bounds": [-1, 0, 1]
+    },
+    "free_benchmark": {
+        "scope": [], 
+        "list": [],
+        "fuzz_device": "npu",
+        "pert_mode": "improve_precision",
+        "handler_type": "check",
+        "fuzz_level": "L1",
+        "fuzz_stage": "forward",
+        "if_preheat": false,
+        "preheat_step": 15,
+        "max_sample": 20
+    }
 }

msprobe/{pytorch → core}/advisor/advisor.py

@@ -1,124 +1,124 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-
-import os
-
-from msprobe.pytorch.advisor.advisor_result import AdvisorResult
-from msprobe.pytorch.advisor.advisor_const import AdvisorConst
-from msprobe.pytorch.common.log import logger
-from msprobe.core.common.utils import CompareException
-from msprobe.core.common.file_check import FileChecker
-from msprobe.core.common.const import Const, CompareConst, FileCheckConst
-
-class Advisor:
-    """
-    Class for generate advisor
-    """
-
-    def __init__(self, input_data, out_path=""):
-        self.input_data = input_data
-        self.out_path = os.path.realpath(out_path)
-        self.file_type = None
-
-    @staticmethod
-    def deterministic_advisor(message, node_name):
-        for api_name in AdvisorConst.NEED_DETERMINISTIC_API:
-            if api_name in node_name:
-                return AdvisorConst.DETERMINISTIC_SUGGEST
-        return message
-
-    @staticmethod
-    def batch_norm_advisor(message, node_name):
-        if AdvisorConst.FUNC_BATCH_NORM in node_name and AdvisorConst.FORWARD_INPUT_1 in node_name:
-            message = AdvisorConst.BATCH_NORM_SUGGEST
-        return message
-
-    def analyze_unmatched(self, analyze_data):
-        if self.file_type == Const.ALL:
-            accuracy_unmatched = analyze_data[
-                analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_UNMATCH]
-        else:
-            accuracy_unmatched = analyze_data[(analyze_data[CompareConst.NPU_SHAPE] == CompareConst.NAN) |
-                                              (analyze_data[CompareConst.BENCH_SHAPE] == CompareConst.NAN)]
-        num_unmatch = len(accuracy_unmatched)
-        if num_unmatch != 0:
-            for i in range(len(accuracy_unmatched)):
-                item = accuracy_unmatched.iloc[i]
-                logger.warning("The tensor name matches but the shape or dtype does not match: {}"
-                            .format(item[CompareConst.NPU_NAME]))
-
-    def gen_advisor_result(self, pd_data):
-        first_failing_data = pd_data.iloc[0]
-        node_name = first_failing_data[CompareConst.NPU_NAME]
-        index = first_failing_data['index']
-        message = self.gen_advisor_message(node_name)
-        logger.warning("Find %s accuracy not reached, the line is %s" % (node_name, index))
-        result = AdvisorResult(node_name, index, message)
-        return result
-
-    def gen_advisor_message(self, node_name):
-        if AdvisorConst.FORWARD in node_name:
-            if AdvisorConst.INPUT in node_name:
-                message = AdvisorConst.FORWARD_INPUT_SUGGEST
-            else:
-                message = AdvisorConst.FORWARD_OUTPUT_SUGGEST
-                message = self.deterministic_advisor(message, node_name)
-        else:
-            if AdvisorConst.INPUT in node_name:
-                message = AdvisorConst.BACKWARD_INPUT_SUGGEST
-            else:
-                message = AdvisorConst.BACKWARD_OUTPUT_SUGGEST
-                message = self.deterministic_advisor(message, node_name)
-        message = self.batch_norm_advisor(message, node_name)
-        return message
-
-    def analysis(self):
-        self._check_path_vaild()
-        analyze_data = self._parse_input_data()
-        logger.info("Start analyzing the comparison result: %s" % self.file_type)
-        self.analyze_unmatched(analyze_data)
-        if self.file_type == Const.ALL:
-            failing_data = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_NO]
-        elif self.file_type == Const.MD5:
-            failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.DIFF]
-        elif self.file_type == Const.SUMMARY:
-            failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.WARNING]
-        if failing_data.empty:
-            logger.info("All data from api input/output accuracy reached")
-            result = AdvisorResult(AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERR_SUGGEST)
-        else:
-            result = self.gen_advisor_result(failing_data)
-        message_list = result.print_advisor_log()
-        result.gen_summary_file(self.out_path, message_list)
-
-    def _parse_input_data(self):
-        data_columns = self.input_data.columns.values
-        if {CompareConst.ACCURACY, CompareConst.NPU_NAME}.issubset(data_columns):
-            self.file_type = Const.ALL
-        elif {CompareConst.RESULT, CompareConst.NPU_MD5}.issubset(data_columns):
-            self.file_type = Const.MD5
-        elif {CompareConst.MAX_DIFF, CompareConst.RESULT}.issubset(data_columns):
-            self.file_type = Const.SUMMARY
-        else:
-            logger.error('Compare result does not meet the required conditions.')
-            raise CompareException(CompareException.INVALID_DATA_ERROR)
-        df = self.input_data.reset_index()
-        return df
-
-    def _check_path_vaild(self):
-        out_path_checker = FileChecker(self.out_path, FileCheckConst.DIR, FileCheckConst.WRITE_ABLE)
-        out_path_checker.common_check()
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+import os
+
+from msprobe.core.advisor.advisor_result import AdvisorResult
+from msprobe.core.advisor.advisor_const import AdvisorConst
+from msprobe.core.common.log import logger
+from msprobe.core.common.utils import CompareException
+from msprobe.core.common.file_utils import FileChecker
+from msprobe.core.common.const import Const, CompareConst, FileCheckConst
+
+class Advisor:
+    """
+    Class for generate advisor
+    """
+
+    def __init__(self, input_data, out_path=""):
+        self.input_data = input_data
+        self.out_path = os.path.realpath(out_path)
+        self.file_type = None
+
+    @staticmethod
+    def deterministic_advisor(message, node_name):
+        for api_name in AdvisorConst.NEED_DETERMINISTIC_API:
+            if api_name in node_name:
+                return AdvisorConst.DETERMINISTIC_SUGGEST
+        return message
+
+    @staticmethod
+    def batch_norm_advisor(message, node_name):
+        if AdvisorConst.FUNC_BATCH_NORM in node_name and AdvisorConst.FORWARD_INPUT_1 in node_name:
+            message = AdvisorConst.BATCH_NORM_SUGGEST
+        return message
+
+    def analyze_unmatched(self, analyze_data):
+        if self.file_type == Const.ALL:
+            accuracy_unmatched = analyze_data[
+                analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_UNMATCH]
+        else:
+            accuracy_unmatched = analyze_data[(analyze_data[CompareConst.NPU_SHAPE] == CompareConst.NAN) |
+                                              (analyze_data[CompareConst.BENCH_SHAPE] == CompareConst.NAN)]
+        num_unmatch = len(accuracy_unmatched)
+        if num_unmatch != 0:
+            for i in range(len(accuracy_unmatched)):
+                item = accuracy_unmatched.iloc[i]
+                logger.warning("The tensor name matches but the shape or dtype does not match: {}"
+                            .format(item[CompareConst.NPU_NAME]))
+
+    def gen_advisor_result(self, pd_data):
+        first_failing_data = pd_data.iloc[0]
+        node_name = first_failing_data[CompareConst.NPU_NAME]
+        index = first_failing_data['index']
+        message = self.gen_advisor_message(node_name)
+        logger.warning("Find %s accuracy not reached, the line is %s" % (node_name, index))
+        result = AdvisorResult(node_name, index, message)
+        return result
+
+    def gen_advisor_message(self, node_name):
+        if AdvisorConst.FORWARD in node_name:
+            if AdvisorConst.INPUT in node_name:
+                message = AdvisorConst.FORWARD_INPUT_SUGGEST
+            else:
+                message = AdvisorConst.FORWARD_OUTPUT_SUGGEST
+                message = self.deterministic_advisor(message, node_name)
+        else:
+            if AdvisorConst.INPUT in node_name:
+                message = AdvisorConst.BACKWARD_INPUT_SUGGEST
+            else:
+                message = AdvisorConst.BACKWARD_OUTPUT_SUGGEST
+                message = self.deterministic_advisor(message, node_name)
+        message = self.batch_norm_advisor(message, node_name)
+        return message
+
+    def analysis(self):
+        self._check_path_vaild()
+        analyze_data = self._parse_input_data()
+        logger.info("Start analyzing the comparison result: %s" % self.file_type)
+        self.analyze_unmatched(analyze_data)
+        if self.file_type == Const.ALL:
+            failing_data = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_NO]
+        elif self.file_type == Const.MD5:
+            failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.DIFF]
+        elif self.file_type == Const.SUMMARY:
+            failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.WARNING]
+        if failing_data.empty:
+            logger.info("All data from api input/output accuracy reached")
+            result = AdvisorResult(AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERR_SUGGEST)
+        else:
+            result = self.gen_advisor_result(failing_data)
+        message_list = result.print_advisor_log()
+        result.gen_summary_file(self.out_path, message_list)
+
+    def _parse_input_data(self):
+        data_columns = self.input_data.columns.values
+        if {CompareConst.ACCURACY, CompareConst.NPU_NAME}.issubset(data_columns):
+            self.file_type = Const.ALL
+        elif {CompareConst.RESULT, CompareConst.NPU_MD5}.issubset(data_columns):
+            self.file_type = Const.MD5
+        elif {CompareConst.MAX_DIFF, CompareConst.RESULT}.issubset(data_columns):
+            self.file_type = Const.SUMMARY
+        else:
+            logger.error('Compare result does not meet the required conditions.')
+            raise CompareException(CompareException.INVALID_DATA_ERROR)
+        df = self.input_data.reset_index()
+        return df
+
+    def _check_path_vaild(self):
+        out_path_checker = FileChecker(self.out_path, FileCheckConst.DIR, FileCheckConst.WRITE_ABLE)
+        out_path_checker.common_check()

msprobe/{pytorch → core}/advisor/advisor_const.py

@@ -1,59 +1,59 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-
-
-class AdvisorConst:
-    """
-    Class for advisor const
-    """
-
-    # text symbol
-    NEW_LINE = "\n"
-    COLON = ": "
-
-    # advisor summary key
-    SUSPECT_NODES = "Suspect Nodes"
-    LINE = "Line"
-    ADVISOR_SUGGEST = "Expert Advice"
-
-    NO_ERROR_API = "NA"
-
-    # advisor message
-    NO_ERR_SUGGEST = "All data in comparison result meets the accuracy requirements."
-    FORWARD_INPUT_SUGGEST = "1. Analyze the model to view the input source.\n" \
-                            "2. Check whether an inplace API causes the output result to overwrite the input result. That is, the fault is actually caused by a computation error.\n" \
-                            "3. The fault may be caused by memory corruption and further analysis is required."
-    FORWARD_OUTPUT_SUGGEST = "This is a forward API computation error. Check the computation implementation."
-    BACKWARD_INPUT_SUGGEST = "Check whether the forward computation result is affected."
-    BACKWARD_OUTPUT_SUGGEST = "This is a backward API computation error. Check the computation implementation."
-    BATCH_NORM_SUGGEST = "Torch API batch_norm input not fixed, the following suggestions may fix it:\n" \
-                         "1. If use torch.nn.functional.batch_norm, you can set parameter training=False.\n" \
-                         "2. If use torch.nn.BatchNormXXX, you can set parameter affine=False.\n" \
-                         "3. Use seed_all(mode=True) to enable deterministic computing."
-    DETERMINISTIC_SUGGEST = "This torch api may be uncertainty in the calculation, " \
-                            "can seed_all(mode=True) to enable deterministic computing."
-
-    FUNC_BATCH_NORM = "Functional_batch_norm"
-    FORWARD_INPUT_1 = "forward_input.1"
-    NEED_DETERMINISTIC_API = ["conv2d", "conv3d", "matmul", "nll_loss", "layer_norm", "lstm"]
-    BATCH_NORM = "batch_norm"
-
-    # name keyword
-    INPUT = "input"
-    OUTPUT = "output"
-    FORWARD = "forward"
-    BACKWARD = "backward"
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+
+class AdvisorConst:
+    """
+    Class for advisor const
+    """
+
+    # text symbol
+    NEW_LINE = "\n"
+    COLON = ": "
+
+    # advisor summary key
+    SUSPECT_NODES = "Suspect Nodes"
+    LINE = "Line"
+    ADVISOR_SUGGEST = "Expert Advice"
+
+    NO_ERROR_API = "NA"
+
+    # advisor message
+    NO_ERR_SUGGEST = "All data in comparison result meets the accuracy requirements."
+    FORWARD_INPUT_SUGGEST = "1. Analyze the model to view the input source.\n" \
+                            "2. Check whether an inplace API causes the output result to overwrite the input result. That is, the fault is actually caused by a computation error.\n" \
+                            "3. The fault may be caused by memory corruption and further analysis is required."
+    FORWARD_OUTPUT_SUGGEST = "This is a forward API computation error. Check the computation implementation."
+    BACKWARD_INPUT_SUGGEST = "Check whether the forward computation result is affected."
+    BACKWARD_OUTPUT_SUGGEST = "This is a backward API computation error. Check the computation implementation."
+    BATCH_NORM_SUGGEST = "Torch API batch_norm input not fixed, the following suggestions may fix it:\n" \
+                         "1. If use torch.nn.functional.batch_norm, you can set parameter training=False.\n" \
+                         "2. If use torch.nn.BatchNormXXX, you can set parameter affine=False.\n" \
+                         "3. Use seed_all(mode=True) to enable deterministic computing."
+    DETERMINISTIC_SUGGEST = "This torch api may be uncertainty in the calculation, " \
+                            "can seed_all(mode=True) to enable deterministic computing."
+
+    FUNC_BATCH_NORM = "Functional_batch_norm"
+    FORWARD_INPUT_1 = "forward_input.1"
+    NEED_DETERMINISTIC_API = ["conv2d", "conv3d", "matmul", "nll_loss", "layer_norm", "lstm"]
+    BATCH_NORM = "batch_norm"
+
+    # name keyword
+    INPUT = "input"
+    OUTPUT = "output"
+    FORWARD = "forward"
+    BACKWARD = "backward"

msprobe/{pytorch → core}/advisor/advisor_result.py

@@ -1,58 +1,58 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-import os
-import time
-
-from msprobe.pytorch.advisor.advisor_const import AdvisorConst
-from msprobe.pytorch.common.log import logger
-from msprobe.core.common.const import Const, FileCheckConst
-from msprobe.core.common.file_check import change_mode
-
-
-class AdvisorResult:
-    """
-    Class for generate advisor result
-    """
-
-    def __init__(self, node, line, message):
-        self.suspect_node = node
-        self.line = line
-        self.advisor_message = message
-
-    @staticmethod
-    def gen_summary_file(out_path, message_list):
-        file_name = 'advisor_{}.txt'.format(time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())))
-        result_file = os.path.join(out_path, file_name)
-        try:
-            with os.fdopen(os.open(result_file, Const.WRITE_FLAGS, Const.WRITE_MODES), 'w+') as output_file:
-                output_file.truncate(0)
-                message_list = [message + AdvisorConst.NEW_LINE for message in message_list]
-                output_file.writelines(message_list)
-            change_mode(result_file, FileCheckConst.DATA_FILE_AUTHORITY)
-        except IOError as io_error:
-            logger.error("Failed to save %s, the reason is %s." % (result_file, io_error))
-        else:
-            logger.info("The advisor summary is saved in: %s" % result_file)
-
-    def print_advisor_log(self):
-        logger.info("The summary of the expert advice is as follows: ")
-        message_list = [AdvisorConst.LINE + AdvisorConst.COLON + str(self.line),
-                        AdvisorConst.SUSPECT_NODES + AdvisorConst.COLON + self.suspect_node,
-                        AdvisorConst.ADVISOR_SUGGEST + AdvisorConst.COLON + self.advisor_message]
-        for message in message_list:
-            logger.info(message)
-        return message_list
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+import os
+import time
+
+from msprobe.core.advisor.advisor_const import AdvisorConst
+from msprobe.core.common.log import logger
+from msprobe.core.common.const import Const, FileCheckConst
+from msprobe.core.common.file_utils import change_mode, FileOpen
+
+
+class AdvisorResult:
+    """
+    Class for generate advisor result
+    """
+
+    def __init__(self, node, line, message):
+        self.suspect_node = node
+        self.line = line
+        self.advisor_message = message
+
+    @staticmethod
+    def gen_summary_file(out_path, message_list):
+        file_name = 'advisor_{}.txt'.format(time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())))
+        result_file = os.path.join(out_path, file_name)
+        try:
+            with FileOpen(result_file, 'w+') as output_file:
+                output_file.truncate(0)
+                message_list = [message + AdvisorConst.NEW_LINE for message in message_list]
+                output_file.writelines(message_list)
+            change_mode(result_file, FileCheckConst.DATA_FILE_AUTHORITY)
+        except IOError as io_error:
+            logger.error("Failed to save %s, the reason is %s." % (result_file, io_error))
+        else:
+            logger.info("The advisor summary is saved in: %s" % result_file)
+
+    def print_advisor_log(self):
+        logger.info("The summary of the expert advice is as follows: ")
+        message_list = [AdvisorConst.LINE + AdvisorConst.COLON + str(self.line),
+                        AdvisorConst.SUSPECT_NODES + AdvisorConst.COLON + self.suspect_node,
+                        AdvisorConst.ADVISOR_SUGGEST + AdvisorConst.COLON + self.advisor_message]
+        for message in message_list:
+            logger.info(message)
+        return message_list