mindstudio-probe 1.0.4__py3-none-any.whl → 1.1.0__py3-none-any.whl

msprobe/pytorch/common/__init__.py

@@ -1,2 +1,17 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from .parse_json import parse_json_info_forward_backward
 from .utils import seed_all

msprobe/pytorch/common/log.py

@@ -1,9 +1,21 @@
-import os
-import time
-import sys
-from msprobe.pytorch.common.utils import get_rank_if_initialized
-from msprobe.core.common.log import BaseLogger
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from msprobe.core.common.exceptions import DistributedNotInitializedError
+from msprobe.core.common.log import BaseLogger
+from msprobe.pytorch.common.utils import get_rank_if_initialized
 
 
 class PyTorchLogger(BaseLogger):
@@ -18,4 +30,4 @@ class PyTorchLogger(BaseLogger):
         return current_rank
 
 
-logger = PyTorchLogger()
+logger = PyTorchLogger()

msprobe/pytorch/common/parse_json.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import json
 
 from msprobe.core.common.exceptions import ParseJsonException
@@ -5,14 +20,6 @@ from msprobe.core.common.file_utils import FileOpen
 
 
 def parse_json_info_forward_backward(json_path):
-    def parse_data_name_with_pattern(data_name, pattern):
-        name_struct = data_name.split('.')
-        if not name_struct[-1] == pattern:
-            raise ParseJsonException(ParseJsonException.UnexpectedNameStruct,
-                f"{data_name} in file {json_path}")
-        api_name = '.'.join(name_struct[:-1])
-        return api_name
-
     with FileOpen(json_path, 'r') as f:
         dump_json = json.load(f)
 
@@ -27,13 +34,21 @@ def parse_json_info_forward_backward(json_path):
         if "Module" in data_name:
             continue
         if "forward" in data_name:
-            api_name = parse_data_name_with_pattern(data_name, "forward")
+            api_name = parse_data_name_with_pattern(data_name, "forward", json_path)
             forward_data.update({api_name: data_item})
         elif "backward" in data_name:
-            api_name = parse_data_name_with_pattern(data_name, "backward")
+            api_name = parse_data_name_with_pattern(data_name, "backward", json_path)
             backward_data.update({api_name: data_item})
         else:
             raise ParseJsonException(ParseJsonException.UnexpectedNameStruct,
-                f"{data_name} in file {json_path}.")
+                                     f"{data_name} in file {json_path}.")
 
     return forward_data, backward_data, real_data_path
+
+
+def parse_data_name_with_pattern(data_name, pattern, json_path):
+    name_struct = data_name.split('.')
+    if not name_struct[-1] == pattern:
+        raise ParseJsonException(ParseJsonException.UnexpectedNameStruct, f"{data_name} in file {json_path}")
+    api_name = '.'.join(name_struct[:-1])
+    return api_name

msprobe/pytorch/common/utils.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,20 +12,22 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
+
 import io
 import os
 import random
 import stat
+from functools import wraps
+
+import numpy as np
 import torch
 import torch.distributed as dist
-import numpy as np
-from functools import wraps
 from msprobe.core.common.exceptions import DistributedNotInitializedError
-from msprobe.core.common.log import logger
 from msprobe.core.common.file_utils import (FileCheckConst, change_mode,
                                             check_file_or_directory_path, check_path_before_create)
-
+from msprobe.core.common.log import logger
+from msprobe.core.common.utils import check_seed_all
+from packaging import version
 
 try:
     import torch_npu
@@ -35,10 +36,8 @@ except ImportError:
 else:
     is_gpu = False
 
-
 torch_without_guard_version = torch.__version__ >= '2.1'
 
-
 if not is_gpu and not torch_without_guard_version:
     from torch_npu.utils.device_guard import torch_device_guard as torch_npu_device_guard
 
@@ -46,7 +45,6 @@ npu_distributed_api = ['isend', 'irecv']
 
 
 def parameter_adapter(func):
-
     def handle_masked_select(input_tensor, indices):
         masked_select_func = getattr(torch._C._VariableFunctionsClass, "masked_select")
         if input_tensor.dtype == torch.bfloat16:
@@ -80,17 +78,19 @@ def parameter_adapter(func):
         if self.op_name_ == "__eq__" and args[1] is None:
             return False
         return func(self, *args, **kwargs)
+
     return inner
 
 
 def torch_device_guard(func):
     if is_gpu or torch_without_guard_version:
         return func
-    # Parse args/kwargs matched torch.device objects
 
+    # Parse args/kwargs matched torch.device objects
     @torch_npu_device_guard
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
+
     return wrapper
 
 
@@ -105,20 +105,28 @@ def get_rank_if_initialized():
 
 
 def seed_all(seed=1234, mode=False):
-    random.seed(seed)
-    os.environ['PYTHONHASHSEED'] = str(seed)
-    np.random.seed(seed)
-    torch.manual_seed(seed)
-    torch.use_deterministic_algorithms(mode)
-    if is_gpu:
-        torch.cuda.manual_seed_all(seed)
-        torch.cuda.manual_seed(seed)
-        torch.backends.cudnn.deterministic = True
-        torch.backends.cudnn.enable = False
-        torch.backends.cudnn.benchmark = False
-    else:
-        torch_npu.npu.manual_seed_all(seed)
-        torch_npu.npu.manual_seed(seed)
+    check_seed_all(seed, mode)
+    try:
+        random.seed(seed)
+        os.environ['PYTHONHASHSEED'] = str(seed)
+        np.random.seed(seed)
+        torch.manual_seed(seed)
+        cuda_version = torch.version.cuda
+        if cuda_version is not None and version.parse(cuda_version) >= version.parse("10.2"):
+            os.environ['CUBLAS_WORKSPACE_CONFIG'] = ':4096:8'
+        os.environ['HCCL_DETERMINISTIC'] = str(mode)
+        torch.use_deterministic_algorithms(mode)
+        if is_gpu:
+            torch.cuda.manual_seed_all(seed)
+            torch.cuda.manual_seed(seed)
+            torch.backends.cudnn.deterministic = True
+            torch.backends.cudnn.enable = False
+            torch.backends.cudnn.benchmark = False
+        else:
+            torch_npu.npu.manual_seed_all(seed)
+            torch_npu.npu.manual_seed(seed)
+    except Exception as e:
+        logger.error(f"There is an unexpected error while determinating randomness. {e}")
 
 
 class Const:
@@ -191,10 +199,7 @@ class Const:
     ENV_ENABLE = "1"
     ENV_DISABLE = "0"
 
-    MAX_SEED_VALUE = 2**32 - 1
-
-    INPLACE_LIST = ["broadcast", "all_reduce", "reduce", "all_gather", "gather", "scatter", "reduce_scatter",
-                    "_reduce_scatter_base", "_all_gather_base", "all_to_all_single"]
+    MAX_SEED_VALUE = 2 ** 32 - 1
 
     TASK_LIST = ["tensor", "statistics", "overflow_check", "free_benchmark"]
     LEVEL_LIST = ["L0", "L1", "L2", "mix"]
@@ -257,7 +262,7 @@ def print_rank_0(message):
             logger.info(message)
     else:
         logger.info(message)
-        
+
 
 def load_pt(pt_path, to_cpu=False):
     pt_path = os.path.realpath(pt_path)
@@ -279,8 +284,8 @@ def save_pt(tensor, filepath):
         torch.save(tensor, filepath)
     except Exception as e:
         logger.error("Save pt file failed, please check according possible error causes: "
-                            "1. out of disk space or disk error, "
-                            "2. no permission to write files, etc.")
+                     "1. out of disk space or disk error, "
+                     "2. no permission to write files, etc.")
         raise RuntimeError(f"save pt file {filepath} failed") from e
     change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY)
 

msprobe/pytorch/compare/distributed_compare.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2019-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2019-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,14 +12,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
+
 import os
 from msprobe.core.common.utils import CompareException, check_compare_param, \
     check_configuration_param, task_dumppath_get
 from msprobe.core.common.file_utils import create_directory
 from msprobe.core.common.exceptions import FileCheckException
 from msprobe.pytorch.common.log import logger
-from msprobe.core.common.const import Const
 from msprobe.pytorch.compare.pt_compare import PTComparator
 from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json
 
@@ -55,12 +53,14 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs):
         }
         try:
             summary_compare, md5_compare = task_dumppath_get(dump_result_param)
-            check_configuration_param(stack_mode, auto_analyze, fuzzy_match)
+            check_configuration_param(stack_mode, auto_analyze, fuzzy_match,
+                                      dump_result_param.get('is_print_compare_log', True))
             create_directory(output_path)
-            check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare)
+            check_compare_param(dump_result_param, output_path,
+                                summary_compare=summary_compare, md5_compare=md5_compare)
         except (CompareException, FileCheckException) as error:
             logger.error('Compare failed. Please check the arguments and do it again!')
             raise CompareException(error.code) from error
         pt_comparator = PTComparator()
-        pt_comparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare,
-                     md5_compare=md5_compare, **kwargs)
+        pt_comparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}',
+                                   summary_compare=summary_compare, md5_compare=md5_compare, **kwargs)

msprobe/pytorch/compare/match.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
 from msprobe.core.common.utils import CompareException
 from msprobe.core.common.file_utils import load_yaml

msprobe/pytorch/compare/pt_compare.py

@@ -1,17 +1,48 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os.path
 import torch
 from msprobe.core.common.const import FileCheckConst
 from msprobe.pytorch.common.log import logger
 from msprobe.core.common.exceptions import FileCheckException
 from msprobe.core.compare.acc_compare import Comparator
-from msprobe.core.common.utils import check_configuration_param, task_dumppath_get, check_compare_param, CompareException
-from msprobe.core.common.file_utils import FileChecker, create_directory
+from msprobe.core.common.utils import check_configuration_param, task_dumppath_get, check_compare_param, \
+    CompareException
+from msprobe.core.common.file_utils import FileChecker, create_directory, load_yaml
 from msprobe.pytorch.common.utils import load_pt
 
 
 class PTComparator (Comparator):
-    def __init__(self):
+    def __init__(self, data_mapping=None):
         self.frame_name = PTComparator.__name__
+        self.data_mapping = data_mapping
+        if isinstance(self.data_mapping, str) or self.data_mapping is None:
+            self.data_mapping_dict = self.load_mapping_file(self.data_mapping)
+        elif isinstance(self.data_mapping, dict):
+            self.data_mapping_dict = self.data_mapping
+        else:
+            raise TypeError(f"The type of parameter `data_mapping` must be dict, str or None, but got "
+                            f"{type(self.data_mapping)}")
+
+    def load_mapping_file(self, mapping_file):
+        if isinstance(mapping_file, str):
+            mapping_dict = load_yaml(mapping_file)
+        else:
+            mapping_dict = {}
+        return mapping_dict
     
     def read_npy_data(self, dir_path, file_name):
         data_path = os.path.join(dir_path, file_name)
@@ -35,16 +66,17 @@ class PTComparator (Comparator):
         return data_value  
     
     
-def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False):
+def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False, **kwargs):
     try:
         summary_compare, md5_compare = task_dumppath_get(input_param)
-        check_configuration_param(stack_mode, auto_analyze, fuzzy_match)
+        check_configuration_param(stack_mode, auto_analyze, fuzzy_match, input_param.get('is_print_compare_log', True))
         create_directory(output_path)
         check_compare_param(input_param, output_path, summary_compare, md5_compare)
+        data_mapping = kwargs.get('data_mapping', None)
     except (CompareException, FileCheckException) as error:
         logger.error('Compare failed. Please check the arguments and do it again!')
         raise CompareException(error.code) from error
-    pt_comparator = PTComparator()
+    pt_comparator = PTComparator(data_mapping)
     pt_comparator.compare_core(input_param, output_path, stack_mode=stack_mode,
                  auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare,
                  md5_compare=md5_compare)

msprobe/pytorch/debugger/debugger_config.py

@@ -1,6 +1,23 @@
-from msprobe.pytorch.common import seed_all
-from msprobe.pytorch.common.log import logger
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
 from msprobe.core.common.const import Const
+from msprobe.core.common.exceptions import MsprobeException
+from msprobe.pytorch.common.log import logger
 
 
 class DebuggerConfig:
@@ -10,8 +27,6 @@ class DebuggerConfig:
         self.rank = common_config.rank if common_config.rank else []
         self.step = common_config.step if common_config.step else []
         self.level = level or common_config.level or "L1"
-        self.seed = common_config.seed if common_config.seed else 1234
-        self.is_deterministic = common_config.is_deterministic
         self.enable_dataloader = common_config.enable_dataloader
         self.scope = task_config.scope if task_config.scope else []
         self.list = task_config.list if task_config.list else []
@@ -25,15 +40,15 @@ class DebuggerConfig:
         self.framework = Const.PT_FRAMEWORK
 
         if self.task == Const.FREE_BENCHMARK:
-            self.fuzz_device = task_config.fuzz_device if task_config.fuzz_device else 'npu'
-            self.handler_type = task_config.handler_type if task_config.handler_type else 'check'
-            self.pert_mode = task_config.pert_mode if task_config.pert_mode else 'improve_precision'
-            self.fuzz_level = task_config.fuzz_level if task_config.fuzz_level else 'L1'
-            self.fuzz_stage = task_config.fuzz_stage if task_config.fuzz_stage else 'forward'
+            self.fuzz_device = task_config.fuzz_device
+            self.handler_type = task_config.handler_type
+            self.pert_mode = task_config.pert_mode
+            self.fuzz_level = task_config.fuzz_level
+            self.fuzz_stage = task_config.fuzz_stage
             self.preheat_config = {
-                "if_preheat": task_config.if_preheat if task_config.if_preheat is not None else True, 
-                "preheat_step": task_config.preheat_step if task_config.preheat_step else 15, 
-                "max_sample": task_config.max_sample if task_config.max_sample else 20, 
+                "if_preheat": task_config.if_preheat,
+                "preheat_step": task_config.preheat_step,
+                "max_sample": task_config.max_sample
             }
 
         self.online_run_ut = False
@@ -46,8 +61,7 @@ class DebuggerConfig:
             self.port = task_config.port if task_config.port else -1
 
         self.check()
-        if self.step:
-            self.step.sort()
+
         if self.level == "L2":
             if not self.scope or not isinstance(self.scope, list) or len(self.scope) != 1:
                 raise ValueError("scope must be configured as a list with one api name")
@@ -58,38 +72,37 @@ class DebuggerConfig:
                 for index, scope_spec in enumerate(self.scope):
                     self.scope[index] = scope_spec.replace(Const.BACKWARD, Const.FORWARD)
                     self.backward_input[self.scope[index]] = self.backward_input_list[index]
-        seed_all(self.seed, self.is_deterministic)
 
     def check_kwargs(self):
         if self.task and self.task not in Const.TASK_LIST:
-            raise Exception("task is invalid")
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                                   f"The task <{self.task}> is not in the {Const.TASK_LIST}.")
         if self.level and self.level not in Const.LEVEL_LIST:
-            raise Exception("level is invalid")
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                                   f"The level <{self.level}> is not in the {Const.LEVEL_LIST}.")
         if not self.dump_path:
-            raise Exception("Invalid dump path, please check your config")
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                                   f"The dump_path not found.")
 
     def check(self):
         self.check_kwargs()
-        self._check_rank()
-        self._check_step()
         return True
 
-    def check_model(self, model):
-        if self.level in ["L0", "mix"] and not model:
-            raise Exception(
-                f"For level {self.level}, PrecisionDebugger must receive a model argument."
-            )
-
-    def _check_rank(self):
-        if self.rank:
-            for rank_id in self.rank:
-                if not isinstance(rank_id, int) or rank_id < 0:
-                    raise ValueError(f"rank {self.rank} must be an integer and greater than or equal to 0.")
-            else:
-                logger.warning_on_rank_0(f"Rank argument is provided. Only rank {self.rank} data will be dumpped.")
-
-    def _check_step(self):
-        if self.step:
-            for s in self.step:
-                if not isinstance(s, int) or s < 0:
-                    raise ValueError(f"step element {s} must be an integer and greater than or equal to 0.")
+    def check_model(self, instance, start_model):
+        if self.level not in ["L0", "mix"]:
+            if instance.model is not None or start_model is not None:
+                logger.warning_on_rank_0(
+                    f"The current level is not L0 or mix level, so the model parameters will not be used.")
+            return
+        if start_model is None:
+            if instance.model is None:
+                logger.error_on_rank_0(
+                    f"For level {self.level}, PrecisionDebugger or start interface must receive a 'model' argument.")
+                raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, f"missing the parameter 'model'")
+            return
+        if isinstance(start_model, torch.nn.Module):
+            instance.model = start_model
+        else:
+            logger.error_on_rank_0(f"The 'model' parameter of start must be a torch.nn.Module type.")
+            raise MsprobeException(
+                MsprobeException.INVALID_PARAM_ERROR, f"model must be a torch.nn.Module")