mindstudio-probe 1.0.4__py3-none-any.whl → 1.1.1__py3-none-any.whl

msprobe/pytorch/hook_module/wrap_distributed.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2023. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
 
 import os
 from functools import wraps
@@ -23,6 +21,7 @@ from msprobe.pytorch.hook_module.hook_module import HOOKModule
 from msprobe.pytorch.common.utils import torch_device_guard
 from msprobe.core.common.const import Const
 from msprobe.core.common.file_utils import load_yaml
+from msprobe.core.common.inplace_op_checker import InplaceOpChecker
 
 
 cur_path = os.path.dirname(os.path.realpath(__file__))
@@ -50,7 +49,7 @@ class DistributedOPTemplate(HOOKModule):
         self.op_name_ = op_name
         self.prefix_op_name_ = "Distributed" + Const.SEP + str(op_name) + Const.SEP
         super().__init__(build_hook)
-        if not self.stop_hook and self.op_name_ in Const.INPLACE_LIST:
+        if not self.stop_hook and InplaceOpChecker.check(self.op_name_, InplaceOpChecker.OP_DISTRIBUTED):
             self.op_is_inplace = True
 
     @torch_device_guard

msprobe/pytorch/hook_module/wrap_functional.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
 
 import os
 import torch
@@ -32,31 +30,34 @@ def remove_dropout():
         from torch import _VF
         from torch.overrides import has_torch_function_unary, handle_torch_function
 
-        def function_dropout(input: torch.Tensor, p: float = 0.5, training: bool = True,
+        def function_dropout(input_tensor: torch.Tensor, p: float = 0.5, training: bool = True,
                              inplace: bool = False) -> torch.Tensor:
-            if has_torch_function_unary(input):
-                return handle_torch_function(function_dropout, (input,), input, p=0., training=training, inplace=inplace)
+            if has_torch_function_unary(input_tensor):
+                return handle_torch_function(
+                    function_dropout, (input_tensor,), input_tensor, p=0., training=training, inplace=inplace)
             if p < 0.0 or p > 1.0:
                 raise ValueError("dropout probability has to be between 0 and 1, " "but got {}".format(p))
-            return _VF.dropout_(input, 0., training) if inplace else _VF.dropout(input, 0., training)
-
+            return _VF.dropout_(input_tensor, 0., training) if inplace else _VF.dropout(input_tensor, 0., training)
 
-        def function_dropout2d(input: torch.Tensor, p: float = 0.5, training: bool = True,
+        def function_dropout2d(input_tensor: torch.Tensor, p: float = 0.5, training: bool = True,
                                inplace: bool = False) -> torch.Tensor:
-            if has_torch_function_unary(input):
-                return handle_torch_function(function_dropout2d, (input,), input, p=0., training=training, inplace=inplace)
+            if has_torch_function_unary(input_tensor):
+                return handle_torch_function(
+                    function_dropout2d, (input_tensor,), input_tensor, p=0., training=training, inplace=inplace)
             if p < 0.0 or p > 1.0:
                 raise ValueError("dropout probability has to be between 0 and 1, " "but got {}".format(p))
-            return _VF.feature_dropout_(input, 0., training) if inplace else _VF.feature_dropout(input, 0., training)
-
+            return _VF.feature_dropout_(input_tensor, 0., training) if inplace else _VF.feature_dropout(input_tensor,
+                                                                                                        0., training)
 
-        def function_dropout3d(input: torch.Tensor, p: float = 0.5, training: bool = True,
+        def function_dropout3d(input_tensor: torch.Tensor, p: float = 0.5, training: bool = True,
                                inplace: bool = False) -> torch.Tensor:
-            if has_torch_function_unary(input):
-                return handle_torch_function(function_dropout3d, (input,), input, p=0., training=training, inplace=inplace)
+            if has_torch_function_unary(input_tensor):
+                return handle_torch_function(
+                    function_dropout3d, (input_tensor,), input_tensor, p=0., training=training, inplace=inplace)
             if p < 0.0 or p > 1.0:
                 raise ValueError("dropout probability has to be between 0 and 1, " "but got {}".format(p))
-            return _VF.feature_dropout_(input, 0., training) if inplace else _VF.feature_dropout(input, 0., training)
+            return _VF.feature_dropout_(input_tensor, 0., training) if inplace else _VF.feature_dropout(input_tensor,
+                                                                                                        0., training)
 
         F.dropout = function_dropout
         F.dropout2d = function_dropout2d

msprobe/pytorch/hook_module/wrap_npu_custom.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
 
 import os
 import torch
@@ -21,24 +19,19 @@ import torch
 from msprobe.pytorch.hook_module.hook_module import HOOKModule
 from msprobe.pytorch.common.utils import torch_device_guard, torch_without_guard_version
 from msprobe.core.common.const import Const
+from msprobe.core.common.log import logger
 from msprobe.core.common.file_utils import load_yaml
 from msprobe.pytorch.function_factory import npu_custom_functions
 
-cur_path = os.path.dirname(os.path.realpath(__file__))
-yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml")
-
-
 try:
     import torch_npu
 except ImportError:
-    is_gpu = True
-else:
-    is_gpu = False
+    logger.info("Failing to import torch_npu.")
 
 
-cuda_func_mapping = {
-    "npu_fusion_attention" : "gpu_fusion_attention"
-}
+cur_path = os.path.dirname(os.path.realpath(__file__))
+yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml")
+cuda_func_mapping = {"npu_fusion_attention" : "gpu_fusion_attention"}
 
 
 def get_npu_ops():
@@ -83,7 +76,6 @@ class NpuOPTemplate(HOOKModule):
 def wrap_npu_op(op_name, hook):
     def npu_op_template(*args, **kwargs):
         return NpuOPTemplate(op_name, hook)(*args, **kwargs)
-
     return npu_op_template
 
 

msprobe/pytorch/hook_module/wrap_tensor.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
 
 import os
 

msprobe/pytorch/hook_module/wrap_torch.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
 
 import os
 import torch

msprobe/pytorch/hook_module/wrap_vf.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
 
 import os
 import torch

msprobe/pytorch/module_processer.py

@@ -1,10 +1,25 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from functools import wraps
 
 import torch
+from msprobe.core.common.const import Const
+from msprobe.core.data_dump.scope import ModuleRangeScope, MixRangeScope
 from torch.utils.hooks import BackwardHook
 
-from msprobe.core.common.const import Const
-from msprobe.core.data_dump.scope import ModuleRangeScope
 torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0'
 
 
@@ -15,10 +30,7 @@ class ModuleProcesser:
     module_node = {}
 
     def __init__(self, scope):
-        if isinstance(scope, ModuleRangeScope):
-            self.scope = scope
-        else:
-            self.scope = None
+        self.scope = scope if isinstance(scope, (ModuleRangeScope, MixRangeScope)) else None
         BackwardHook.setup_input_hook = ModuleProcesser.clone_return_value(BackwardHook.setup_input_hook)
         BackwardHook.setup_output_hook = ModuleProcesser.clone_return_value(BackwardHook.setup_output_hook)
         BackwardHook.setup_output_hook = ModuleProcesser.filter_tensor_and_tuple(BackwardHook.setup_output_hook)

msprobe/pytorch/monitor/anomaly_analyse.py

@@ -0,0 +1,201 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import argparse
+import ast
+import heapq
+
+from msprobe.core.common.log import logger
+from msprobe.core.common.const import MonitorConst
+from msprobe.core.common.file_utils import check_path_before_create, save_json, create_directory, remove_path, \
+    check_file_or_directory_path, load_json
+from msprobe.pytorch.monitor.anomaly_detect import GradAnomalyData
+
+
+class AnomalyDataWriter:
+    """
+    异常数据写入类，负责将异常数据写入到JSON文件中。
+    """
+
+    def __init__(self, dump_path, rank) -> None:
+        self.dump_path = dump_path
+        self.dump_rank_dir = os.path.join(self.dump_path, f"rank{rank}")
+        self.json_path = os.path.join(self.dump_rank_dir, MonitorConst.ANOMALY_JSON)
+
+    @staticmethod
+    def get_anomaly_dict(anomalies):
+        """将GradAnomalyData列表转换为json"""
+        anomalies_json = {}
+        for anomaly in anomalies:
+            anomalies_json.update({anomaly.get_key(): anomaly.to_dict()})
+        return anomalies_json
+
+    def init_detected_json(self):
+        """初始化落盘文件"""
+        check_path_before_create(self.dump_path)
+        if not os.path.exists(self.dump_path):
+            create_directory(self.dump_path)
+
+        if not os.path.exists(self.dump_rank_dir):
+            create_directory(self.dump_rank_dir)
+
+        if os.path.exists(self.json_path):
+            check_file_or_directory_path(self.json_path, isdir=False)
+            logger.warning(f"The existing file will be deleted: {self.json_path}.")
+            remove_path(self.json_path)
+        save_json(self.json_path, {}, indent=1)
+
+    def write_detected_json(self, anomalies):
+        """
+        落盘异常数据
+        Args:
+        anomalies: GradAnomalyData对象列表
+        """
+        anomalies_json = self.get_anomaly_dict(anomalies)
+        logger.info(f"{MonitorConst.ANOMALY_JSON} is at {self.dump_rank_dir}.")
+
+        data_to_write = load_json(self.json_path) if os.path.exists(self.json_path) else {}
+        data_to_write.update(anomalies_json)
+        save_json(self.json_path, data_to_write, indent=1)
+
+
+class AnomalyDataLoader:
+    def __init__(self, data_path) -> None:
+        self.data_path = data_path
+
+    @staticmethod
+    def create_instances_from_dict(anomalies_dict: dict):
+        instances = []
+        for values in anomalies_dict.values():
+            try:
+                instances.append(GradAnomalyData(**values))
+            except KeyError as e:
+                logger.warning(f"Missing key in anomaly data: {e}.")
+            except Exception as e:
+                logger.warning(f"Value error when creating a GradAnomalyData instance: {e}.")
+        return instances
+
+    def get_anomalies_from_jsons(self):
+        """遍历文件夹,从rankK/anomaly.json中读取异常数据
+        return: anomalies: GradAnomalyData对象列表
+        """
+        anomalies = []
+        check_file_or_directory_path(self.data_path, isdir=True)
+        for rank_dir in os.listdir(self.data_path):
+            rank_path = os.path.join(self.data_path, rank_dir)
+            if not os.path.isdir(rank_path):
+                continue
+            json_path = os.path.join(rank_path, MonitorConst.ANOMALY_JSON)
+            if not os.path.exists(json_path):
+                continue
+            data_anomalies = load_json(json_path)
+            instances = self.create_instances_from_dict(data_anomalies)
+            anomalies.extend(instances)
+        return anomalies
+
+
+class AnomalyAnalyse:
+    def __init__(self) -> None:
+        self.sorted_anomalies = []
+
+    def get_range_top_k(self, topk, step_list, anomalies):
+        """
+        获取前topk个step_list范围内的异常。
+        """
+        if not step_list:
+            filtered_anomalies = anomalies
+        else:
+            filtered_anomalies = [
+                anomaly
+                for anomaly in anomalies
+                if anomaly.step in step_list
+            ]
+        if topk >= len(filtered_anomalies):
+            self.sorted_anomalies = sorted(filtered_anomalies)
+        else:
+            self.sorted_anomalies = list(heapq.nsmallest(topk, filtered_anomalies))
+        return self.sorted_anomalies
+
+    def rewrite_sorted_anomalies(self, output_path):
+        """
+        将排序后的异常数据重新落盘
+        """
+        check_file_or_directory_path(output_path, isdir=True)
+
+        sorted_data = AnomalyDataWriter.get_anomaly_dict(self.sorted_anomalies)
+        logger.info(f"{MonitorConst.ANALYSE_JSON} is at {output_path}.")
+        json_path = os.path.join(output_path, MonitorConst.ANALYSE_JSON)
+        if os.path.exists(json_path):
+            logger.warning(f"The existing file will be deleted: {json_path}.")
+            remove_path(json_path)
+        save_json(json_path, sorted_data, indent=1)
+
+
+def _get_parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-d", "--data_path", dest="data_path_dir", default="./", type=str,
+                        help="<Required> The anomaly detect result dictionary: generate from monitor tool.",
+                        required=True,
+                        )
+    parser.add_argument("-o", "--out_path", dest="out_path", default="", type=str,
+                        help="<optional> The analyse task result out path.",
+                        required=False,
+                        )
+    parser.add_argument("-k", "--topk", dest="top_k_number", default=8, type=int,
+                        help="<optional> Top K number of earliest anomalies.",
+                        required=False,
+                        )
+    parser.add_argument("-s", "--step", dest="step_list", default="[]", type=str,
+                        help="<optional> Analyse which steps.",
+                        required=False,
+                        )
+    return parser.parse_args(sys.argv[1:])
+
+
+def _get_step_and_stop(args):
+    try:
+        step_list = ast.literal_eval(args.step_list)
+        if not isinstance(step_list, list):
+            raise ValueError(f"{args.step_list} is not a list.")
+    except (ValueError, SyntaxError, RecursionError) as e:
+        raise Exception(f"The step list must be a resolvable list type.") from e
+    if args.top_k_number <= 0:
+        raise Exception("The top k number must be greater than 0.")
+    return step_list, args.top_k_number
+
+
+def _anomaly_analyse():
+    args = _get_parse_args()
+    step_list, top_k_number = _get_step_and_stop(args)
+    loader = AnomalyDataLoader(args.data_path_dir)
+    anomalies = loader.get_anomalies_from_jsons()
+    analyser = AnomalyAnalyse()
+    top_anomalies = analyser.get_range_top_k(
+        top_k_number, step_list, anomalies
+    )
+    analyser.rewrite_sorted_anomalies(
+        args.out_path if args.out_path else args.data_path_dir
+    )
+
+    logger.info(f"Top {top_k_number} anomalies are listed as follows:")
+    for index, anomaly in enumerate(top_anomalies):
+        logger.info(f"{index}: {anomaly.message}")
+
+
+if __name__ == "__main__":
+    _anomaly_analyse()
+    logger.info("Analyse task completed.")