mindstudio-probe 1.0.3__py3-none-any.whl → 1.1.0__py3-none-any.whl

msprobe/pytorch/debugger/precision_debugger.py

@@ -1,125 +1,173 @@
-import torch
-from torch.utils.data import dataloader
-from msprobe.pytorch.debugger.debugger_config import DebuggerConfig
-from msprobe.pytorch.service import Service
-from msprobe.pytorch.common.log import logger
-from msprobe.pytorch.pt_config import parse_json_config
-from msprobe.core.common.exceptions import MsprobeException
-from msprobe.core.common.const import Const
-from msprobe.pytorch.grad_probe.grad_monitor import GradientMonitor
-
-
-class PrecisionDebugger:
-    _instance = None
-    tasks_not_need_debugger = [Const.GRAD_PROBE]
-
-    def __new__(cls, *args, **kwargs):
-        if cls._instance is None:
-            cls._instance = super(PrecisionDebugger, cls).__new__(cls)
-            cls._instance.config = None
-            cls._instance.enable_dataloader = False
-        return cls._instance
-
-    def __init__(
-        self,
-        config_path=None,
-        task=None,
-        dump_path=None,
-        level=None,
-        model=None,
-        step=None,
-    ):
-        if not hasattr(self, "initialized"):
-            self.api_origin = False
-            self.initialized = True
-            self.model = self.check_model_valid(model)
-            common_config, task_config = parse_json_config(config_path, task)
-            self.task = common_config.task
-            if self.task == Const.GRAD_PROBE:
-                self.gm = GradientMonitor(common_config, task_config)
-                return
-            if step:
-                common_config.step = step
-            self.config = DebuggerConfig(
-                common_config, task_config, task, dump_path, level
-            )
-            self.config.check_model(self.model)
-            self.service = Service(self.config)
-            self.enable_dataloader = self.config.enable_dataloader
-            if self.enable_dataloader:
-                logger.warning_on_rank_0("The enable_dataloader feature will be deprecated in the future.")
-                dataloader._BaseDataLoaderIter.__next__ = iter_tracer(dataloader._BaseDataLoaderIter.__next__)
-
-    @property
-    def instance(self):
-        return self._instance
-
-    @staticmethod
-    def check_model_valid(model):
-        if not model or isinstance(model, torch.nn.Module):
-            return model
-        raise MsprobeException(
-            MsprobeException.INVALID_PARAM_ERROR, "model 参数必须是torch.nn.Module类型。"
-        )
-
-    @classmethod
-    def start(cls):
-        instance = cls._instance
-        if instance.task in PrecisionDebugger.tasks_not_need_debugger:
-            return
-        if not instance:
-            raise Exception("No instance of PrecisionDebugger found.")
-        if instance.enable_dataloader:
-            logger.warning_on_rank_0("DataLoader is enabled, start() skipped.")
-        else:
-            instance.service.start(instance.model, instance.api_origin)
-            instance.api_origin = False
-
-    # 指定代码段dump前反向结束符，之后的计算过程数据将被忽略，无法被dump
-    @classmethod
-    def forward_backward_dump_end(cls):
-        instance = cls._instance
-        instance.service.forward_backward_dump_end()
-        instance.api_origin = True
-
-    @classmethod
-    def stop(cls):
-        instance = cls._instance
-        if instance.task in PrecisionDebugger.tasks_not_need_debugger:
-            return
-        if not instance:
-            raise Exception("PrecisionDebugger instance is not created.")
-        if instance.enable_dataloader:
-            logger.warning_on_rank_0("DataLoader is enabled, stop() skipped.")
-        else:
-            instance.service.stop()
-
-    @classmethod
-    def step(cls):
-        if cls._instance.task in PrecisionDebugger.tasks_not_need_debugger:
-            return
-        if not cls._instance:
-            raise Exception("PrecisionDebugger instance is not created.")
-        cls._instance.service.step()
-
-    @classmethod
-    def monitor(cls, model):
-        if not cls._instance:
-            raise Exception("PrecisionDebugger instance is not created.")
-        if cls._instance.task != Const.GRAD_PROBE:
-            return
-        cls._instance.gm.monitor(model)
-
-
-def iter_tracer(func):
-    def func_wrapper(*args, **kwargs):
-        debugger_instance = PrecisionDebugger.instance
-        debugger_instance.enable_dataloader = False
-        if not debugger_instance.service.first_start:
-            debugger_instance.stop()
-            debugger_instance.step()
-        result = func(*args, **kwargs)
-        debugger_instance.start()
-        debugger_instance.enable_dataloader = True
-        return result
-    return func_wrapper
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import namedtuple
+
+import torch
+from msprobe.core.common.const import Const, FileCheckConst, MsgConst
+from msprobe.core.common.exceptions import MsprobeException
+from msprobe.core.common.file_utils import FileChecker
+from msprobe.core.common.utils import get_real_step_or_rank
+from msprobe.pytorch.common.log import logger
+from msprobe.pytorch.debugger.debugger_config import DebuggerConfig
+from msprobe.pytorch.grad_probe.grad_monitor import GradientMonitor
+from msprobe.pytorch.pt_config import parse_json_config
+from msprobe.pytorch.service import Service
+from torch.utils.data import dataloader
+
+ConfigParameters = namedtuple("ConfigParameters", ["config_path", "task",
+                                                   "dump_path", "level", "model"])
+
+
+class PrecisionDebugger:
+    _instance = None
+    tasks_not_need_debugger = [Const.GRAD_PROBE]
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = super(PrecisionDebugger, cls).__new__(cls)
+            cls._instance.config = None
+            cls._instance.enable_dataloader = False
+        return cls._instance
+
+    def __init__(
+        self,
+        config_path=None,
+        task=None,
+        dump_path=None,
+        level=None,
+        model=None,
+        step=None,
+    ):
+        if not hasattr(self, "initialized"):
+            config_params = ConfigParameters(config_path,
+                                             task,
+                                             dump_path,
+                                             level,
+                                             model)
+            self.check_input_params(config_params)
+
+            self.api_origin = False
+            self.initialized = True
+            self.model = model
+            common_config, task_config = parse_json_config(config_path, task)
+            self.task = task if task else common_config.task
+            if self.task == Const.GRAD_PROBE:
+                self.gm = GradientMonitor(common_config, task_config)
+                return
+            if step:
+                common_config.step = get_real_step_or_rank(step, Const.STEP)
+            self.config = DebuggerConfig(
+                common_config, task_config, task, dump_path, level
+            )
+            self.service = Service(self.config)
+            self.enable_dataloader = self.config.enable_dataloader
+            if self.enable_dataloader:
+                logger.warning_on_rank_0("The enable_dataloader feature will be deprecated in the future.")
+                dataloader._BaseDataLoaderIter.__next__ = iter_tracer(dataloader._BaseDataLoaderIter.__next__)
+
+    @property
+    def instance(self):
+        return self._instance
+
+    @staticmethod
+    def check_input_params(args):
+        if args.config_path is not None:
+            if not isinstance(args.config_path, str):
+                raise MsprobeException(
+                    MsprobeException.INVALID_PARAM_ERROR, f"config_path must be a string")
+            file_checker = FileChecker(
+                file_path=args.config_path, path_type=FileCheckConst.FILE, file_type=FileCheckConst.JSON_SUFFIX)
+            file_checker.common_check()
+
+        if args.task is not None and args.task not in Const.TASK_LIST:
+            raise MsprobeException(
+                MsprobeException.INVALID_PARAM_ERROR, f"task must be one of {Const.TASK_LIST}")
+
+        if args.dump_path is not None:
+            if not isinstance(args.dump_path, str):
+                raise MsprobeException(
+                    MsprobeException.INVALID_PARAM_ERROR, f"dump_path must be a string")
+
+        if args.level is not None and args.level not in Const.LEVEL_LIST:
+            raise MsprobeException(
+                MsprobeException.INVALID_PARAM_ERROR, f"level must be one of {Const.LEVEL_LIST}")
+
+        if args.model is not None and not isinstance(args.model, torch.nn.Module):
+            raise MsprobeException(
+                MsprobeException.INVALID_PARAM_ERROR, f"model must be a torch.nn.Module")
+
+    @classmethod
+    def start(cls, model=None):
+        instance = cls._instance
+        if not instance:
+            raise Exception(MsgConst.NOT_CREATED_INSTANCE)
+        if instance.task in PrecisionDebugger.tasks_not_need_debugger:
+            return
+        instance.config.check_model(instance, model)
+        if instance.enable_dataloader:
+            logger.warning_on_rank_0("DataLoader is enabled, start() skipped.")
+        else:
+            instance.service.start(instance.model, instance.api_origin)
+            instance.api_origin = False
+
+    # 指定代码段dump前反向结束符，之后的计算过程数据将被忽略，无法被dump
+    @classmethod
+    def forward_backward_dump_end(cls):
+        instance = cls._instance
+        instance.service.forward_backward_dump_end()
+        instance.api_origin = True
+
+    @classmethod
+    def stop(cls):
+        instance = cls._instance
+        if not instance:
+            raise Exception(MsgConst.NOT_CREATED_INSTANCE)
+        if instance.task in PrecisionDebugger.tasks_not_need_debugger:
+            return
+        if instance.enable_dataloader:
+            logger.warning_on_rank_0("DataLoader is enabled, stop() skipped.")
+        else:
+            instance.service.stop()
+
+    @classmethod
+    def step(cls):
+        if not cls._instance:
+            raise Exception(MsgConst.NOT_CREATED_INSTANCE)
+        if cls._instance.task in PrecisionDebugger.tasks_not_need_debugger:
+            return
+        cls._instance.service.step()
+
+    @classmethod
+    def monitor(cls, model):
+        if not cls._instance:
+            raise Exception(MsgConst.NOT_CREATED_INSTANCE)
+        if cls._instance.task != Const.GRAD_PROBE:
+            return
+        cls._instance.gm.monitor(model)
+
+
+def iter_tracer(func):
+    def func_wrapper(*args, **kwargs):
+        debugger_instance = PrecisionDebugger.instance
+        debugger_instance.enable_dataloader = False
+        if not debugger_instance.service.first_start:
+            debugger_instance.stop()
+            debugger_instance.step()
+        result = func(*args, **kwargs)
+        debugger_instance.start()
+        debugger_instance.enable_dataloader = True
+        return result
+    return func_wrapper

msprobe/pytorch/free_benchmark/__init__.py

@@ -1,8 +1,23 @@
-from msprobe.core.common.log import logger
-from msprobe.core.common.exceptions import FreeBenchmarkException
-from msprobe.core.common.const import Const
-
-from .main import FreeBenchmarkCheck
-from .common.params import UnequalRow
-
-__all__ = [FreeBenchmarkCheck, UnequalRow]
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__all__ = ["FreeBenchmarkCheck", "UnequalRow"]
+
+from msprobe.core.common.const import Const
+from msprobe.core.common.exceptions import FreeBenchmarkException
+from msprobe.pytorch.common.log import logger
+
+from .common.params import UnequalRow
+from .main import FreeBenchmarkCheck

msprobe/pytorch/free_benchmark/common/constant.py

@@ -1,70 +1,70 @@
-from typing import Dict
-
-import numpy as np
-import torch
-from msprobe.pytorch.free_benchmark.common.enums import FuzzThreshold
-from msprobe.pytorch.free_benchmark.common.params import BenchmarkThd
-
-
-class CommonField:
-    DEVICE = "device"
-    META = "meta"
-    FUZZ_TENSOR = "fuzz_tensor"
-    REQUIRES_GRAD = "requires_grad"
-    HOLD_PLACE = "hold_place"
-    DISTRIBUTED_OP = "torch.distributed"
-    GRADSAVER = "grad_saver"
-
-
-class ThresholdConfig:
-    PERTURBATION_VALUE_DICT: Dict = {
-        torch.bfloat16: FuzzThreshold.BF16_THD,
-        torch.float16: FuzzThreshold.F16_THD,
-        torch.float32: FuzzThreshold.F32_THD,
-        torch.float64: FuzzThreshold.F64_THD,
-    }
-
-    ABS_TOL_VALUE_DICT: Dict = {
-        torch.bfloat16: FuzzThreshold.BF16_THD,
-        torch.float16: FuzzThreshold.F16_THD,
-        torch.float32: FuzzThreshold.F32_THD,
-        torch.float64: FuzzThreshold.F64_THD,
-    }
-
-    # bit翻转需要匹配到等长或更长的整型
-    PERTURBATION_BIT_DICT = {
-        torch.bfloat16: torch.int16,
-        torch.float16: torch.int16,
-        torch.float32: torch.int32,
-        torch.float64: torch.int64,
-    }
-
-    # 输入噪声下界
-    NOISE_INPUT_LOWER_BOUND = 1e-8
-    COMP_CONSISTENT = 1.0
-    COMP_NAN = np.nan
-    SYMBOL_FLIPPING = "symbol_flipping"
-    BACKWARD_OUTPUT_LOWER_BOUND = 1e-3
-    SMALL_VALUE = 1.0
-    # 预热初始阈值
-    PREHEAT_INITIAL_THD = 2.05
-    API_THD_STEP = 2.0
-
-    DTYPE_PER_THD = {
-        torch.float16: 1.002,
-        torch.bfloat16: 1.004,
-        torch.float32: 1.0002,
-    }
-    BENCHMARK_THD_DICT = {
-        torch.float32: BenchmarkThd(2**-14, 1.0, 2**-14, 1e-4),
-        torch.float16: BenchmarkThd(2**-11, 1.0, 2**-11, 1e-4),
-        torch.bfloat16: BenchmarkThd(2**-8, 1.0, 2**-8, 1e-4),
-    }
-
-    TENSOR_SPLIT_MAX_CHUNK = 128
-
-
-class PreheatConfig:
-    IF_PREHEAT = "if_preheat"
-    PREHEAT_STEP = "preheat_step"
-    MAX_SAMPLE = "max_sample"
+from typing import Dict
+
+import numpy as np
+import torch
+from msprobe.pytorch.free_benchmark.common.enums import FuzzThreshold
+from msprobe.pytorch.free_benchmark.common.params import BenchmarkThd
+
+
+class CommonField:
+    DEVICE = "device"
+    META = "meta"
+    FUZZ_TENSOR = "fuzz_tensor"
+    REQUIRES_GRAD = "requires_grad"
+    HOLD_PLACE = "hold_place"
+    DISTRIBUTED_OP = "torch.distributed"
+    GRADSAVER = "grad_saver"
+
+
+class ThresholdConfig:
+    PERTURBATION_VALUE_DICT: Dict = {
+        torch.bfloat16: FuzzThreshold.BF16_THD,
+        torch.float16: FuzzThreshold.F16_THD,
+        torch.float32: FuzzThreshold.F32_THD,
+        torch.float64: FuzzThreshold.F64_THD,
+    }
+
+    ABS_TOL_VALUE_DICT: Dict = {
+        torch.bfloat16: FuzzThreshold.BF16_THD,
+        torch.float16: FuzzThreshold.F16_THD,
+        torch.float32: FuzzThreshold.F32_THD,
+        torch.float64: FuzzThreshold.F64_THD,
+    }
+
+    # bit翻转需要匹配到等长或更长的整型
+    PERTURBATION_BIT_DICT = {
+        torch.bfloat16: torch.int16,
+        torch.float16: torch.int16,
+        torch.float32: torch.int32,
+        torch.float64: torch.int64,
+    }
+
+    # 输入噪声下界
+    NOISE_INPUT_LOWER_BOUND = 1e-8
+    COMP_CONSISTENT = 1.0
+    COMP_NAN = np.nan
+    SYMBOL_FLIPPING = "symbol_flipping"
+    BACKWARD_OUTPUT_LOWER_BOUND = 1e-3
+    SMALL_VALUE = 1.0
+    # 预热初始阈值
+    PREHEAT_INITIAL_THD = 2.05
+    API_THD_STEP = 2.0
+
+    DTYPE_PER_THD = {
+        torch.float16: 1.002,
+        torch.bfloat16: 1.004,
+        torch.float32: 1.0002,
+    }
+    BENCHMARK_THD_DICT = {
+        torch.float32: BenchmarkThd(2**-14, 1.0, 2**-14, 1e-4),
+        torch.float16: BenchmarkThd(2**-11, 1.0, 2**-11, 1e-4),
+        torch.bfloat16: BenchmarkThd(2**-8, 1.0, 2**-8, 1e-4),
+    }
+
+    TENSOR_SPLIT_MAX_CHUNK = 128
+
+
+class PreheatConfig:
+    IF_PREHEAT = "if_preheat"
+    PREHEAT_STEP = "preheat_step"
+    MAX_SAMPLE = "max_sample"

msprobe/pytorch/free_benchmark/common/counter.py

@@ -1,72 +1,72 @@
-from collections import defaultdict
-from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
-
-
-class PreheatCounter:
-    def __init__(self) -> None:
-        self.api_called_time: dict = defaultdict(int)
-        self.api_sample_time: dict = defaultdict(int)
-        self.one_step_used_api: dict = defaultdict(int)
-        self.api_thd: dict = defaultdict(dict)
-        self.preheat_record: dict = defaultdict(dict)
-        self.dtype_map: dict = {}
-        self.if_preheat: dict = defaultdict(dict)
-        self.step = 0
-
-    def clear_step(self):
-        self.preheat_record.clear()
-        self.api_called_time.clear()
-        self.api_sample_time.clear()
-    
-    def check_step(self, current_step):
-        if current_step != self.step:
-            self.clear_step()
-            self.step = current_step
-
-    def add_api_called_time(self, api_name: str):
-        self.api_called_time[api_name] += 1
-
-    def get_api_called_time(self, api_name: str) -> int:
-        return self.api_called_time[api_name]
-    
-    def add_api_sample_time(self, api_name: str):
-        self.api_sample_time[api_name] += 1
-
-    def get_api_sample_time(self, api_name: str) -> int:
-        return self.api_sample_time[api_name]
-
-    def add_one_step_used_api(self, api_name: str):
-        self.one_step_used_api[api_name] += 1
-
-    def get_one_step_used_api(self, api_name: str):
-        return self.one_step_used_api[api_name]
-
-    def update_preheat_record(self, api_name, dtype, cmp_result):
-        # 记录预热阶段CPU标杆比对的结果
-        if str(dtype) not in self.preheat_record[api_name].keys():
-            self.preheat_record[api_name][str(dtype)] = list()
-        self.preheat_record[api_name][str(dtype)].append(cmp_result)
-        self.dtype_map[str(dtype)] = dtype
-
-    def update_api_thd(self, api_name, dtype, threshold, dthreshold):
-        self.api_thd[api_name][str(dtype)] = (
-            threshold if threshold > dthreshold else dthreshold
-        )
-
-    def get_api_thd(self, api_name, dtype):
-        if not str(dtype) in self.api_thd[api_name]:
-            self.api_thd[api_name][str(dtype)] = ThresholdConfig.PREHEAT_INITIAL_THD
-            self.dtype_map[str(dtype)] = dtype
-        return self.api_thd[api_name][str(dtype)]
-
-    def set_api_preheat(self, api_name, dtype_str, is_preheat=True):
-        # 标记cpu不一致的dtype 不再进行预热
-        self.if_preheat[api_name][dtype_str] = is_preheat
-
-    def get_api_preheat(self, api_name, dtype):
-        # 标记cpu不一致的dtype 不再进行预热
-        if str(dtype) not in self.if_preheat[api_name]:
-            return True
-        return self.if_preheat[api_name][str(dtype)]
-
+from collections import defaultdict
+from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
+
+
+class PreheatCounter:
+    def __init__(self) -> None:
+        self.api_called_time: dict = defaultdict(int)
+        self.api_sample_time: dict = defaultdict(int)
+        self.one_step_used_api: dict = defaultdict(int)
+        self.api_thd: dict = defaultdict(dict)
+        self.preheat_record: dict = defaultdict(dict)
+        self.dtype_map: dict = {}
+        self.if_preheat: dict = defaultdict(dict)
+        self.step = 0
+
+    def clear_step(self):
+        self.preheat_record.clear()
+        self.api_called_time.clear()
+        self.api_sample_time.clear()
+    
+    def check_step(self, current_step):
+        if current_step != self.step:
+            self.clear_step()
+            self.step = current_step
+
+    def add_api_called_time(self, api_name: str):
+        self.api_called_time[api_name] += 1
+
+    def get_api_called_time(self, api_name: str) -> int:
+        return self.api_called_time[api_name]
+    
+    def add_api_sample_time(self, api_name: str):
+        self.api_sample_time[api_name] += 1
+
+    def get_api_sample_time(self, api_name: str) -> int:
+        return self.api_sample_time[api_name]
+
+    def add_one_step_used_api(self, api_name: str):
+        self.one_step_used_api[api_name] += 1
+
+    def get_one_step_used_api(self, api_name: str):
+        return self.one_step_used_api[api_name]
+
+    def update_preheat_record(self, api_name, dtype, cmp_result):
+        # 记录预热阶段CPU标杆比对的结果
+        if str(dtype) not in self.preheat_record[api_name].keys():
+            self.preheat_record[api_name][str(dtype)] = list()
+        self.preheat_record[api_name][str(dtype)].append(cmp_result)
+        self.dtype_map[str(dtype)] = dtype
+
+    def update_api_thd(self, api_name, dtype, threshold, dthreshold):
+        self.api_thd[api_name][str(dtype)] = (
+            threshold if threshold > dthreshold else dthreshold
+        )
+
+    def get_api_thd(self, api_name, dtype):
+        if not str(dtype) in self.api_thd[api_name]:
+            self.api_thd[api_name][str(dtype)] = ThresholdConfig.PREHEAT_INITIAL_THD
+            self.dtype_map[str(dtype)] = dtype
+        return self.api_thd[api_name][str(dtype)]
+
+    def set_api_preheat(self, api_name, dtype_str, is_preheat=True):
+        # 标记cpu不一致的dtype 不再进行预热
+        self.if_preheat[api_name][dtype_str] = is_preheat
+
+    def get_api_preheat(self, api_name, dtype):
+        # 标记cpu不一致的dtype 不再进行预热
+        if str(dtype) not in self.if_preheat[api_name]:
+            return True
+        return self.if_preheat[api_name][str(dtype)]
+
 preheat_counter = PreheatCounter()