mindstudio-probe 1.0.3__py3-none-any.whl → 1.0.4__py3-none-any.whl

msprobe/mindspore/grad_probe/grad_monitor.py

@@ -1,27 +1,27 @@
-from msprobe.mindspore.grad_probe.global_context import grad_context
-from msprobe.mindspore.grad_probe.grad_analyzer import csv_generator
-from msprobe.mindspore.grad_probe.hook import hook_optimizer
-from msprobe.core.grad_probe.constant import GradConst
-
-
-class GradientMonitor:
-
-    def __init__(self, common_dict, task_config):
-        config = {}
-        config[GradConst.OUTPUT_PATH] = common_dict.dump_path
-        config[GradConst.STEP] = common_dict.step
-        config[GradConst.RANK] = common_dict.rank
-        config[GradConst.PARAM_LIST] = task_config.param_list
-        config[GradConst.LEVEL] = task_config.grad_level
-        config[GradConst.BOUNDS] = task_config.bounds
-        self.config = config
-        grad_context.init_context(self.config)
-
-    @staticmethod
-    def monitor(opt):
-        csv_generator.init(grad_context)
-        hook_optimizer(opt)
-
-    @staticmethod
-    def stop():
-        csv_generator.stop()
+from msprobe.mindspore.grad_probe.global_context import grad_context
+from msprobe.mindspore.grad_probe.grad_analyzer import csv_generator
+from msprobe.mindspore.grad_probe.hook import hook_optimizer
+from msprobe.core.grad_probe.constant import GradConst
+
+
+class GradientMonitor:
+
+    def __init__(self, common_dict, task_config):
+        config = {}
+        config[GradConst.OUTPUT_PATH] = common_dict.dump_path
+        config[GradConst.STEP] = common_dict.step
+        config[GradConst.RANK] = common_dict.rank
+        config[GradConst.PARAM_LIST] = task_config.param_list
+        config[GradConst.LEVEL] = task_config.grad_level
+        config[GradConst.BOUNDS] = task_config.bounds
+        self.config = config
+        grad_context.init_context(self.config)
+
+    @staticmethod
+    def monitor(opt):
+        csv_generator.init(grad_context)
+        hook_optimizer(opt)
+
+    @staticmethod
+    def stop():
+        csv_generator.stop()

msprobe/mindspore/grad_probe/grad_stat_csv.py

@@ -1,132 +1,132 @@
-from abc import ABC, abstractmethod
-import hashlib
-
-import mindspore
-from mindspore import ops, Tensor
-from msprobe.core.grad_probe.constant import GradConst
-
-
-class CsvInput:
-    def __init__(self, param_name, grad, bounds):
-        self.param_name = param_name
-        self.grad = grad
-        self.bounds = bounds
-
-class GradStatCsv:
-    csv = {}
-
-    @staticmethod
-    def get_csv_header(level, csv_input):
-        header = ["param_name"]
-        for key in level["header"]:
-            header.extend(GradStatCsv.csv[key].generate_csv_header(csv_input))
-        return header
-
-    @staticmethod
-    def get_csv_line(level, csv_input):
-        line = [csv_input.param_name]
-        for key in level["header"]:
-            line.extend(GradStatCsv.csv[key].generate_csv_content(csv_input))
-        return line
-
-
-def register_csv_item(key, cls=None):
-    if cls is None:
-        # 无参数时，返回装饰器函数
-        return lambda cls: register_csv_item(key, cls)
-    GradStatCsv.csv[key] = cls
-    return cls
-
-
-class CsvItem(ABC):
-    @staticmethod
-    @abstractmethod
-    def generate_csv_header(csv_input):
-        pass
-
-    @staticmethod
-    @abstractmethod
-    def generate_csv_content(csv_input):
-        pass
-
-
-@register_csv_item(GradConst.MD5)
-class CsvMd5(CsvItem):
-    def generate_csv_header(csv_input):
-        return ["MD5"]
-
-    def generate_csv_content(csv_input):
-        grad = csv_input.grad
-        tensor_bytes = grad.float().numpy().tobytes()
-        md5_hash = hashlib.md5(tensor_bytes)
-        return [md5_hash.hexdigest()]
-
-
-@register_csv_item(GradConst.DISTRIBUTION)
-class CsvDistribution(CsvItem):
-    def generate_csv_header(csv_input):
-        bounds = csv_input.bounds
-        intervals = []
-        if bounds:
-            intervals.append(f"(-inf, {bounds[0]}]")
-            for i in range(1, len(bounds)):
-                intervals.append(f"({bounds[i-1]}, {bounds[i]}]")
-        if intervals:
-            intervals.append(f"({bounds[-1]}, inf)")
-        intervals.append("=0")
-    
-        return intervals
-
-    def generate_csv_content(csv_input):
-        grad = csv_input.grad
-        bounds = csv_input.bounds
-        if grad.dtype == mindspore.bfloat16:
-            grad = grad.to(mindspore.float32)
-        element_num = grad.numel()
-        grad_equal_0_num = (grad == 0).sum().item()
-        bucketsize_result = ops.bucketize(grad.float(), bounds)
-        bucketsize_result = bucketsize_result.astype(mindspore.int8)
-        interval_nums = [(bucketsize_result == i).sum().item() for i in range(len(bounds) + 1)]
-        interval_nums.append(grad_equal_0_num)
-        return_list = [x / element_num if element_num != 0 else 0 for x in interval_nums]
-        return return_list
-
-
-@register_csv_item(GradConst.MAX)
-class CsvMax(CsvItem):
-    def generate_csv_header(csv_input):
-        return ["max"]
-
-    def generate_csv_content(csv_input):
-        grad = csv_input.grad
-        return [ops.amax(grad).float().numpy().tolist()]
-
-
-@register_csv_item(GradConst.MIN)
-class CsvMin(CsvItem):
-    def generate_csv_header(csv_input):
-        return ["min"]
-
-    def generate_csv_content(csv_input):
-        grad = csv_input.grad
-        return [ops.amin(grad).float().numpy().tolist()]
-
-
-@register_csv_item(GradConst.NORM)
-class CsvNorm(CsvItem):
-    def generate_csv_header(csv_input):
-        return ["norm"]
-
-    def generate_csv_content(csv_input):
-        grad = csv_input.grad
-        return [ops.norm(grad).float().numpy().tolist()]
-
-
-@register_csv_item(GradConst.SHAPE)
-class CsvShape(CsvItem):
-    def generate_csv_header(csv_input):
-        return ["shape"]
-
-    def generate_csv_content(csv_input):
-        grad = csv_input.grad
+from abc import ABC, abstractmethod
+import hashlib
+
+import mindspore
+from mindspore import ops, Tensor
+from msprobe.core.grad_probe.constant import GradConst
+
+
+class CsvInput:
+    def __init__(self, param_name, grad, bounds):
+        self.param_name = param_name
+        self.grad = grad
+        self.bounds = bounds
+
+class GradStatCsv:
+    csv = {}
+
+    @staticmethod
+    def get_csv_header(level, csv_input):
+        header = ["param_name"]
+        for key in level["header"]:
+            header.extend(GradStatCsv.csv[key].generate_csv_header(csv_input))
+        return header
+
+    @staticmethod
+    def get_csv_line(level, csv_input):
+        line = [csv_input.param_name]
+        for key in level["header"]:
+            line.extend(GradStatCsv.csv[key].generate_csv_content(csv_input))
+        return line
+
+
+def register_csv_item(key, cls=None):
+    if cls is None:
+        # 无参数时，返回装饰器函数
+        return lambda cls: register_csv_item(key, cls)
+    GradStatCsv.csv[key] = cls
+    return cls
+
+
+class CsvItem(ABC):
+    @staticmethod
+    @abstractmethod
+    def generate_csv_header(csv_input):
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def generate_csv_content(csv_input):
+        pass
+
+
+@register_csv_item(GradConst.MD5)
+class CsvMd5(CsvItem):
+    def generate_csv_header(csv_input):
+        return ["MD5"]
+
+    def generate_csv_content(csv_input):
+        grad = csv_input.grad
+        tensor_bytes = grad.float().numpy().tobytes()
+        md5_hash = hashlib.md5(tensor_bytes)
+        return [md5_hash.hexdigest()]
+
+
+@register_csv_item(GradConst.DISTRIBUTION)
+class CsvDistribution(CsvItem):
+    def generate_csv_header(csv_input):
+        bounds = csv_input.bounds
+        intervals = []
+        if bounds:
+            intervals.append(f"(-inf, {bounds[0]}]")
+            for i in range(1, len(bounds)):
+                intervals.append(f"({bounds[i-1]}, {bounds[i]}]")
+        if intervals:
+            intervals.append(f"({bounds[-1]}, inf)")
+        intervals.append("=0")
+    
+        return intervals
+
+    def generate_csv_content(csv_input):
+        grad = csv_input.grad
+        bounds = csv_input.bounds
+        if grad.dtype == mindspore.bfloat16:
+            grad = grad.to(mindspore.float32)
+        element_num = grad.numel()
+        grad_equal_0_num = (grad == 0).sum().item()
+        bucketsize_result = ops.bucketize(grad.float(), bounds)
+        bucketsize_result = bucketsize_result.astype(mindspore.int8)
+        interval_nums = [(bucketsize_result == i).sum().item() for i in range(len(bounds) + 1)]
+        interval_nums.append(grad_equal_0_num)
+        return_list = [x / element_num if element_num != 0 else 0 for x in interval_nums]
+        return return_list
+
+
+@register_csv_item(GradConst.MAX)
+class CsvMax(CsvItem):
+    def generate_csv_header(csv_input):
+        return ["max"]
+
+    def generate_csv_content(csv_input):
+        grad = csv_input.grad
+        return [ops.amax(grad).float().numpy().tolist()]
+
+
+@register_csv_item(GradConst.MIN)
+class CsvMin(CsvItem):
+    def generate_csv_header(csv_input):
+        return ["min"]
+
+    def generate_csv_content(csv_input):
+        grad = csv_input.grad
+        return [ops.amin(grad).float().numpy().tolist()]
+
+
+@register_csv_item(GradConst.NORM)
+class CsvNorm(CsvItem):
+    def generate_csv_header(csv_input):
+        return ["norm"]
+
+    def generate_csv_content(csv_input):
+        grad = csv_input.grad
+        return [ops.norm(grad).float().numpy().tolist()]
+
+
+@register_csv_item(GradConst.SHAPE)
+class CsvShape(CsvItem):
+    def generate_csv_header(csv_input):
+        return ["shape"]
+
+    def generate_csv_content(csv_input):
+        grad = csv_input.grad
         return [list(grad.shape)]

msprobe/mindspore/grad_probe/hook.py

@@ -1,92 +1,94 @@
-
-import os
-
-import mindspore
-import mindspore as ms
-from mindspore.common.api import jit
-from mindspore.nn.optim.optimizer import Optimizer
-from mindspore.common.parameter import Parameter
-from mindspore.common.initializer import initializer
-
-from msprobe.core.grad_probe.constant import GradConst
-from msprobe.core.common.log import logger
-
-from msprobe.core.common.utils import write_csv, remove_path
-from msprobe.mindspore.grad_probe.global_context import grad_context
-from msprobe.mindspore.grad_probe.grad_analyzer import grad_dump, get_rank_id
-from msprobe.mindspore.grad_probe.grad_analyzer import csv_generator
-from msprobe.mindspore.grad_probe.grad_stat_csv import GradStatCsv, CsvInput
-from msprobe.mindspore.grad_probe.utils import save_grad_direction, get_adapted_level
-
-class HookInput:
-
-    '''
-    HookInput is a class wrapping all the variables used for hooking optimizer
-    '''
-
-    def __init__(self, opt) -> None:
-        self.func = opt.construct
-        self.g_names = [param.name for param in opt._parameters]
-        self.param_list = grad_context.get_context(GradConst.PARAM_LIST)
-        self.rank_id = get_rank_id()
-        output_path = grad_context.get_context(GradConst.OUTPUT_PATH)
-        self.dump_dir = os.path.join(output_path, f"rank{self.rank_id}", "Dump")
-        self.save_dir = os.path.join(output_path, f"rank{self.rank_id}")
-        self.step_finish_flag = os.path.join(self.dump_dir, GradConst.STEP_FINISH)
-        if os.path.exists(self.save_dir):
-            logger.warning(f"Delete existing path {self.save_dir}.")
-            remove_path(self.save_dir)
-        self.level = grad_context.get_context(GradConst.LEVEL)
-        self.bounds = grad_context.get_context(GradConst.BOUNDS)
-        self.mode = mindspore.get_context("mode")
-
-def hook_graph_mode_optimizer(opt, hook_input):
-    @jit
-    def new_construct(self, gradients):
-        for index, grad_value in enumerate(gradients):
-            if hook_input.param_list and hook_input.g_names[index] not in hook_input.param_list:
-                continue
-            grad_dump(hook_input.dump_dir, hook_input.g_names[index], self.dump_step,
-                    grad_value, hook_input.level, hook_input.bounds)
-        ms.ops.TensorDump()(hook_input.step_finish_flag, self.dump_step)
-        self.assignadd(self.dump_step, self.global_step_increase_tensor)
-        out = hook_input.func(gradients)
-        return out
-
-    opt.dump_step = Parameter(initializer(0, [1], ms.int32), name="dump_step")
-    opt.construct = new_construct.__get__(opt, type(opt))
-    csv_generator.start()
-
-def hook_pynative_optimizer(opt, hook_input):
-    level_adapted = get_adapted_level(hook_input.level)
-
-    def hook_fn(cell, input):
-        gradients, = input
-        cur_step = grad_context.get_context(GradConst.CURRENT_STEP)
-        if grad_context.step_need_dump(cur_step) and grad_context.rank_need_dump(hook_input.rank_id):
-            output_lines = []
-            for index, grad_value in enumerate(gradients):
-                param_name = hook_input.g_names[index]
-                if hook_input.param_list and param_name not in hook_input.param_list:
-                    continue
-                csv_input = CsvInput(param_name, grad_value, hook_input.bounds)
-                grad_info = GradStatCsv.get_csv_line(level_adapted, csv_input)
-                output_lines.append(grad_info)
-                if level_adapted["have_grad_direction"]:
-                    save_grad_direction(param_name, grad_value, os.path.join(hook_input.save_dir, f'step{cur_step}'))
-            output_csv_path = os.path.join(hook_input.save_dir, f"grad_summary_{cur_step}.csv")
-            dummy_csv_input = CsvInput(None, None, hook_input.bounds)
-            output_lines.insert(0, GradStatCsv.get_csv_header(level_adapted, dummy_csv_input))
-            write_csv(output_lines, output_csv_path)
-        grad_context.update_step()
-
-    opt.register_forward_pre_hook(hook_fn)
-
-
-def hook_optimizer(opt: Optimizer):
-    hook_input = HookInput(opt)
-
-    if hook_input.mode == mindspore.GRAPH_MODE:
-        hook_graph_mode_optimizer(opt, hook_input)
-    else:
-        hook_pynative_optimizer(opt, hook_input)
+
+import os
+
+import mindspore
+import mindspore as ms
+from mindspore.common.api import jit
+from mindspore.nn.optim.optimizer import Optimizer
+from mindspore.common.parameter import Parameter
+from mindspore.common.initializer import initializer
+
+from msprobe.core.grad_probe.constant import GradConst
+from msprobe.mindspore.common.log import logger
+
+from msprobe.core.common.file_utils import remove_path, write_csv, create_directory
+from msprobe.mindspore.grad_probe.global_context import grad_context
+from msprobe.mindspore.grad_probe.grad_analyzer import grad_dump, get_rank_id
+from msprobe.mindspore.grad_probe.grad_analyzer import csv_generator
+from msprobe.mindspore.grad_probe.grad_stat_csv import GradStatCsv, CsvInput
+from msprobe.mindspore.grad_probe.utils import save_grad_direction, get_adapted_level
+
+class HookInput:
+
+    '''
+    HookInput is a class wrapping all the variables used for hooking optimizer
+    '''
+
+    def __init__(self, opt) -> None:
+        self.func = opt.construct
+        self.g_names = [param.name for param in opt._parameters]
+        self.param_list = grad_context.get_context(GradConst.PARAM_LIST)
+        self.rank_id = get_rank_id()
+        output_path = grad_context.get_context(GradConst.OUTPUT_PATH)
+        self.dump_dir = os.path.join(output_path, f"rank{self.rank_id}", "Dump")
+        self.save_dir = os.path.join(output_path, f"rank{self.rank_id}")
+        self.step_finish_flag = os.path.join(self.dump_dir, GradConst.STEP_FINISH)
+        if os.path.exists(self.save_dir):
+            logger.warning(f"Delete existing path {self.save_dir}.")
+            remove_path(self.save_dir)
+        self.level = grad_context.get_context(GradConst.LEVEL)
+        self.bounds = grad_context.get_context(GradConst.BOUNDS)
+        self.mode = mindspore.get_context("mode")
+
+def hook_graph_mode_optimizer(opt, hook_input):
+    @jit
+    def new_construct(self, gradients):
+        for index, grad_value in enumerate(gradients):
+            if hook_input.param_list and hook_input.g_names[index] not in hook_input.param_list:
+                continue
+            grad_dump(hook_input.dump_dir, hook_input.g_names[index], self.dump_step,
+                    grad_value, hook_input.level, hook_input.bounds)
+        ms.ops.TensorDump()(hook_input.step_finish_flag, self.dump_step)
+        self.assignadd(self.dump_step, self.global_step_increase_tensor)
+        out = hook_input.func(gradients)
+        return out
+
+    opt.dump_step = Parameter(initializer(0, [1], ms.int32), name="dump_step")
+    opt.construct = new_construct.__get__(opt, type(opt))
+    csv_generator.start()
+
+def hook_pynative_optimizer(opt, hook_input):
+    level_adapted = get_adapted_level(hook_input.level)
+
+    def hook_fn(cell, input):
+        gradients, = input
+        cur_step = grad_context.get_context(GradConst.CURRENT_STEP)
+        if grad_context.step_need_dump(cur_step) and grad_context.rank_need_dump(hook_input.rank_id):
+            create_directory(hook_input.save_dir)
+            output_lines = []
+            for index, grad_value in enumerate(gradients):
+                param_name = hook_input.g_names[index]
+                if hook_input.param_list and param_name not in hook_input.param_list:
+                    continue
+                csv_input = CsvInput(param_name, grad_value, hook_input.bounds)
+                grad_info = GradStatCsv.get_csv_line(level_adapted, csv_input)
+                output_lines.append(grad_info)
+                if level_adapted["have_grad_direction"]:
+                    save_grad_direction(param_name, grad_value, os.path.join(hook_input.save_dir, f'step{cur_step}'))
+            output_csv_path = os.path.join(hook_input.save_dir, f"grad_summary_{cur_step}.csv")
+            dummy_csv_input = CsvInput(None, None, hook_input.bounds)
+            output_lines.insert(0, GradStatCsv.get_csv_header(level_adapted, dummy_csv_input))
+            write_csv(output_lines, output_csv_path)
+            logger.info(f"write grad data to {output_csv_path}")
+        grad_context.update_step()
+
+    opt.register_forward_pre_hook(hook_fn)
+
+
+def hook_optimizer(opt: Optimizer):
+    hook_input = HookInput(opt)
+
+    if hook_input.mode == mindspore.GRAPH_MODE:
+        hook_graph_mode_optimizer(opt, hook_input)
+    else:
+        hook_pynative_optimizer(opt, hook_input)

msprobe/mindspore/grad_probe/utils.py

@@ -1,29 +1,30 @@
-import os
-
-import numpy as np
-import mindspore
-from msprobe.core.grad_probe.constant import GradConst, level_adp
-from msprobe.core.grad_probe.utils import check_param
-from msprobe.core.common.file_check import create_directory
-from msprobe.core.common.utils import check_path_before_create, change_mode, check_file_or_directory_path, save_npy
-
-
-def save_grad_direction(param_name, grad, save_path):
-    if not os.path.exists(save_path):
-        create_directory(save_path)
-    check_file_or_directory_path(save_path, isdir=True)
-    check_param(param_name)
-    save_filepath = os.path.join(save_path, f"{param_name}.npy")
-    check_path_before_create(save_filepath)
-
-    if grad.dtype == mindspore.bfloat16:
-        grad = grad.to(mindspore.float32)
-    grad_direction_tensor = grad > 0
-    grad_direction_ndarray = grad_direction_tensor.numpy()
-
-    save_npy(grad_direction_ndarray, save_filepath)
-
-
-def get_adapted_level(level: str):
-    level_adapted = level_adp.get(level)
+import os
+
+import mindspore
+from msprobe.core.grad_probe.constant import level_adp
+from msprobe.core.grad_probe.utils import check_param
+from msprobe.core.common.file_utils import (create_directory,
+                                            check_path_before_create,
+                                            check_file_or_directory_path,
+                                            save_npy)
+
+
+def save_grad_direction(param_name, grad, save_path):
+    if not os.path.exists(save_path):
+        create_directory(save_path)
+    check_file_or_directory_path(save_path, isdir=True)
+    check_param(param_name)
+    save_filepath = os.path.join(save_path, f"{param_name}.npy")
+    check_path_before_create(save_filepath)
+
+    if grad.dtype == mindspore.bfloat16:
+        grad = grad.to(mindspore.float32)
+    grad_direction_tensor = grad > 0
+    grad_direction_ndarray = grad_direction_tensor.numpy()
+
+    save_npy(grad_direction_ndarray, save_filepath)
+
+
+def get_adapted_level(level: str):
+    level_adapted = level_adp.get(level)
     return level_adapted