mindstudio-probe 1.2.1__py3-none-any.whl → 1.3.0__py3-none-any.whl

msprobe/pytorch/monitor/anomaly_detect.py

@@ -136,8 +136,8 @@ class AnomalyDataFactory(ABC):
         tag_name = tag[0]
         param_name = tag_name.split('/')[0]
         call_id = self.name2callid.get(tag_name, -1)
-        if MonitorConst.VPP_SEP in param_name:
-            vpp_stage = int(param_name.split(MonitorConst.VPP_SEP)[0])
+        if MonitorConst.NAME_SEP in param_name:
+            vpp_stage = int(param_name.split(MonitorConst.NAME_SEP)[0])
         else:
             vpp_stage = 0
 
@@ -161,10 +161,10 @@ class TrainStage:
     OPTIMIZER_STAGE = 2
 
 
-FORWARD_KEY = [MonitorConst.ACTV_IN, MonitorConst.ACTV_OUT]
-BACKWARD_KEY = [MonitorConst.ACTVGRAD_IN, MonitorConst.ACTVGRAD_OUT,
-                MonitorConst.PRE_GRAD, MonitorConst.POST_GRAD, MonitorConst.ACC_GRAD]
-OPTIMIZER_KEY = [MonitorConst.EXP_AVG, MonitorConst.EFXP_AVG_SQ]
+FORWARD_KEY = [MonitorConst.ACTV]
+BACKWARD_KEY = [MonitorConst.ACTVGRAD, MonitorConst.PRE_GRAD, 
+                MonitorConst.POST_GRAD, MonitorConst.ACC_GRAD]
+OPTIMIZER_KEY = [MonitorConst.EXP_AVG, MonitorConst.EXP_AVG_SQ]
 TRAIN_STAGE = {
     **{key_: TrainStage.FORWARD_STAGE for key_ in FORWARD_KEY},
     **{key_: TrainStage.BACKWARD_STAGE for key_ in BACKWARD_KEY},
@@ -221,7 +221,7 @@ class GradAnomalyData:
     @staticmethod
     def get_train_stage(tag_name):
         """
-        :param tag_name: "0:fc2_0/rank0/input", "0:fc1.weight/rank0/post_grad", "0:fc2.weight/rank0/efxp_avg_sq"
+        :param tag_name: "0:fc2.input:0/rank0/actv", "0:fc1.weight/rank0/post_grad", "0:fc2.weight/rank0/exp_avg_sq"
         :return: int, if forward return 0; if backward return 1; if optimizer return 2
         """
         key_ = tag_name.split("/")[-1]
@@ -361,10 +361,10 @@ class CSVWriterWithAD(BaseWriterWithAD):
 
         new_data = []
         for name, metric_value in self.context_dict.items():
-            if MonitorConst.VPP_SEP not in name:
-                new_data.append([name] + [step] + metric_value)
-            else:
-                new_data.append(name.split(MonitorConst.VPP_SEP) + [step] + metric_value)
+            new_line = name.split(MonitorConst.NAME_SEP) + metric_value
+            new_line.insert(2, step)
+            new_data.append(new_line)
+
         new_data = pd.DataFrame(new_data).round(self.ndigits).fillna("nan")
         write_df_to_csv(new_data, filepath, mode='a+', header=False)
         self.context_dict = defaultdict(list)
@@ -381,26 +381,11 @@ class CSVWriterWithAD(BaseWriterWithAD):
     def write_metrics(self, ops, metric_value, step, prefix=''):
         super().write_metrics(ops, metric_value, step, prefix='')
 
-        # generate csv headers
-        # set hashmap to reduce the number of headers generated.
-        # 前向的norm用input.ops_和output.ops_，反向的用input_grad.ops_和output_grad.ops_
-        if prefix in {"actv", "actv_grad"}:
-            if prefix == "actv":
-                input_and_output = [MonitorConst.ACTV_IN, MonitorConst.ACTV_OUT]
-            else:
-                input_and_output = [MonitorConst.ACTVGRAD_IN, MonitorConst.ACTVGRAD_OUT]
-            ops_ = [MonitorConst.DOT.join(i) for i in itertools.product(input_and_output, ops)]
-            csv_header = ["module_name", "step", *ops_]
+        if prefix in [MonitorConst.ACTV, MonitorConst.ACTVGRAD]:
+            self.header = MonitorConst.CSV_HEADER_XY + ops
         else:
-            csv_header = ["param_name", "step", *ops]
-
-        keys = list(metric_value.keys())
-        if keys and MonitorConst.VPP_SEP in keys[0]:
-            csv_header.insert(0, "vpp_stage")
-
-        self.header = csv_header
+            self.header = MonitorConst.CSV_HEADER + ops
         self.write_csv(prefix, step)
-        self.header = []
 
     def close(self):
         pass

msprobe/pytorch/monitor/csv2tb.py

@@ -22,37 +22,37 @@ from torch.utils.tensorboard import SummaryWriter
 from tqdm import tqdm
 
 from msprobe.core.common.const import MonitorConst
-from msprobe.core.common.file_utils import read_csv, create_directory, remove_path
+from msprobe.core.common.file_utils import read_csv, create_directory, remove_path, recursive_chmod
 from msprobe.core.common.utils import is_int
+from msprobe.core.common.decorator import recursion_depth_decorator
 from msprobe.pytorch.common.log import logger
 from msprobe.pytorch.monitor.utils import get_target_output_dir
 
 all_data_type_list = ["actv", "actv_grad", "exp_avg", "exp_avg_sq", "grad_unreduced", "grad_reduced", "param"]
 CSV_FILE_SUFFIX = r"_\d+-\d+\.csv"
+MAX_PROCESS_NUM = 128
 
 
-def parse_step_line(data, line_id, name, ops):
-    vp_id = data["vpp_stage"][line_id]
-    module_name = data[name][line_id]
-    step = data["step"][line_id]
+def parse_step_line(line, ops):
+    vp_id = line["vpp_stage"]
+    module_name = line[MonitorConst.HEADER_NAME]
+    step = line["step"]
     vpp_name = f"vp{vp_id}:{module_name}"
+    if 'micro_step' in line:
+        vpp_name = f'{vpp_name}{MonitorConst.NAME_SEP}micro{line["micro_step"]}'
     ops_result = {}
     for op in ops:
-        ops_result[op] = data[op][line_id]
+        ops_result[op] = line[op]
     return vpp_name, step, ops_result
 
 
 def parse_step_fn(filepath):
     data = read_csv(filepath)
-
-    header = list(data.keys())
-    name = header[MonitorConst.HEADER_NAME_INDEX]
-    ops = header[MonitorConst.OPS_START_INDEX:]
-
+    ops = [k for k in data.keys() if k in MonitorConst.OP_LIST]
     parse_step_result = {}
 
-    for line_id in range(len(data)):
-        vpp_name, step, ops_result = parse_step_line(data, line_id, name, ops)
+    for _, line in data.iterrows():
+        vpp_name, step, ops_result = parse_step_line(line, ops)
         if vpp_name not in parse_step_result:
             parse_step_result[vpp_name] = {}
         if step in parse_step_result[vpp_name]:
@@ -78,6 +78,7 @@ def write_step(output_dirpath, parse_step_result, rank, data_type):
                 writer.add_scalar(tag, value, step)
 
 
+@recursion_depth_decorator("update_dict", max_depth=50)
 def update_dict(dict1, dict2):
     for key, value in dict2.items():
         if key in dict1:
@@ -117,11 +118,13 @@ def csv2tb_by_step_work(target_output_dirs, output_dirpath, data_type_list):
 def check_process_num(process_num):
     if not is_int(process_num) or process_num <= 0:
         raise ValueError(f"process_num({process_num}) is not a positive integer")
+    if process_num > MAX_PROCESS_NUM:
+        raise ValueError(f"The maximum supported process_num is {MAX_PROCESS_NUM}, current value: {process_num}.")
 
 
 def check_data_type_list(data_type_list):
     if data_type_list is None:
-        logger.info(f"data_type_list is None, use defualt all_data_type_list: {all_data_type_list}")
+        logger.info(f"data_type_list is None, use default all_data_type_list: {all_data_type_list}")
         return
     if not isinstance(data_type_list, list):
         raise ValueError(f"data_type_list({data_type_list}) is not a list")
@@ -163,4 +166,5 @@ def csv2tensorboard_by_step(
         p.start()
     for p in processes:
         p.join()
+    recursive_chmod(output_dirpath)
     logger.info(f"output has been saved to: {output_dirpath}")

msprobe/pytorch/monitor/distributed/wrap_distributed.py

@@ -24,6 +24,7 @@ import torch.nn as nn
 from msprobe.core.common.const import MonitorConst
 from msprobe.core.common.file_utils import load_yaml
 from msprobe.pytorch.monitor.module_metric import get_metrics, get_summary_writer_tag_name
+from msprobe.pytorch.common.log import logger
 
 try:
     import torch_npu
@@ -37,6 +38,7 @@ WrapDistributedOps = load_yaml(OpsPath).get("distributed", [])
 
 StackBlackListPath = os.path.join(os.path.dirname(__file__), "stack_blacklist.yaml")
 StackBlackList = load_yaml(StackBlackListPath).get("stack", [])
+MAX_STRING_LENGTH = 1000
 
 distributed_func = {}
 for f in dir(dist):
@@ -139,6 +141,8 @@ def get_process_group(process_group):
 
 
 def stack_filter(stack):
+    if len(stack) > MAX_STRING_LENGTH:
+        logger.warning(f'The character string contains more than {MAX_STRING_LENGTH}. re match is skipped.')
     for pattern in StackBlackList:
         if re.search(pattern, stack):
             return False
@@ -188,10 +192,12 @@ def update_data(old, new):
 
 
 def is_target_line(codeline):
-    stack = get_callstack()
-    whole_stack = ';'.join(stack)
     if codeline == []:
         return True
+    stack = get_callstack()
+    whole_stack = ';'.join(stack)
+    if len(whole_stack) > MAX_STRING_LENGTH:
+        logger.warning(f'The character string contains more than {MAX_STRING_LENGTH}. re match is skipped.')
     for pattern in codeline:
         if re.search(pattern, whole_stack):
             return True