mindstudio-probe 1.2.1__tar.gz → 1.2.2__tar.gz

{mindstudio_probe-1.2.1 → mindstudio_probe-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mindstudio-probe
-Version: 1.2.1
+Version: 1.2.2
 Summary: Ascend Probe Utils
 Home-page: https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe
 Author: Ascend Team

{mindstudio_probe-1.2.1 → mindstudio_probe-1.2.2}/mindstudio_probe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mindstudio-probe
-Version: 1.2.1
+Version: 1.2.2
 Summary: Ascend Probe Utils
 Home-page: https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe
 Author: Ascend Team

{mindstudio_probe-1.2.1 → mindstudio_probe-1.2.2}/mindstudio_probe.egg-info/SOURCES.txt

@@ -38,6 +38,7 @@ msprobe/core/compare/layer_mapping/layer_mapping.py
 msprobe/core/compare/layer_mapping/postprocess_pass.py
 msprobe/core/compare/merge_result/merge_result.py
 msprobe/core/compare/merge_result/merge_result_cli.py
+msprobe/core/compare/merge_result/utils.py
 msprobe/core/data_dump/data_collector.py
 msprobe/core/data_dump/json_writer.py
 msprobe/core/data_dump/scope.py
@@ -83,6 +84,8 @@ msprobe/docs/24.code_mapping_Mindspore.md
 msprobe/docs/25.tool_function_introduction.md
 msprobe/docs/26.data_dump_PyTorch_baseline.md
 msprobe/docs/27.dump_json_instruction.md
+msprobe/docs/28.debugger_save_instruction.md
+msprobe/docs/28.kernel_dump_MindSpore.md
 msprobe/docs/FAQ.md
 msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md
 msprobe/docs/accuracy_checker_MindSpore/accuracy_checker_MindSpore_baseline.md
@@ -124,6 +127,7 @@ msprobe/docs/img/ms_dump.png
 msprobe/docs/img/ms_layer.png
 msprobe/docs/img/pt_dump.png
 msprobe/docs/img/monitor/cpu_info.png
+msprobe/docs/img/monitor/step_count_per_record.png
 msprobe/docs/img/visualization/fuzzy_match_ms.png
 msprobe/docs/img/visualization/fuzzy_match_pt.png
 msprobe/docs/img/visualization/tensorboard_1.png
@@ -164,6 +168,7 @@ msprobe/mindspore/api_accuracy_checker/data_manager.py
 msprobe/mindspore/api_accuracy_checker/main.py
 msprobe/mindspore/api_accuracy_checker/multi_api_accuracy_checker.py
 msprobe/mindspore/api_accuracy_checker/multi_data_manager.py
+msprobe/mindspore/api_accuracy_checker/torch_mindtorch_importer.py
 msprobe/mindspore/api_accuracy_checker/type_mapping.py
 msprobe/mindspore/api_accuracy_checker/utils.py
 msprobe/mindspore/code_mapping/__init__.py
@@ -225,6 +230,15 @@ msprobe/mindspore/grad_probe/hook.py
 msprobe/mindspore/grad_probe/utils.py
 msprobe/mindspore/mindtorch/__init__.py
 msprobe/mindspore/mindtorch/mindtorch_adaptor.py
+msprobe/mindspore/monitor/anomaly_detect.py
+msprobe/mindspore/monitor/features.py
+msprobe/mindspore/monitor/module_hook.py
+msprobe/mindspore/monitor/module_spec_verifier.py
+msprobe/mindspore/monitor/utils.py
+msprobe/mindspore/monitor/distributed/__init__.py
+msprobe/mindspore/monitor/distributed/distributed_ops.yaml
+msprobe/mindspore/monitor/distributed/stack_blacklist.yaml
+msprobe/mindspore/monitor/distributed/wrap_distributed.py
 msprobe/mindspore/overflow_check/__init__.py
 msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py
 msprobe/mindspore/overflow_check/overflow_check_tool_factory.py
@@ -278,16 +292,21 @@ msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py
 msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/torch_ops_config.yaml
 msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/utils.py
 msprobe/pytorch/bench_functions/__init__.py
+msprobe/pytorch/bench_functions/apply_adam.py
 msprobe/pytorch/bench_functions/apply_adam_w.py
 msprobe/pytorch/bench_functions/confusion_transpose.py
 msprobe/pytorch/bench_functions/fast_gelu.py
+msprobe/pytorch/bench_functions/group_norm_silu.py
 msprobe/pytorch/bench_functions/layer_norm_eval.py
 msprobe/pytorch/bench_functions/linear.py
 msprobe/pytorch/bench_functions/matmul_backward.py
+msprobe/pytorch/bench_functions/mish.py
+msprobe/pytorch/bench_functions/moe_gating_top_k_softmax.py
 msprobe/pytorch/bench_functions/npu_fusion_attention.py
 msprobe/pytorch/bench_functions/rms_norm.py
 msprobe/pytorch/bench_functions/rotary_mul.py
 msprobe/pytorch/bench_functions/scaled_mask_softmax.py
+msprobe/pytorch/bench_functions/sort_v2.py
 msprobe/pytorch/bench_functions/swiglu.py
 msprobe/pytorch/common/__init__.py
 msprobe/pytorch/common/compare_script.template

{mindstudio_probe-1.2.1 → mindstudio_probe-1.2.2}/msprobe/README.md

@@ -15,7 +15,7 @@ debugger = PrecisionDebugger(config_path='./config.json')
 ...
 debugger.start() # 一般在训练循环开头启动工具
 ... # 循环体
-debugger.stop() # 一般在训练循环末尾结束工具
+debugger.stop() # 一般在训练循环末尾结束工具。必须调用，否则可能导致精度数据落盘不全
 debugger.step() # 在训练循环的最后需要重置工具，非循环场景不需要
 ```
 
@@ -127,7 +127,7 @@ MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore.
 
 该功能收集和聚合模型训练过程中的网络层，优化器， 通信算子的中间值，帮助诊断模型训练过程中计算， 通信，优化器各部分出现的异常情况。
 
-[PyTorch 场景的训练状态监控](./docs/19.monitor.md)
+[兼容 PyTorch 和 MindSpore 框架的训练状态监控](./docs/19.monitor.md)
 
 ### 10 分级可视化构图比对
 

{mindstudio_probe-1.2.1 → mindstudio_probe-1.2.2}/msprobe/core/common/const.py

@@ -103,14 +103,16 @@ class Const:
     FREE_BENCHMARK = "free_benchmark"
     RUN_UT = "run_ut"
     GRAD_PROBE = "grad_probe"
-    TASK_LIST = [TENSOR, STATISTICS, OVERFLOW_CHECK, FREE_BENCHMARK, RUN_UT, GRAD_PROBE]
-    DUMP_DATA_COLLECTION_LIST = [STATISTICS, TENSOR]
+    STRUCTURE = "structure"
+    TASK_LIST = [TENSOR, STATISTICS, OVERFLOW_CHECK, FREE_BENCHMARK, RUN_UT, GRAD_PROBE, STRUCTURE]
+    DUMP_DATA_COLLECTION_LIST = [STATISTICS, TENSOR, STRUCTURE]
     DUMP_DATA_MODE_LIST = [ALL, INPUT, OUTPUT, FORWARD, BACKWARD]
     LEVEL_L0 = "L0"
     LEVEL_L1 = "L1"
     LEVEL_L2 = "L2"
     LEVEL_MIX = "mix"
-    LEVEL_LIST = [LEVEL_L0, LEVEL_L1, LEVEL_L2, LEVEL_MIX]
+    LEVEL_DEBUG = "debug"
+    LEVEL_LIST = [LEVEL_L0, LEVEL_L1, LEVEL_L2, LEVEL_MIX, LEVEL_DEBUG]
     ATTR_NAME_PREFIX = "wrap_"
     ATTR_NAME_PREFIX_LEN = len(ATTR_NAME_PREFIX)
     KERNEL_DUMP = "kernel_dump"
@@ -543,9 +545,24 @@ class MsCompareConst:
     TENSOR_API = "Tensor"
 
     API_NAME_STR_LENGTH = 4
+    MAX_RECURSION_DEPTH = 20
+
+    # Mindtorch api_info field
+    MINDTORCH_TENSOR = "Tensor"
+    MINDTORCH = "Torch"
+    MINDTORCH_FUNC = "Functional"
+    MINDTORCH_NPU = "NPU"
+    MINDTORCH_DIST = "Distributed"
+
+
+
+    MT_VALID_API_TYPES = [
+        MINDTORCH, MINDTORCH_FUNC, MINDTORCH_TENSOR
+    ]
 
     TASK_FIELD = "task"
     STATISTICS_TASK = "statistics"
+    FRAMEWORK = "framework"
     TENSOR_TASK = "tensor"
     DUMP_DATA_DIR_FIELD = "dump_data_dir"
     DATA_FIELD = "data"
@@ -623,29 +640,37 @@ class MonitorConst:
         "DeepSpeedZeroOptimizer_Stage1_or_2",
         "DeepSpeedZeroOptimizer_Stage3"
     )
+    DEEPSPEED_ZERO_OPT_FILTER = "DeepSpeedZeroOptimizer"
     RULE_NAME = ['AnomalyTurbulence']
 
     SLICE_SIZE = 20480
+    # used for name
     DOT = "."
-    VPP_SEP = ":"
+    NAME_SEP = ":"
+    INPUT_GRAD = "input_grad"
+    OUTPUT_GRAD = "output_grad"
     ACTV_IN = "input"
     ACTV_OUT = "output"
     ACTVGRAD_IN = "input_grad"
     ACTVGRAD_OUT = "output_grad"
+    # used for tasks
+    ACTV = "actv"
+    ACTVGRAD = "actv_grad"
     POST_GRAD = "post_grad"
     PRE_GRAD = "pre_grad"
     ACC_GRAD = "acc_grad"
     PREFIX_POST = "post"
     PREFIX_PRE = "pre"
-    OUTPUT_DIR_PATTERN = r"([\w-]{0,20})-rank(\d{1,5})-"
-
     EXP_AVG = "exp_avg"
-    EFXP_AVG_SQ = "efxp_avg_sq"
+    EXP_AVG_SQ = "exp_avg_sq"
+    PARAM = "param"
 
+    CSV_HEADER = ["vpp_stage", "name", "step"]
+    CSV_HEADER_XY = ["vpp_stage", "name", "step", "micro_step"]
+    OUTPUT_DIR_PATTERN = r"([\w-]{0,20})-rank(\d{1,5})-"
     ANOMALY_JSON = "anomaly.json"
     ANALYSE_JSON = "anomaly_analyse.json"
     TENSORBOARD = "tensorboard"
     CSV = "csv"
     API = "api"
-    OPS_START_INDEX = 3
-    HEADER_NAME_INDEX = 1
+    HEADER_NAME = 'name'

{mindstudio_probe-1.2.1 → mindstudio_probe-1.2.2}/msprobe/core/common/inplace_ops.yaml

@@ -250,5 +250,6 @@ inplace_distributed_op:
   - all_to_all
   - all_gather_into_tensor
   - reduce_scatter_tensor
+  - batch_isend_irecv
 
 

{mindstudio_probe-1.2.1 → mindstudio_probe-1.2.2}/msprobe/core/common/utils.py

@@ -239,6 +239,8 @@ def md5_find(data):
                 for data_detail in data[key_op][api_info]:
                     if data_detail and 'md5' in data_detail:
                         return True
+            if isinstance(data[key_op][api_info], bool):
+                continue
             elif data[key_op][api_info] and 'md5' in data[key_op][api_info]:
                 return True
     return False
@@ -303,6 +305,9 @@ def get_dump_mode(input_param):
     if npu_task == Const.TENSOR:
         return Const.ALL
 
+    if npu_task == Const.STRUCTURE:
+        return Const.STRUCTURE
+
     if npu_task == Const.STATISTICS:
         npu_md5_compare = md5_find(npu_json_data['data'])
         bench_md5_compare = md5_find(bench_json_data['data'])
@@ -496,3 +501,12 @@ def check_str_param(param):
     if not re.match(Const.REGEX_PREFIX_PATTERN, param):
         logger.error('The parameter {} contains special characters.'.format(param))
         raise MsprobeBaseException(MsprobeBaseException.INVALID_CHAR_ERROR)
+
+
+class DumpPathAggregation:
+    dump_file_path = None
+    stack_file_path = None
+    construct_file_path = None
+    dump_tensor_data_dir = None
+    free_benchmark_file_path = None
+    debug_file_path = None

{mindstudio_probe-1.2.1 → mindstudio_probe-1.2.2}/msprobe/core/compare/layer_mapping/data_scope_parser.py

@@ -112,7 +112,7 @@ class DumpDataItem:
             self.layer_scope = Const.SEP.join(data_list[:Const.TYPE_NAME_INDEX])
         else:
             self.layer_scope = Const.TOP_LAYER
-        if construct_info:
+        if construct_info and Const.SEP in construct_info:
             construct_list = construct_info.split(Const.SEP)
             if len(construct_list) < abs(Const.LAYER_NAME_INDEX):
                 logger.error(

{mindstudio_probe-1.2.1 → mindstudio_probe-1.2.2}/msprobe/core/compare/merge_result/merge_result.py

@@ -25,6 +25,7 @@ from msprobe.core.common.file_utils import load_yaml, logger, FileChecker, save_
 from msprobe.core.common.const import FileCheckConst, Const, CompareConst
 from msprobe.core.common.utils import CompareException, add_time_with_xlsx
 from msprobe.core.compare.utils import table_value_is_valid
+from msprobe.core.compare.merge_result.utils import replace_compare_index_dict, check_config
 
 
 def check_compare_result_name(file_name):
@@ -170,6 +171,8 @@ def search_api_index_result(api_list, compare_index_list, result_df, rank_num, c
             table_value_check(index_value)
             api_index_dict.setdefault(api_full_name, {})[rank_num] = index_value  # update api_index_dict
         compare_index_dict[compare_index] = api_index_dict
+
+    compare_index_dict = replace_compare_index_dict(compare_index_dict, compare_index_list, rank_num)
     return compare_index_dict
 
 
@@ -203,10 +206,13 @@ def result_process(compare_result_path_list, api_list):
             compare_index_list = check_index_dump_mode_consistent(dump_mode, rank_num)
             if len(compare_index_list) == 0:
                 return [], [], []
-            compare_index_dict = search_api_index_result(api_list, share_compare_index_list,
+            compare_index_list.extend([CompareConst.NPU_MAX, CompareConst.BENCH_MAX])
+            compare_index_dict = search_api_index_result(api_list, compare_index_list,
                                                          result_df, rank_num, compare_index_dict)
             compare_index_dict_list.append(compare_index_dict)
             rank_num_list.append(rank_num)
+            compare_index_list.pop()
+            compare_index_list.pop()
         else:
             logger.warning(f"Rank{rank_num} compare result is empty and will not shown in merged result.")
 
@@ -362,13 +368,8 @@ def merge_result(input_dir, output_dir, config_path):
     compare_result_path_list = get_result_path(input_dir)   # 获得的input_dir中所有比对结果件的全路径，数量少于2，便提示退出
 
     config = load_yaml(config_path)
-    if not config:
-        logger.error('config.yaml is empty, please check.')
-        raise CompareException(CompareException.MERGE_COMPARE_RESULT_ERROR)
+    config = check_config(config)
     api_list = config.get('api')
-    if not api_list:
-        logger.error('The APIs required to merge data were not found')
-        raise CompareException(CompareException.MERGE_COMPARE_RESULT_ERROR)
 
     # 初始化共享全局变量share_compare_index_list
     initialize_compare_index(config)

mindstudio_probe-1.2.2/msprobe/core/compare/merge_result/utils.py

@@ -0,0 +1,81 @@
+# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from msprobe.core.common.const import CompareConst
+from msprobe.core.common.file_utils import logger
+from msprobe.core.common.utils import CompareException
+
+
+def replace_compare_index_dict(compare_index_dict, compare_index_list, rank_num):
+    """
+    比对指标值为N/A、unsupported、Nan，将比对指标值替换成NPU max 和 Bench max(几个统计量相同)
+
+    示例：
+    Distributed.all_reduce.0.forward.output.group的比对指标值是N/A
+    替换后：
+    比对指标值为:
+        NPU: tp-0-1-2-3
+        Bench: tp-0-1-2-3
+    """
+
+    if CompareConst.NPU_MAX not in compare_index_dict or CompareConst.BENCH_MAX not in compare_index_dict:
+        compare_index_dict.pop(CompareConst.NPU_MAX, None)
+        compare_index_dict.pop(CompareConst.BENCH_MAX, None)
+        return compare_index_dict
+
+    # 遍历比对指标列表，排除最后两个指标NPU max， Bench max
+    for compare_index in compare_index_list[:-2]:
+        op_name_index_dict = compare_index_dict[compare_index]
+        # 遍历op_item名称和对应的比对指标值
+        for op_name, index_value in op_name_index_dict.items():
+            npu_max = compare_index_dict[CompareConst.NPU_MAX][op_name][rank_num]
+            bench_max = compare_index_dict[CompareConst.BENCH_MAX][op_name][rank_num]
+            # 如果当前比对指标值是N/A、unsupported、Nan，并且NPU和Bench的最大值是类型相同，进行替换
+            if index_value[rank_num] in [CompareConst.N_A, CompareConst.UNSUPPORTED, CompareConst.NAN]:
+                compare_index_dict[compare_index][op_name][rank_num] = f'NPU:{str(npu_max)}  Bench:{str(bench_max)}'
+
+    # 删除NPU_MAX和BENCH_MAX
+    compare_index_dict.pop(CompareConst.NPU_MAX, None)
+    compare_index_dict.pop(CompareConst.BENCH_MAX, None)
+    return compare_index_dict
+
+
+def check_config(config):
+    """
+    config.yaml 内容检查
+    Args: config:
+    Returns: config
+    """
+    if not config:
+        logger.error('config.yaml is empty, please check.')
+        raise CompareException(CompareException.MERGE_COMPARE_RESULT_ERROR)
+
+    api_list = config.get('api')
+    if not api_list:
+        logger.error('The APIs required to merge data were not found.')
+        raise CompareException(CompareException.MERGE_COMPARE_RESULT_ERROR)
+    if not isinstance(api_list, list):
+        logger.error("The config format of 'api' is incorrect, please check.")
+        raise CompareException(CompareException.MERGE_COMPARE_RESULT_ERROR)
+
+    compare_index_list = config.get('compare_index', [])
+    if compare_index_list is None:
+        compare_index_list = []
+        config['compare_index'] = compare_index_list
+    if not isinstance(compare_index_list, list):
+        logger.error("The config format of 'compare_index' is incorrect, please check.")
+        raise CompareException(CompareException.MERGE_COMPARE_RESULT_ERROR)
+
+    return config

{mindstudio_probe-1.2.1 → mindstudio_probe-1.2.2}/msprobe/core/compare/utils.py

@@ -170,6 +170,16 @@ def gen_op_item(op_data, op_name):
         elif op_item.get('type') == 'slice':
             op_item['dtype'] = op_data.get('type')
             op_item['shape'] = str(np.shape(np.array(op_data.get('value'))))
+        elif op_item.get('type') == 'ellipsis':
+            op_item['dtype'] = op_data.get('type')
+            op_item['shape'] = '[]'
+            for i in params:
+                op_item[i] = op_data.get('value')
+        elif op_item.get('type') == 'torch.ProcessGroup':
+            op_item['dtype'] = op_data.get('type')
+            op_item['shape'] = '[]'
+            for i in params:
+                op_item[i] = str(op_data.get('group_ranks'))
         else:
             op_item['dtype'] = str(type(op_data.get('value')))
             op_item['shape'] = '[]'

{mindstudio_probe-1.2.1 → mindstudio_probe-1.2.2}/msprobe/core/data_dump/data_collector.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
 # All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0  (the "License");
@@ -40,6 +40,7 @@ class DataCollector:
         self.scope = ScopeFactory(self.config).build_scope()
         self.backward_module_names = {}
         self.optimizer_status = ""
+        self.optimizer_status_first_start = {Const.OPTIMIZER: True, Const.CLIP_GRAD: True}
         atexit.register(self.write_json)
 
     @property
@@ -54,6 +55,17 @@ class DataCollector:
     def check_scope_and_pid(scope, name, pid):
         return (not scope or scope.check(name)) and pid == os.getpid()
 
+    @staticmethod
+    def set_is_recomputable(data_info, is_recompute):
+        if data_info and len(data_info) == 1 and is_recompute is not None: # 正常情况下data_info的长度应改为1
+            data_info[list(data_info.keys())[0]]["is_recompute"] = is_recompute
+
+    def reset_status(self):
+        self.optimizer_status = ""
+        self.optimizer_status_first_start = {Const.OPTIMIZER: True, Const.CLIP_GRAD: True}
+        self.data_writer.reset_cache()
+        self.backward_module_names.clear()
+
     def if_return_forward_new_output(self):
         return self.data_processor.if_return_forward_new_output()
 
@@ -77,7 +89,7 @@ class DataCollector:
         logger.debug(msg)
         self.data_writer.update_data(data_info)
 
-    def forward_input_data_collect(self, name, module, pid, module_input_output):
+    def forward_input_data_collect(self, name, module, pid, module_input_output, is_recompute=None):
         if self.config.task == Const.FREE_BENCHMARK:
             backward_name = name.replace(Const.FORWARD, Const.BACKWARD)
             if self.check_scope_and_pid(self.scope, backward_name, pid):
@@ -87,37 +99,48 @@ class DataCollector:
         if not self.check_scope_and_pid(self.scope, name, pid):
             return
 
-        data_info = self.data_processor.analyze_forward_input(name, module, module_input_output)
+        data_info = {}
+        if self.config.task != Const.STRUCTURE:
+            data_info = self.data_processor.analyze_forward_input(name, module, module_input_output)
+        self.set_is_recomputable(data_info, is_recompute)
         if self.config.level == Const.LEVEL_L2:
             return
         self.handle_data(name, data_info, flush=self.data_processor.is_terminated)
 
-    def forward_output_data_collect(self, name, module, pid, module_input_output):
+    def forward_output_data_collect(self, name, module, pid, module_input_output, is_recompute=None):
         self.update_construct(name)
         if not self.check_scope_and_pid(self.scope, name, pid):
             return
 
-        data_info = self.data_processor.analyze_forward_output(name, module, module_input_output)
+        data_info = {}
+        if self.config.task != Const.STRUCTURE:
+            data_info = self.data_processor.analyze_forward_output(name, module, module_input_output)
+        self.set_is_recomputable(data_info, is_recompute)
         if self.config.level == Const.LEVEL_L2:
             return
         self.data_writer.update_stack(self.data_processor.analyze_api_call_stack(name))
         self.handle_data(name, data_info, flush=self.data_processor.is_terminated)
 
-    def forward_data_collect(self, name, module, pid, module_input_output):
+    def forward_data_collect(self, name, module, pid, module_input_output, is_recompute=None):
         self.update_construct(name)
         if not self.check_scope_and_pid(self.scope, name, pid):
             return
 
-        data_info = self.data_processor.analyze_forward(name, module, module_input_output)
+        data_info = {}
+        if self.config.task != Const.STRUCTURE:
+            data_info = self.data_processor.analyze_forward(name, module, module_input_output)
+        self.set_is_recomputable(data_info, is_recompute)
         self.data_writer.update_stack(self.data_processor.analyze_api_call_stack(name))
         self.handle_data(name, data_info, flush=self.data_processor.is_terminated)
 
-    def backward_data_collect(self, name, module, pid, module_input_output):
+    def backward_data_collect(self, name, module, pid, module_input_output, is_recompute=None):
         self.update_construct(name)
         if not self.check_scope_and_pid(self.scope, name, pid):
             return
 
-        data_info = self.data_processor.analyze_backward(name, module, module_input_output)
+        data_info = {}
+        if self.config.task != Const.STRUCTURE:
+            data_info = self.data_processor.analyze_backward(name, module, module_input_output)
         if self.config.level == Const.LEVEL_L2:
             return
         # 获取执行反向的模块名称
@@ -127,25 +150,34 @@ class DataCollector:
             self.backward_module_names[module_name] = True
         self.handle_data(name, data_info, flush=self.data_processor.is_terminated)
 
-    def backward_input_data_collect(self, name, module, pid, module_input_output):
+    def backward_input_data_collect(self, name, module, pid, module_input_output, is_recompute=None):
         self.update_construct(name)
         if not self.check_scope_and_pid(self.scope, name, pid):
             return
 
-        data_info = self.data_processor.analyze_backward_input(name, module, module_input_output)
+        data_info = {}
+        if self.config.task != Const.STRUCTURE:
+            data_info = self.data_processor.analyze_backward_input(name, module, module_input_output)
+        self.set_is_recomputable(data_info, is_recompute)
         self.handle_data(name, data_info)
 
-    def backward_output_data_collect(self, name, module, pid, module_input_output):
+    def backward_output_data_collect(self, name, module, pid, module_input_output, is_recompute=None):
         self.update_construct(name)
         if not self.check_scope_and_pid(self.scope, name, pid):
             return
 
-        data_info = self.data_processor.analyze_backward_output(name, module, module_input_output)
+        data_info = {}
+        if self.config.task != Const.STRUCTURE:
+            data_info = self.data_processor.analyze_backward_output(name, module, module_input_output)
+        self.set_is_recomputable(data_info, is_recompute)
         self.handle_data(name, data_info)
 
     def update_construct(self, name):
         if self.config.level not in DataCollector.level_without_construct:
             if self.optimizer_status in [Const.OPTIMIZER, Const.CLIP_GRAD]:
+                if self.optimizer_status_first_start[self.optimizer_status]:
+                    self.data_writer.update_construct({self.optimizer_status: None})
+                    self.optimizer_status_first_start[self.optimizer_status] = False
                 self.data_writer.update_construct({name: self.optimizer_status})
             else:
                 self.data_writer.update_construct({name: self.module_processor.api_parent_node})
@@ -183,3 +215,16 @@ class DataCollector:
 
     def fill_stack_tensor_data(self):
         self.data_writer.fill_stack_tensor_data()
+
+    def debug_data_collect_forward(self, variable, name_with_count):
+
+        data_info = self.data_processor.analyze_debug_forward(variable, name_with_count)
+        self.data_writer.update_debug({name_with_count: data_info})
+
+    def debug_data_collect_backward(self, variable, grad_name_with_count):
+        # prepare all None nested data structure
+        all_none_data_info = self.data_processor.analyze_element_to_all_none(variable)
+        self.data_writer.update_debug({grad_name_with_count: all_none_data_info})
+
+        # register tensor backward hook
+        self.data_processor.analyze_debug_backward(variable, grad_name_with_count, self.data_writer.cache_debug['data'])