mindstudio-probe 8.1.0__tar.gz → 8.1.1__tar.gz

{mindstudio_probe-8.1.0 → mindstudio_probe-8.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mindstudio-probe
-Version: 8.1.0
+Version: 8.1.1
 Summary: Ascend Probe Utils
 Home-page: https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe
 Author: Ascend Team
@@ -28,11 +28,12 @@ Requires-Dist: pyyaml
 Requires-Dist: rich
 Requires-Dist: tqdm
 Requires-Dist: openpyxl>=3.0.6
-Requires-Dist: pyopenssl
+Requires-Dist: pyopenssl==24.2.1
 Requires-Dist: twisted
 Requires-Dist: matplotlib
 Requires-Dist: tensorboard
 Requires-Dist: tabulate
 Requires-Dist: pwinput
+Requires-Dist: psutil
 
 MindStudio-Probe is a set of tools for diagnosing and improving model accuracy on Ascend NPU, including API acc checker, ptdbg, grad tool etc.

{mindstudio_probe-8.1.0 → mindstudio_probe-8.1.1}/mindstudio_probe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mindstudio-probe
-Version: 8.1.0
+Version: 8.1.1
 Summary: Ascend Probe Utils
 Home-page: https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe
 Author: Ascend Team
@@ -28,11 +28,12 @@ Requires-Dist: pyyaml
 Requires-Dist: rich
 Requires-Dist: tqdm
 Requires-Dist: openpyxl>=3.0.6
-Requires-Dist: pyopenssl
+Requires-Dist: pyopenssl==24.2.1
 Requires-Dist: twisted
 Requires-Dist: matplotlib
 Requires-Dist: tensorboard
 Requires-Dist: tabulate
 Requires-Dist: pwinput
+Requires-Dist: psutil
 
 MindStudio-Probe is a set of tools for diagnosing and improving model accuracy on Ascend NPU, including API acc checker, ptdbg, grad tool etc.

{mindstudio_probe-8.1.0 → mindstudio_probe-8.1.1}/mindstudio_probe.egg-info/SOURCES.txt

@@ -218,7 +218,6 @@ msprobe/mindspore/api_accuracy_checker/type_mapping.py
 msprobe/mindspore/api_accuracy_checker/utils.py
 msprobe/mindspore/api_accuracy_checker/bench_functions/flash_attention_score.py
 msprobe/mindspore/api_accuracy_checker/bench_functions/fusion_operator.py
-msprobe/mindspore/api_accuracy_checker/generate_op_script/config_op.json
 msprobe/mindspore/api_accuracy_checker/generate_op_script/op_generator.py
 msprobe/mindspore/api_accuracy_checker/generate_op_script/operator_replication.template
 msprobe/mindspore/code_mapping/__init__.py

{mindstudio_probe-8.1.0 → mindstudio_probe-8.1.1}/mindstudio_probe.egg-info/requires.txt

@@ -6,9 +6,10 @@ pyyaml
 rich
 tqdm
 openpyxl>=3.0.6
-pyopenssl
+pyopenssl==24.2.1
 twisted
 matplotlib
 tensorboard
 tabulate
 pwinput
+psutil

{mindstudio_probe-8.1.0 → mindstudio_probe-8.1.1}/msprobe/core/common/const.py

@@ -681,6 +681,7 @@ class FileCheckConst:
     MAX_IR_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     MAX_ZIP_SIZE = 10737418240  # 10 * 1024 * 1024 * 1024
     MAX_FILE_IN_ZIP_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
+    MAX_FILE_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     COMMOM_FILE_SIZE = 1048576  # 1 * 1024 * 1024
     DIR = "dir"
     FILE = "file"

{mindstudio_probe-8.1.0 → mindstudio_probe-8.1.1}/msprobe/core/common/file_utils.py

@@ -33,7 +33,7 @@ import pandas as pd
 from msprobe.core.common.decorator import recursion_depth_decorator
 from msprobe.core.common.log import logger
 from msprobe.core.common.exceptions import FileCheckException
-from msprobe.core.common.const import FileCheckConst
+from msprobe.core.common.const import FileCheckConst, CompareConst
 from msprobe.core.common.global_lock import global_lock, is_main_process
 
 proc_lock = multiprocessing.Lock()
@@ -460,6 +460,17 @@ def save_excel(path, data):
                 return "list"
         raise ValueError("Data must be a DataFrame or a list of (DataFrame, sheet_name) pairs.")
 
+    def save_in_slice(df, base_name):
+        df_length = len(df)
+        if df_length < CompareConst.MAX_EXCEL_LENGTH:
+            df.to_excel(writer, sheet_name=base_name if base_name else 'Sheet1', index=False)
+        else:
+            slice_num = (df_length + CompareConst.MAX_EXCEL_LENGTH - 1) // CompareConst.MAX_EXCEL_LENGTH
+            slice_size = (df_length + slice_num - 1) // slice_num
+            for i in range(slice_num):
+                df.iloc[i * slice_size: min((i + 1) * slice_size, df_length)] \
+                    .to_excel(writer, sheet_name=f'{base_name}_part_{i}' if base_name else f'part_{i}', index=False)
+
     check_path_before_create(path)
     path = os.path.realpath(path)
 
@@ -467,12 +478,12 @@ def save_excel(path, data):
     data_type = validate_data(data)
 
     try:
-        if data_type == "single":
-            data.to_excel(path, index=False)
-        elif data_type == "list":
-            with pd.ExcelWriter(path) as writer:
+        with pd.ExcelWriter(path) as writer:
+            if data_type == "single":
+                save_in_slice(data, None)
+            elif data_type == "list":
                 for data_df, sheet_name in data:
-                    data_df.to_excel(writer, sheet_name=sheet_name, index=False)
+                    save_in_slice(data_df, sheet_name)
     except Exception as e:
         logger.error(f'Save excel file "{os.path.basename(path)}" failed.')
         raise RuntimeError(f"Save excel file {path} failed.") from e
@@ -677,8 +688,23 @@ def os_walk_for_files(path, depth):
     return res
 
 
+def check_zip_file(zip_file_path):
+    with zipfile.ZipFile(zip_file_path, 'r') as zip_file:
+        total_size = 0
+        if len(zip_file.infolist()) > FileCheckConst.MAX_FILE_IN_ZIP_SIZE:
+            raise ValueError(f"Too many files in {os.path.basename(zip_file_path)}")
+        for file_info in zip_file.infolist():
+            if file_info.file_size > FileCheckConst.MAX_FILE_SIZE:
+                raise ValueError(f"File {file_info.filename} is too large to extract")
+
+            total_size += file_info.file_size
+            if total_size > FileCheckConst.MAX_ZIP_SIZE:
+                raise ValueError(f"Total extracted size exceeds the limit of {FileCheckConst.MAX_ZIP_SIZE} bytes")
+
+
 def read_xlsx(file_path, sheet_name=None):
     check_file_or_directory_path(file_path)
+    check_zip_file(file_path)
     try:
         if sheet_name:
             result_df = pd.read_excel(file_path, keep_default_na=False, sheet_name=sheet_name)
@@ -783,17 +809,7 @@ def extract_zip(zip_file_path, extract_dir):
     check_file_suffix(zip_file_path, FileCheckConst.ZIP_SUFFIX)
     try:
         proc_lock.acquire()
-        with zipfile.ZipFile(zip_file_path, 'r') as zip_file:
-            total_size = 0
-            if len(zip_file.infolist()) > FileCheckConst.MAX_FILE_IN_ZIP_SIZE:
-                raise ValueError(f"Too many files in {os.path.basename(zip_file_path)}")
-            for file_info in zip_file.infolist():
-                if file_info.file_size > FileCheckConst.MAX_FILE_IN_ZIP_SIZE:
-                    raise ValueError(f"File {file_info.filename} is too large to extract")
-
-                total_size += file_info.file_size
-                if total_size > FileCheckConst.MAX_ZIP_SIZE:
-                    raise ValueError(f"Total extracted size exceeds the limit of {FileCheckConst.MAX_ZIP_SIZE} bytes")
+        check_zip_file(zip_file_path)
     except Exception as e:
         logger.error(f'Save content to file "{os.path.basename(zip_file_path)}" failed.')
         raise RuntimeError(f"Save content to file {os.path.basename(zip_file_path)} failed.") from e
@@ -877,7 +893,8 @@ class SharedDict:
             self._shm = shared_memory.SharedMemory(create=False, name=name)
         except FileNotFoundError:
             try:
-                self._shm = shared_memory.SharedMemory(create=True, name=name, size=1024 * 1024)
+                # 共享内存空间增加至5M
+                self._shm = shared_memory.SharedMemory(create=True, name=name, size=1024 * 1024 * 5)
                 data = pickle.dumps({})
                 self._shm.buf[0:len(data)] = bytearray(data)
                 logger.debug(f'create shared memory, name: {name}')
@@ -892,6 +909,7 @@ class SharedDict:
             except Exception as e:
                 logger.debug(f'shared dict is unreadable, reason: {e}, create new dict.')
                 self._dict = {}
+                self._shm.buf[:] = bytearray(b'\x00' * len(self._shm.buf))  # 清空内存
                 self._changed = True
 
 

{mindstudio_probe-8.1.0 → mindstudio_probe-8.1.1}/msprobe/core/common/utils.py

@@ -303,6 +303,21 @@ def get_file_type(file_path):
     return file_type
 
 
+def check_dump_json_key(json_data, device_type):
+    task = json_data.get('task', None)
+    if not task:
+        logger.error(f"Task for {device_type} is empty, please check.")
+        raise CompareException(CompareException.INVALID_TASK_ERROR)
+    if 'data' not in json_data:
+        logger.error(f"Missing 'data' in dump.json, please check dump.json of {device_type}.")
+        raise CompareException(CompareException.INVALID_DATA_ERROR)
+    api_data = json_data.get('data')
+    if not isinstance(api_data, dict):
+        logger.error(f"Invalid type for 'data': expected a dict. Please check dump.json of {device_type}.")
+        raise CompareException(CompareException.INVALID_DATA_ERROR)
+    return task, api_data
+
+
 def get_dump_mode(input_param):
     npu_path = input_param.get("npu_json_path", None)
     bench_path = input_param.get("bench_json_path", None)
@@ -310,12 +325,8 @@ def get_dump_mode(input_param):
     bench_json_data = load_json(bench_path)
     json_type = get_file_type(file_path=npu_path)
 
-    npu_task = npu_json_data.get('task', None)
-    bench_task = bench_json_data.get('task', None)
-
-    if not npu_task or not bench_task:
-        logger.error(f"Please check the dump task is correct, npu's task is {npu_task}, bench's task is {bench_task}.")
-        raise CompareException(CompareException.INVALID_TASK_ERROR)
+    npu_task, npu_api_data = check_dump_json_key(npu_json_data, 'npu')
+    bench_task, bench_api_data = check_dump_json_key(bench_json_data, 'bench')
 
     if npu_task != bench_task:
         logger.error(f"Please check the dump task is consistent.")
@@ -328,8 +339,8 @@ def get_dump_mode(input_param):
         return Const.STRUCTURE
 
     if npu_task == Const.STATISTICS:
-        npu_md5_compare = md5_find(npu_json_data['data'], json_type)
-        bench_md5_compare = md5_find(bench_json_data['data'], json_type)
+        npu_md5_compare = md5_find(npu_api_data, json_type)
+        bench_md5_compare = md5_find(bench_api_data, json_type)
         if npu_md5_compare == bench_md5_compare:
             return Const.MD5 if npu_md5_compare else Const.SUMMARY
         else:

{mindstudio_probe-8.1.0 → mindstudio_probe-8.1.1}/msprobe/core/compare/acc_compare.py

@@ -105,7 +105,7 @@ class Comparator:
         stack_json = input_param.get("stack_json_path")
         result_df = self.compare_statistics([npu_json, bench_json, stack_json])
         if not result_df.values.tolist():
-            logger.warning("Can`t match any op.")
+            logger.warning("Can`t match any op. No compare result file generated.")
             return
 
         # compare real data
@@ -186,11 +186,16 @@ class ParseData:
         elif self.mode_config.dump_mode == Const.MD5:
             result[Const.MD5] = []
 
-        api_nums = len(data_json['data'])
+        apis_data = data_json.get('data', None)
+        if not apis_data:
+            logger.warning('No APIs found in dump.json.')
+            return pd.DataFrame(result)
+
+        api_nums = len(apis_data)
         progress_bar = tqdm(total=api_nums, desc="API/Module Read Progress", unit="api/module", ncols=100)
 
         # 从json中循环解析API数据，遍历所有API
-        for data_name in data_json['data']:
+        for data_name in apis_data:
             check_op_str_pattern_valid(data_name)
             merge_list = self.gen_merge_list(data_json, data_name, stack_json_data)
             if not merge_list:
@@ -599,8 +604,12 @@ class CreateTable:
         npu_summary = [CompareConst.NPU_MAX, CompareConst.NPU_MIN, CompareConst.NPU_MEAN, CompareConst.NPU_NORM]
         bench_summary = [CompareConst.BENCH_MAX, CompareConst.BENCH_MIN, CompareConst.BENCH_MEAN,
                          CompareConst.BENCH_NORM]
-        result[npu_summary] = result['summary_x'].apply(self.set_summary).tolist()
-        result[bench_summary] = result['summary_y'].apply(self.set_summary).tolist()
+        if result.empty:
+            result[npu_summary] = pd.DataFrame(columns=npu_summary)
+            result[bench_summary] = pd.DataFrame(columns=bench_summary)
+        else:
+            result[npu_summary] = result['summary_x'].apply(self.set_summary).tolist()
+            result[bench_summary] = result['summary_y'].apply(self.set_summary).tolist()
 
         result_df = pd.DataFrame(columns=header)
         for h in header:

{mindstudio_probe-8.1.0 → mindstudio_probe-8.1.1}/msprobe/core/compare/utils.py

@@ -590,4 +590,10 @@ def compare_distributed_inner(npu_dump_dir, bench_dump_dir, output_path, compare
                 'bench_json_path': bench_path,
                 'is_print_compare_log': is_print_compare_log
             }
-            compare_func(input_param=dump_result_param, output_path=output_path, suffix=f'_{nr}', **kwargs)
+            try:
+                compare_func(input_param=dump_result_param, output_path=output_path, suffix=f'_{nr}', **kwargs)
+            except CompareException as e:
+                if e.code == CompareException.INVALID_DATA_ERROR:
+                    logger.error(f"Invalid or missing 'data' in dump.json. Skipping {nr} comparison.")
+                if e.code == CompareException.INVALID_TASK_ERROR:
+                    logger.error(f"Invalid or missing 'task' in dump.json. Skipping {nr} comparison.")

{mindstudio_probe-8.1.0 → mindstudio_probe-8.1.1}/msprobe/core/data_dump/data_collector.py

@@ -15,6 +15,7 @@
 
 import atexit
 import os
+import traceback
 
 from msprobe.core.data_dump.scope import ScopeFactory
 from msprobe.core.data_dump.json_writer import DataWriter
@@ -99,100 +100,150 @@ class DataCollector:
         self.data_writer.update_stack(name, stack_info)
 
     def forward_input_data_collect(self, name, module, pid, module_input_output, is_recompute=None):
-        if self.config.task == Const.FREE_BENCHMARK:
-            backward_name = name.replace(Const.FORWARD, Const.BACKWARD)
-            if self.check_scope_and_pid(self.scope, backward_name, pid):
-                self.data_processor.analyze_forward_input(backward_name, module, module_input_output)
-            return
-
-        if not self.check_scope_and_pid(self.scope, name, pid):
-            return
-
-        data_info = {}
-        if self.config.task != Const.STRUCTURE:
-            data_info = self.data_processor.analyze_forward_input(name, module, module_input_output)
-        self.set_is_recomputable(data_info, is_recompute)
-        if self.config.level == Const.LEVEL_L2:
-            return
-        self.handle_data(name, data_info, flush=self.data_processor.is_terminated)
+        try:
+
+            if self.config.task == Const.FREE_BENCHMARK:
+                backward_name = name.replace(Const.FORWARD, Const.BACKWARD)
+                if self.check_scope_and_pid(self.scope, backward_name, pid):
+                    self.data_processor.analyze_forward_input(backward_name, module, module_input_output)
+                return
+
+            if not self.check_scope_and_pid(self.scope, name, pid):
+                return
+
+            data_info = {}
+            if self.config.task != Const.STRUCTURE:
+                data_info = self.data_processor.analyze_forward_input(name, module, module_input_output)
+            self.set_is_recomputable(data_info, is_recompute)
+            if self.config.level == Const.LEVEL_L2:
+                return
+            self.handle_data(name, data_info, flush=self.data_processor.is_terminated)
+
+        except Exception:
+            tb = traceback.format_exc()
+            self.data_writer.write_error_log(
+                f"[ERROR] forward_input_data_collect failed: name={name}, pid={pid}\n{tb}"
+            )
 
     def forward_output_data_collect(self, name, module, pid, module_input_output, is_recompute=None):
-        self.update_construct(name)
-        if not self.check_scope_and_pid(self.scope, name, pid):
-            return
-
-        data_info = {}
-        if self.config.task != Const.STRUCTURE:
-            data_info = self.data_processor.analyze_forward_output(name, module, module_input_output)
-        self.set_is_recomputable(data_info, is_recompute)
-        if self.config.level == Const.LEVEL_L2:
-            return
-        self.call_stack_collect(name)
-        self.handle_data(name, data_info, flush=self.data_processor.is_terminated)
+        try:
+
+            self.update_construct(name)
+            if not self.check_scope_and_pid(self.scope, name, pid):
+                return
+
+            data_info = {}
+            if self.config.task != Const.STRUCTURE:
+                data_info = self.data_processor.analyze_forward_output(name, module, module_input_output)
+            self.set_is_recomputable(data_info, is_recompute)
+            if self.config.level == Const.LEVEL_L2:
+                return
+            self.call_stack_collect(name)
+            self.handle_data(name, data_info, flush=self.data_processor.is_terminated)
+
+        except Exception:
+            tb = traceback.format_exc()
+            self.data_writer.write_error_log(
+                f"[ERROR] forward_output_data_collect failed: name={name}, pid={pid}\n{tb}"
+            )
 
     def forward_data_collect_only_tensor(self, name, module, pid, module_input_output):
-        if not self.check_scope_and_pid(self.scope, name, pid):
-            return
-
-        self.data_processor.analyze_forward(name, module, module_input_output)
+        try:
+            if not self.check_scope_and_pid(self.scope, name, pid):
+                return
+            self.data_processor.analyze_forward(name, module, module_input_output)
 
+        except Exception:
+            tb = traceback.format_exc()
+            self.data_writer.write_error_log(
+                f"[ERROR] forward_data_collect_only_tensor failed: name={name}, pid={pid}\n{tb}"
+            )
 
     def forward_data_collect(self, name, module, pid, module_input_output, is_recompute=None):
-        self.update_construct(name)
-        if not self.check_scope_and_pid(self.scope, name, pid):
-            return
-
-        data_info = {}
-        if self.config.task != Const.STRUCTURE:
-            data_info = self.data_processor.analyze_forward(name, module, module_input_output)
-        self.set_is_recomputable(data_info, is_recompute)
-        self.call_stack_collect(name)
-        self.handle_data(name, data_info, flush=self.data_processor.is_terminated)
+        try:
+
+            self.update_construct(name)
+            if not self.check_scope_and_pid(self.scope, name, pid):
+                return
+            data_info = {}
+            if self.config.task != Const.STRUCTURE:
+                data_info = self.data_processor.analyze_forward(name, module, module_input_output)
+            self.set_is_recomputable(data_info, is_recompute)
+            self.call_stack_collect(name)
+            self.handle_data(name, data_info, flush=self.data_processor.is_terminated)
+
+        except Exception:
+            tb = traceback.format_exc()
+            self.data_writer.write_error_log(
+                f"[ERROR] forward_data_collect failed: name={name}, pid={pid}\n{tb}"
+            )
 
     def backward_data_collect_only_tensor(self, name, module, pid, module_input_output, is_recompute=None):
-        if not self.check_scope_and_pid(self.scope, name, pid):
-            return
+        try:
+            if not self.check_scope_and_pid(self.scope, name, pid):
+                return
+            self.data_processor.analyze_backward(name, module, module_input_output)
 
-        self.data_processor.analyze_backward(name, module, module_input_output)
+        except Exception:
+            tb = traceback.format_exc()
+            self.data_writer.write_error_log(
+                f"[ERROR] backward_data_collect_only_tensor failed: name={name}, pid={pid}\n{tb}"
+            )
 
     def backward_data_collect(self, name, module, pid, module_input_output, is_recompute=None):
-        self.update_construct(name)
-        if not self.check_scope_and_pid(self.scope, name, pid):
-            return
-
-        data_info = {}
-        if self.config.task != Const.STRUCTURE:
-            data_info = self.data_processor.analyze_backward(name, module, module_input_output)
-        if self.config.level == Const.LEVEL_L2:
-            return
-        # 获取执行反向的模块名称
-        if data_info and name.split(Const.SEP)[0] in Const.MODULE_PREFIX:
-            module_name = name.rsplit(Const.SEP, 2)[0]
-            # 将模块名称加入到反向模块名称集合中，用于梯度收集时判断是否需要收集梯度
-            self.backward_module_names[module_name] = True
-        self.handle_data(name, data_info, flush=self.data_processor.is_terminated)
+        try:
+            self.update_construct(name)
+            if not self.check_scope_and_pid(self.scope, name, pid):
+                return
+            data_info = {}
+            if self.config.task != Const.STRUCTURE:
+                data_info = self.data_processor.analyze_backward(name, module, module_input_output)
+            if self.config.level == Const.LEVEL_L2:
+                return
+            if data_info and name.split(Const.SEP)[0] in Const.MODULE_PREFIX:
+                module_name = name.rsplit(Const.SEP, 2)[0]
+                self.backward_module_names[module_name] = True
+            self.handle_data(name, data_info, flush=self.data_processor.is_terminated)
+
+        except Exception:
+            tb = traceback.format_exc()
+            self.data_writer.write_error_log(
+                f"[ERROR] backward_data_collect failed: name={name}, pid={pid}\n{tb}"
+            )
 
     def backward_input_data_collect(self, name, module, pid, module_input_output, is_recompute=None):
-        self.update_construct(name)
-        if not self.check_scope_and_pid(self.scope, name, pid):
-            return
-
-        data_info = {}
-        if self.config.task != Const.STRUCTURE:
-            data_info = self.data_processor.analyze_backward_input(name, module, module_input_output)
-        self.set_is_recomputable(data_info, is_recompute)
-        self.handle_data(name, data_info)
+        try:
+            self.update_construct(name)
+            if not self.check_scope_and_pid(self.scope, name, pid):
+                return
+            data_info = {}
+            if self.config.task != Const.STRUCTURE:
+                data_info = self.data_processor.analyze_backward_input(name, module, module_input_output)
+            self.set_is_recomputable(data_info, is_recompute)
+            self.handle_data(name, data_info)
+
+        except Exception:
+            tb = traceback.format_exc()
+            self.data_writer.write_error_log(
+                f"[ERROR] backward_input_data_collect failed: name={name}, pid={pid}\n{tb}"
+            )
 
     def backward_output_data_collect(self, name, module, pid, module_input_output, is_recompute=None):
-        self.update_construct(name)
-        if not self.check_scope_and_pid(self.scope, name, pid):
-            return
-
-        data_info = {}
-        if self.config.task != Const.STRUCTURE:
-            data_info = self.data_processor.analyze_backward_output(name, module, module_input_output)
-        self.set_is_recomputable(data_info, is_recompute)
-        self.handle_data(name, data_info)
+        try:
+            self.update_construct(name)
+            if not self.check_scope_and_pid(self.scope, name, pid):
+                return
+            data_info = {}
+            if self.config.task != Const.STRUCTURE:
+                data_info = self.data_processor.analyze_backward_output(name, module, module_input_output)
+            self.set_is_recomputable(data_info, is_recompute)
+            self.handle_data(name, data_info)
+
+        except Exception:
+            tb = traceback.format_exc()
+            self.data_writer.write_error_log(
+                f"[ERROR] backward_output_data_collect failed: name={name}, pid={pid}\n{tb}"
+            )
 
     def update_construct(self, name):
         if self.config.level not in DataCollector.level_without_construct:
@@ -228,20 +279,23 @@ class DataCollector:
         self.data_processor.update_iter(current_iter)
 
     def params_data_collect(self, name, param_name, pid, data):
-        grad_name = name + Const.SEP + Const.PARAMS_GRAD
-        self.update_api_or_module_name(grad_name)
-        # 校验scope和pid，以及当前name是否有过反向计算
-        if not self.check_scope_and_pid(self.scope, name, pid) and not self.backward_module_names.get(name):
-            # 如果没有反向计算，则需要清除之前占位写入的grad数据
-            if self.data_writer.cache_data.get("data"):
-                self.data_writer.cache_data.get("data").pop(grad_name, None)
-            return
-        data_info = self.data_processor.analyze_params(grad_name, param_name, data)
-        self.handle_data(grad_name, data_info, flush=self.data_processor.is_terminated)
-
+        try:
+            grad_name = name + Const.SEP + Const.PARAMS_GRAD
+            self.update_api_or_module_name(grad_name)
+            if not self.check_scope_and_pid(self.scope, name, pid) and not self.backward_module_names.get(name):
+                if self.data_writer.cache_data.get("data"):
+                    self.data_writer.cache_data.get("data").pop(grad_name, None)
+                return
+            data_info = self.data_processor.analyze_params(grad_name, param_name, data)
+            self.handle_data(grad_name, data_info, flush=self.data_processor.is_terminated)
+        except Exception:
+            tb = traceback.format_exc()
+            self.data_writer.write_error_log(
+                f"[ERROR] params_data_collect failed: "
+                f"name={name}, param_name={param_name}, pid={pid}\n{tb}"
+            )
 
     def debug_data_collect_forward(self, variable, name_with_count):
-
         data_info = self.data_processor.analyze_debug_forward(variable, name_with_count)
         name_with_count_category = name_with_count + Const.SEP + Const.DEBUG
         self.data_writer.update_debug({name_with_count_category: data_info})

{mindstudio_probe-8.1.0 → mindstudio_probe-8.1.1}/msprobe/core/data_dump/json_writer.py

@@ -17,9 +17,11 @@ import csv
 import os
 import copy
 import threading
+import traceback
+from datetime import datetime, timezone, timedelta
 
 from msprobe.core.common.const import Const, FileCheckConst
-from msprobe.core.common.file_utils import change_mode, FileOpen, save_json, load_json
+from msprobe.core.common.file_utils import change_mode, FileOpen, save_json, load_json, check_path_before_create
 from msprobe.core.common.log import logger
 from msprobe.core.common.decorator import recursion_depth_decorator
 
@@ -35,6 +37,7 @@ class DataWriter:
         self.free_benchmark_file_path = None
         self.dump_tensor_data_dir = None
         self.debug_file_path = None
+        self.dump_error_info_path = None
         self.flush_size = 1000
         self.larger_flush_size = 20000
         self.cache_data = {}
@@ -42,6 +45,7 @@ class DataWriter:
         self.cache_construct = {}
         self.cache_debug = {}
         self.stat_stack_list = []
+        self._error_log_initialized = False
 
     @staticmethod
     def write_data_to_csv(result: list, result_header: tuple, file_path: str):
@@ -128,6 +132,7 @@ class DataWriter:
         self.dump_tensor_data_dir = dump_path_aggregation.dump_tensor_data_dir
         self.free_benchmark_file_path = dump_path_aggregation.free_benchmark_file_path
         self.debug_file_path = dump_path_aggregation.debug_file_path
+        self.dump_error_info_path = dump_path_aggregation.dump_error_info_path
 
     def flush_data_periodically(self):
         dump_data = self.cache_data.get(Const.DATA)
@@ -142,6 +147,31 @@ class DataWriter:
         if length % threshold == 0:
             self.write_json()
 
+    def write_error_log(self, message: str):
+        """
+        写错误日志：
+          - 第一次调用时以 'w' 模式清空文件，之后都用 'a' 模式追加
+          - 添加时间戳
+          - 在 message 后写入当前的调用栈（方便追踪日志来源）
+        """
+        try:
+            mode = "w" if not self._error_log_initialized else "a"
+            self._error_log_initialized = True
+
+            check_path_before_create(self.dump_error_info_path)
+
+            with FileOpen(self.dump_error_info_path, mode) as f:
+                cst_timezone = timezone(timedelta(hours=8), name="CST")
+                timestamp = datetime.now(cst_timezone).strftime("%Y-%m-%d %H:%M:%S %z")
+                f.write(f"[{timestamp}] {message}\n")
+                f.write("Call stack (most recent call last):\n")
+
+                f.write("".join(traceback.format_stack()[:-1]))  # 去掉自己这一层
+                f.write("\n")
+        except Exception as e:
+            # 如果连写日志都失败了，就打印到 stderr
+            logger.warning(f"[FallbackError] Failed to write error log: {e}")
+
     def update_data(self, new_data):
         with lock:
             if not isinstance(new_data, dict) or len(new_data.keys()) != 1: