mindstudio-probe 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

msprobe/pytorch/online_dispatch/dispatch.py

@@ -1,10 +1,24 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
 import os
 import time
-import json
 from multiprocessing import Pool
 
 import torch
-
 from torch.utils._python_dispatch import TorchDispatchMode
 
 try:
@@ -14,15 +28,15 @@ except ImportError:
 else:
     is_npu = True
 
-from msprobe.core.common.file_utils import check_path_before_create, check_file_or_directory_path, load_yaml
+from msprobe.core.common.file_utils import check_file_or_directory_path, load_yaml, FileOpen, create_directory
 from msprobe.core.common.const import Const, CompareConst
 from msprobe.pytorch.common.log import logger
-from msprobe.pytorch.online_dispatch.dump_compare import dispatch_workflow, dispatch_multiprocess, error_call, TimeStatistics, \
-    DispatchRunParam, DisPatchDataInfo
-from msprobe.pytorch.online_dispatch.utils import get_callstack, data_to_cpu,  get_sys_info, DispatchException, COMPARE_LOGO
+from msprobe.pytorch.online_dispatch.dump_compare import dispatch_workflow, dispatch_multiprocess, error_call, \
+    TimeStatistics, DispatchRunParam, DisPatchDataInfo
+from msprobe.pytorch.online_dispatch.utils import get_callstack, data_to_cpu, get_sys_info, DispatchException, \
+    COMPARE_LOGO
 from msprobe.pytorch.online_dispatch.compare import Comparator
-from msprobe.core.common.file_utils import FileOpen, create_directory
-
+from msprobe.core.common.utils import check_str_param, safe_get_value
 
 current_time = time.strftime("%Y%m%d%H%M%S")
 RESULT_FILE_NAME = "accuracy_checking_result_" + current_time + ".csv"
@@ -42,7 +56,7 @@ class PtdbgDispatch(TorchDispatchMode):
 
         self.device_id = torch_npu._C._npu_getDevice()
         self.dump_mode = dump_mode
-        self.dump_api_list = api_list
+        self.dump_api_list = api_list or []
         self.debug_flag = debug
         self.api_index = 0
         self.single_api_index_dict = {}
@@ -51,14 +65,13 @@ class PtdbgDispatch(TorchDispatchMode):
         self.all_summary = []
         self.call_stack_list = []
         self.process_num = process_num
-        self.filter_dump_api()
+        self.tag = tag
         self.check_param()
+        self.filter_dump_api()
         dir_name = self.get_dir_name(tag)
         self.root_path = os.path.join(os.path.realpath(dump_path), dir_name)
         self.root_cpu_path = os.path.join(self.root_path, f'cpu')
         self.root_npu_path = os.path.join(self.root_path, f'npu')
-        check_path_before_create(self.root_cpu_path)
-        check_path_before_create(self.root_npu_path)
         create_directory(self.root_cpu_path)
         create_directory(self.root_npu_path)
 
@@ -67,7 +80,7 @@ class PtdbgDispatch(TorchDispatchMode):
         self.comparator = Comparator(self.result_csv_path, self.detail_csv_path, False)
 
         self.aten_ops_blacklist = []
-        self.npu_adjust_autogard = []
+        self.npu_adjust_autograd = []
         yaml_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "torch_ops_config.yaml")
         self.get_ops(yaml_path)
 
@@ -76,8 +89,8 @@ class PtdbgDispatch(TorchDispatchMode):
             self.pool = Pool(process_num)
         if debug:
             logger.info(f'Main pid:{os.getpid()} device:{self.device_id} dump_list:{self.dump_api_list} '
-                         f'dump_mode:{self.dump_mode} cpu_path[{self.root_cpu_path}], npu_path[{self.root_npu_path}], '
-                         f'process[{process_num}]')
+                        f'dump_mode:{self.dump_mode} cpu_path[{self.root_cpu_path}], npu_path[{self.root_npu_path}], '
+                        f'process[{process_num}]')
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         super().__exit__(exc_type, exc_val, exc_tb)
@@ -119,7 +132,7 @@ class PtdbgDispatch(TorchDispatchMode):
                         output_num = output_num + 1
                     total_num = total_num + 1
             logger.info(f'Dispatch exit: Device[{self.device_id}], Pid[{os.getpid()} Input[{input_num}] '
-                         f'Output[{output_num}] Total[{total_num}] API_Total[{self.api_index}]]')
+                        f'Output[{output_num}] Total[{total_num}] API_Total[{self.api_index}]]')
 
     def __torch_dispatch__(self, func, types, args=(), kwargs=None):
         if not is_npu:
@@ -134,7 +147,7 @@ class PtdbgDispatch(TorchDispatchMode):
             logger.error(f"Please check the func name {func.__name__}!")
             return func(*args, **kwargs)
 
-        self.enable_autogard(aten_api)
+        self.enable_autograd(aten_api)
         if aten_api in self.aten_ops_blacklist:
             npu_out = func(*args, **kwargs)
             return npu_out
@@ -151,24 +164,31 @@ class PtdbgDispatch(TorchDispatchMode):
 
         if self.debug_flag:
             logger.info(f'Dispatch Info: Rank[{self.device_id}], Pid[{os.getpid()}], Func[{func.__name__}], '
-                         f'Name[{run_param.aten_api}_{run_param.single_api_index}], '
-                         f'Count[{self.api_index}], Sys[{get_sys_info()}]')
+                        f'Name[{run_param.aten_api}_{run_param.single_api_index}], '
+                        f'Count[{self.api_index}], Sys[{get_sys_info()}]')
 
         cpu_args = []
         cpu_kwargs = []
         data_to_cpu(args, 0, cpu_args)
         data_to_cpu(kwargs, 0, cpu_kwargs)
-        cpu_args = cpu_args[0]
-        cpu_kwargs = cpu_kwargs[0]
+        
+        cpu_args = safe_get_value(cpu_args, 0, "cpu_args")
+        cpu_kwargs = safe_get_value(cpu_kwargs, 0, "cpu_kwargs")
 
         with TimeStatistics("NPU RUN", run_param):
             npu_out = func(*args, **kwargs)
         npu_out_cpu = []
         data_to_cpu(npu_out, 0, npu_out_cpu)
-        npu_out_cpu = npu_out_cpu[0]
+        npu_out_cpu = safe_get_value(npu_out_cpu, 0, "npu_out_cpu")
 
         with TimeStatistics("CPU RUN", run_param):
-            cpu_out = func(*cpu_args, **cpu_kwargs)
+            try:
+                cpu_out = func(*cpu_args, **cpu_kwargs)
+            except RuntimeError as e:
+                self.api_index -= 1
+                logger.warning(f"RuntimeError: {e}")
+                logger.warning(f"This aten_api {aten_api} does not support running on cpu, so skip it.")
+                return npu_out
 
         if isinstance(cpu_out, torch.Tensor) and cpu_out.dtype in [torch.bfloat16, torch.float16, torch.half]:
             cpu_out = cpu_out.float()
@@ -216,7 +236,7 @@ class PtdbgDispatch(TorchDispatchMode):
     def get_ops(self, file_path):
         yaml_file = load_yaml(file_path)
         self.aten_ops_blacklist = yaml_file.get('aten_ops_blacklist')
-        self.npu_adjust_autogard = yaml_file.get('npu_adjust_autogard')
+        self.npu_adjust_autograd = yaml_file.get('npu_adjust_autograd')
 
     def filter_dump_api(self):
         if self.dump_mode != Const.LIST or not self.dump_api_list:
@@ -260,6 +280,17 @@ class PtdbgDispatch(TorchDispatchMode):
         if not isinstance(self.dump_api_list, list):
             logger.error('The type of parameter "api_list" can only be list.')
             raise DispatchException(DispatchException.INVALID_PARAMETER)
+        if not all(isinstance(item, str) for item in self.dump_api_list):
+            logger.error('The type of parameter in "api_list" can only be str.')
+            raise DispatchException(DispatchException.INVALID_PARAMETER)
+        if len(self.dump_api_list) > Const.STEP_RANK_MAXIMUM_VALUE:
+            logger.error('The length of parameter "api_list" should not be greater '
+                         f'than {Const.STEP_RANK_MAXIMUM_VALUE}.')
+            raise DispatchException(DispatchException.INVALID_PARAMETER)
+        for item in self.dump_api_list:
+            check_str_param(item)
+        if self.tag is not None:
+            check_str_param(self.tag)
         if not isinstance(self.debug_flag, bool):
             logger.error('The type of parameter "debug" can only be bool.')
             raise DispatchException(DispatchException.INVALID_PARAMETER)
@@ -267,6 +298,6 @@ class PtdbgDispatch(TorchDispatchMode):
             logger.error('The type of parameter "process_num" can only be int and it should not be less than 0.')
             raise DispatchException(DispatchException.INVALID_PARAMETER)
 
-    def enable_autogard(self, aten_api):
-        if aten_api in self.npu_adjust_autogard:
-            torch._C._dispatch_tls_set_dispatch_key_excluded(torch._C.DispatchKey.AutogradFunctionality, False)
+    def enable_autograd(self, aten_api):
+        if aten_api in self.npu_adjust_autograd:
+            torch._C._dispatch_tls_set_dispatch_key_excluded(torch._C.DispatchKey.AutogradFunctionality, False)

msprobe/pytorch/online_dispatch/dump_compare.py

@@ -1,11 +1,26 @@
-import os
-import json
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import copy
+import json
+import os
 from datetime import datetime, timezone
 
 import torch
+from msprobe.core.common.file_utils import FileOpen, save_npy, save_json
 from msprobe.pytorch.common.log import logger
-from msprobe.core.common.file_utils import FileOpen, save_npy
 
 
 class DispatchRunParam:
@@ -55,7 +70,7 @@ class TimeStatistics:
         if self.debug:
             self.time = datetime.now(tz=timezone.utc)
             logger.info(f'Time[{self.tag}]-ENTER: Dev[{self.device}], Pid[{os.getpid()}], Fun[{self.fun}], ' \
-                         f'Id[{self.index}]')
+                        f'Id[{self.index}]')
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         if self.debug:
@@ -92,10 +107,8 @@ def dump_data(data, prefix, dump_path):
 def save_temp_summary(api_index, single_api_summary, path, lock):
     summary_path = os.path.join(path, f'summary.json')
     lock.acquire()
-    with FileOpen(summary_path, "a") as f:
-        json.dump([api_index, single_api_summary], f)
-        f.write('\n')
-    lock.release()
+    data = [api_index, single_api_summary]
+    save_json(summary_path, data, mode='a')
 
 
 def dispatch_workflow(run_param: DispatchRunParam, data_info: DisPatchDataInfo):
@@ -152,4 +165,3 @@ def dispatch_multiprocess(run_param, dispatch_data_info):
 
 def error_call(err):
     logger.error(f'multiprocess {err}')
-

msprobe/pytorch/online_dispatch/single_compare.py

@@ -1,9 +1,27 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import logging
+from collections import namedtuple
 from functools import wraps
+
 import torch
-from prettytable import PrettyTable
-from collections import namedtuple
 from msprobe.pytorch.common.log import logger
+from msprobe.pytorch.online_dispatch.utils import check_idx_valid
+from prettytable import PrettyTable
+
 
 def func_log_wrapper():
     def _out_wrapper(func):
@@ -13,9 +31,9 @@ def func_log_wrapper():
             x = func(*kargs, **kwargs)
             logger.info(f"end to run: {func.__name__}")
             return x
-        
+
         return _in_wrapper
-    
+
     return _out_wrapper
 
 
@@ -31,7 +49,7 @@ class SingleBenchmarkCompareStandard:
                        torch.bfloat16: 2 ** -7,
                        torch.float32: 2 ** -14,
                        torch.float64: 2 ** -14}
-        
+
     def get_error_thd(self, dtype):
         if dtype in self.error_thd.keys():
             if dtype == torch.float64:
@@ -42,12 +60,12 @@ class SingleBenchmarkCompareStandard:
             "in fp16, bf16, fp32. "
         )
         return None
-    
+
     def get_eb_thd(self, dtype):
         if dtype in self.eb_thd.keys():
             return self.eb_thd.get(dtype)
         return None
-    
+
 
 class SingleBenchmarkAccuracyResult:
     def __init__(
@@ -82,7 +100,7 @@ class SingleBenchmarkAccuracyCompare:
     @func_log_wrapper()
     def check_output_size(cls, npu_out, bench_out):
         acc_result = None
-        if npu_out.numel() == 0 and bench_out.nuimel() == 0:
+        if npu_out.numel() == 0 and bench_out.numel() == 0:
             info = (
                 "The npu_output is [], and it is same as benchmark_output, "
                 "the result of data_compare is Pass"
@@ -99,14 +117,14 @@ class SingleBenchmarkAccuracyCompare:
             logging.error(error_info)
             acc_result = SingleBenchmarkAccuracyResult(result=False)
         return acc_result
-    
+
     @classmethod
     @func_log_wrapper()
     def check_output_invalid_value(cls, output):
         has_nan = torch.isnan(output).any()
         has_inf = torch.isinf(output).any()
         return has_nan or has_inf
-    
+
     @classmethod
     @func_log_wrapper()
     def precision_compare_for_case(cls, npu_out, bench_out, benchmark_standard: SingleBenchmarkCompareStandard):
@@ -119,19 +137,19 @@ class SingleBenchmarkAccuracyCompare:
         if acc_result:
             failed_info = "比对数据的shape不一致"
             return CompareResultInfo(acc_result, error_thd, eb_thd, failed_info)
-        
+
         if cls.check_output_invalid_value(bench_out):
             logging.info("The benchmark result contains nan/inf value. ")
             failed_info = "标杆结果存在nan值或inf值, 依照单标杆标准该用例通过"
             acc_result = SingleBenchmarkAccuracyResult(result=True)
             return CompareResultInfo(acc_result, error_thd, eb_thd, failed_info)
-        
+
         if cls.check_output_invalid_value(npu_out):
             logging.info("The NPU result contains nan/inf value. ")
             failed_info = "NPU结果存在nan值或inf值, 依照单标杆标准该用例不通过"
             acc_result = SingleBenchmarkAccuracyResult(result=False)
             return CompareResultInfo(acc_result, error_thd, eb_thd, failed_info)
-        
+
         data_type = npu_out.dtype
         if data_type not in [torch.float16, torch.float32, torch.float64, torch.bfloat16]:
             acc_result = cls.compute_binary_diff(npu_out, bench_out)
@@ -159,7 +177,6 @@ class SingleBenchmarkAccuracyCompare:
                 acc_result.get_result(eb_thd, error_thd)
         return CompareResultInfo(acc_result, error_thd, eb_thd, None)
 
-
     @classmethod
     @func_log_wrapper()
     def compute_binary_diff(cls, npu_out, bench_out):
@@ -167,7 +184,7 @@ class SingleBenchmarkAccuracyCompare:
         if result:
             logger.info("二进制精度比对通过, 无需单标杆比对法验证")
         return SingleBenchmarkAccuracyResult(result=result, max_abs_diff=0, max_rel_diff=0, error_balance=0)
-    
+
     @classmethod
     @func_log_wrapper()
     def compute_error_balance(cls, npu_out, bench_out, benchmark_standard: SingleBenchmarkCompareStandard):
@@ -176,11 +193,11 @@ class SingleBenchmarkAccuracyCompare:
         abs_mask_idx = torch.where(torch.abs(bench_out) < benchmark_standard.small_value, ones, zeros)
         abs_mask_idx = abs_mask_idx.type(torch.bool)
         diff_value = torch.subtract(npu_out, bench_out)
-        diff_value_rel = diff_value / (torch.abs(bench_out) + torch.finfo(torch.float).eps )
+        diff_value_rel = diff_value / (torch.abs(bench_out) + torch.finfo(torch.float).eps)
         rel_and_abs = torch.where(abs_mask_idx, diff_value, diff_value_rel)
         eb_float = float(torch.mean(rel_and_abs))
         return eb_float
-    
+
     @classmethod
     @func_log_wrapper()
     def compute_abs_diff(cls, npu_out, bench_out, error_thd, benchmark_standard: SingleBenchmarkCompareStandard):
@@ -200,15 +217,16 @@ class SingleBenchmarkAccuracyCompare:
             err_for_max = torch.where(abs_err_idx == 1, diff_abs, zeros)
             logging.debug("err_for_max for abs %s", err_for_max)
             max_abs_idx = torch.argmax(err_for_max)
-            max_abs_diff = diff_abs[max_abs_idx]
+            if check_idx_valid(diff_abs, max_abs_idx):
+                max_abs_diff = diff_abs[max_abs_idx]
         elif torch.sum(abs_mask_idx) > 0:
             err_for_max = torch.where(abs_mask_idx == 1, diff_abs, zeros)
             logging.debug("error_for_max for abs %s", err_for_max)
             max_abs_idx = torch.argmax(err_for_max)
-            if err_for_max.max() != 0:
+            if err_for_max.max() != 0 and check_idx_valid(diff_abs, max_abs_idx):
                 max_abs_diff = diff_abs[max_abs_idx]
         return (float(max_abs_diff), int(max_abs_idx) if torch.is_tensor(max_abs_idx) else max_abs_idx)
-    
+
     @classmethod
     @func_log_wrapper()
     def compute_rel_diff(cls, npu_out, bench_out, error_thd, benchmark_standard: SingleBenchmarkCompareStandard):
@@ -221,7 +239,7 @@ class SingleBenchmarkAccuracyCompare:
         diff_abs = torch.abs(diff_value)
 
         rel_mask_idx = torch.where(torch.abs(bench_out) >= benchmark_standard.small_value, ones, zeros)
-        rel_err = diff_abs / (torch.abs(bench_out) + torch.finfo(torch.float).eps )
+        rel_err = diff_abs / (torch.abs(bench_out) + torch.finfo(torch.float).eps)
         diff_rel = rel_err
         rel_err_idx = torch.where(rel_err > error_thd, ones, zeros)
         rel_err_idx = rel_err_idx * rel_mask_idx
@@ -230,19 +248,20 @@ class SingleBenchmarkAccuracyCompare:
             err_for_max = torch.where(rel_err_idx == 1, diff_rel, zeros)
             logging.debug("error_for_max for rel %s", err_for_max)
             max_rel_idx = torch.argmax(err_for_max)
-            max_rel_diff = diff_rel[max_rel_idx]
+            if check_idx_valid(diff_rel, max_rel_idx):
+                max_rel_diff = diff_rel[max_rel_idx]
         elif torch.sum(rel_mask_idx > 0):
             err_for_max = torch.where(rel_mask_idx == 1, diff_rel, zeros)
             logging.debug("err_for_max for rel %s", err_for_max)
             max_rel_idx = torch.argmax(err_for_max)
-            if torch.sum(err_for_max) != 0:
+            if torch.sum(err_for_max) != 0 and check_idx_valid(diff_rel, max_rel_idx):
                 max_rel_diff = diff_rel[max_rel_idx]
         return (float(max_rel_diff), int(max_rel_idx) if torch.is_tensor(max_rel_idx) else max_rel_idx)
 
 
 class SingleBenchSummary:
     def __init__(self, precision_result: SingleBenchmarkAccuracyResult, npu_dtype=None,
-                bench_dtype=None, shape=None, error_thd=None, eb_thd=None, failed_info=None):
+                 bench_dtype=None, shape=None, error_thd=None, eb_thd=None, failed_info=None):
         self.npu_dtype = npu_dtype
         self.bench_dtype = bench_dtype
         self.shape = shape
@@ -261,12 +280,13 @@ class SingleBenchSummary:
             return "PASS"
         else:
             return "FAILED"
-        
+
     def get_result_msg(self):
         result_str = ""
         if self.failed_info:
-            return self.failed_info
-        
+            result_str = self.failed_info
+            return result_str
+
         if self.result:
             result_str += "误差均衡性EB: %s <= 阈值%s\n" % (self.error_balance, self.eb_thd)
             result_str += "最大绝对误差: %s <= 阈值%s\n" % (self.max_abs_diff, self.error_thd)
@@ -290,7 +310,7 @@ class SingleBenchSummary:
                     self.max_rel_diff,
                 )
         return result_str
-    
+
     def print_detail_table(self):
         table = PrettyTable()
         table.title = "Single Benchmark Metrics Info"
@@ -307,7 +327,7 @@ class SingleBenchSummary:
         return [self.bench_dtype, self.npu_dtype, self.shape, self.error_balance,
                 self.max_abs_diff, self.max_abs_idx, self.max_rel_diff, self.max_rel_idx,
                 self.eb_thd, self.error_thd, self.result, self.failed_info]
-    
+
 
 def single_benchmark_compare(npu_out: torch.Tensor, bench_out: torch.Tensor, high_precision: bool = True):
     benchmark_standard = SingleBenchmarkCompareStandard(high_precision)
@@ -322,8 +342,9 @@ def single_benchmark_compare(npu_out: torch.Tensor, bench_out: torch.Tensor, hig
         failed_info
     ) = (compare_results.accuracy_result, compare_results.error_threshold,
          compare_results.eb_threshold, compare_results.failed_information)
-    
-    summary = SingleBenchSummary(precision_result, str(npu_out.dtype), str(bench_out.dtype), tuple(npu_out.shape), error_thd, eb_thd, failed_info)
+
+    summary = SingleBenchSummary(precision_result, str(npu_out.dtype), str(bench_out.dtype), tuple(npu_out.shape),
+                                 error_thd, eb_thd, failed_info)
     result = summary.result
     details = summary.to_column_value()
     return result, details
@@ -349,7 +370,7 @@ def calc_status_details_dict(npu_out, bench_out, summary):
         summary.failed_info = "bench and npu_output dict keys are different."
         return False, summary.to_column_value()
     else:
-        status, details = single_benchmark_compare_wrap(list(bench_out.values(), list(npu_out.values())))
+        status, details = single_benchmark_compare_wrap(list(bench_out.values()), list(npu_out.values()))
         return status, details
 
 

msprobe/pytorch/online_dispatch/torch_ops_config.yaml

@@ -49,7 +49,7 @@ aten_ops_blacklist:
   - zeros
   - zeros_like
 
-npu_adjust_autogard:
+npu_adjust_autograd:
   - adaptive_avg_pool2d
   - batch_norm
   - log_softmax

msprobe/pytorch/online_dispatch/utils.py

@@ -1,7 +1,23 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import inspect
+
+import numpy as np
 import psutil
 import torch
-import numpy as np
 
 try:
     import torch_npu
@@ -11,6 +27,7 @@ else:
     pta_cpu_device = torch.device("cpu")
 
 from msprobe.core.common.const import CompareConst
+from msprobe.pytorch.common.log import logger
 
 cpu_device = torch._C.device("cpu")
 COLOR_RED = '\033[31m'
@@ -31,24 +48,26 @@ COMPARE_LOGO = '''
                                                  |_|    
 '''
 
-CSV_COLUMN_NAME = [CompareConst.NPU_NAME,
-                   CompareConst.BENCH_NAME,
-                   CompareConst.NPU_DTYPE,
-                   CompareConst.BENCH_DTYPE,
-                   CompareConst.NPU_SHAPE,
-                   CompareConst.BENCH_SHAPE,
-                   CompareConst.NPU_MAX,
-                   CompareConst.NPU_MIN,
-                   CompareConst.NPU_MEAN,
-                   CompareConst.BENCH_MAX,
-                   CompareConst.BENCH_MIN,
-                   CompareConst.BENCH_MEAN,
-                   CompareConst.COSINE,
-                   CompareConst.MAX_ABS_ERR,
-                   CompareConst.MAX_RELATIVE_ERR,
-                   CompareConst.ACCURACY,
-                   CompareConst.STACK,
-                   CompareConst.ERROR_MESSAGE]
+CSV_COLUMN_NAME = [
+    CompareConst.NPU_NAME,
+    CompareConst.BENCH_NAME,
+    CompareConst.NPU_DTYPE,
+    CompareConst.BENCH_DTYPE,
+    CompareConst.NPU_SHAPE,
+    CompareConst.BENCH_SHAPE,
+    CompareConst.NPU_MAX,
+    CompareConst.NPU_MIN,
+    CompareConst.NPU_MEAN,
+    CompareConst.BENCH_MAX,
+    CompareConst.BENCH_MIN,
+    CompareConst.BENCH_MEAN,
+    CompareConst.COSINE,
+    CompareConst.MAX_ABS_ERR,
+    CompareConst.MAX_RELATIVE_ERR,
+    CompareConst.ACCURACY,
+    CompareConst.STACK,
+    CompareConst.ERROR_MESSAGE
+]
 
 FLOAT_TYPE = [np.half, np.single, float, np.double, np.float64, np.longdouble, np.float32, np.float16]
 BOOL_TYPE = [bool, np.uint8]
@@ -58,8 +77,11 @@ INT_TYPE = [np.int32, np.int64]
 def get_callstack():
     callstack = []
     for (_, path, line, func, code, _) in inspect.stack()[2:]:
-        stack_line = [path, str(line), func, code[0].strip() if code else code]
-        callstack.append(stack_line)
+        try:
+            stack_line = [path, str(line), func, code[0].strip() if code else code]
+            callstack.append(stack_line)
+        except IndexError:
+            logger.error("Failed to get callstack for code:{} index out of range".format(code))
     return callstack
 
 
@@ -125,3 +147,9 @@ class DispatchException(Exception):
 
     def __str__(self):
         return self.err_msg
+
+
+def check_idx_valid(data, idx):
+    if data is not None and data.numel() > 0 and 0 <= idx < data.numel():
+        return True
+    return False