mindspore 2.1.0__cp38-cp38-manylinux1_x86_64.whl → 2.2.11__cp38-cp38-manylinux1_x86_64.whl

mindspore/parallel/checkpoint_transform.py

@@ -21,6 +21,7 @@ import copy
 from collections import defaultdict
 import numpy as np
 import mindspore as ms
+from mindspore.common import dtype as mstype
 from mindspore.parallel._parallel_serialization import _rank_list_for_transform_parallel_checkpoint, \
     _transform_parallel_checkpoint, _get_device_num_from_strategy, _make_dir, \
     _extract_layout_map, _extract_src_dst_layout_map, _parameter_not_in_local_stage, _extract_pipeline_stage_num, \
@@ -35,8 +36,7 @@ def merge_pipeline_strategys(src_strategy_dirs, dst_strategy_file):
     """
     Merge parallel strategy between all pipeline stages in pipeline parallel mode.
     For more details about converting distributed Checkpoint, please refer to
-    `Distributed Resilience Training and
-    Inference <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/resilience_train_and_predict.html>`_.
+    `Model Transformation <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/model_transformation.html>`_.
 
     Note:
         Strategy file of each pipeline stage should be included in src_strategy_dirs.
@@ -76,17 +76,16 @@ def rank_list_for_transform(rank_id, src_strategy_file=None, dst_strategy_file=N
     """
     List of original distributed checkpoint rank index for obtaining the target checkpoint of a rank_id
     during the distributed checkpoint conversion. For more details about converting distributed Checkpoint,
-    please refer to `Distributed Resilience Training and
-    Inference <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/resilience_train_and_predict.html>`_.
+    please refer to `Model Transformation <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/model_transformation.html>`_.
 
     Args:
         rank_id (int): The rank of which distributed checkpoint needs to be obtained after conversion.
         src_strategy_file (str): Name of source sharding strategy file which saved by
-                                 'mindspore.set_auto_parallel_context(strategy_ckpt_save_file)'.
+                                 `mindspore.set_auto_parallel_context(strategy_ckpt_save_file)`.
                                  when the 'src_strategy_file' is None, it means that the source sharding strategy is
                                  without any sharing for each parameter. Default:None.
         dst_strategy_file (str): Name of destination sharding strategy file which saved by
-                                 'mindspore.set_auto_parallel_context(strategy_ckpt_save_file)'.
+                                 `mindspore.set_auto_parallel_context(strategy_ckpt_save_file)`.
                                  when the 'dst_strategy_file' is None, it means that the destination sharding strategy
                                  is without any sharing for each parameter. Default:None.
 
@@ -139,8 +138,7 @@ def transform_checkpoint_by_rank(rank_id, checkpoint_files_map, save_checkpoint_
     """
     Transform distributed checkpoint from source sharding strategy to destination sharding strategy by rank
     for a network. For more details about converting distributed Checkpoint, please refer to
-    `Distributed Resilience Training and
-    Inference <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/resilience_train_and_predict.html>`_.
+    `Model Transformation <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/model_transformation.html>`_.
 
     Args:
         rank_id (int): The rank of which distributed checkpoint needs to be obtained after conversion.
@@ -195,6 +193,7 @@ def transform_checkpoint_by_rank(rank_id, checkpoint_files_map, save_checkpoint_
             raise ValueError("Checkpoint file {} in rank {} not exits: ".format(local_file, rank))
     param_total_dict = defaultdict(dict)
     param_attr_dict = defaultdict(dict)
+    param_type_dict = defaultdict(dict)
     src_strategy_list, dst_strategy_list = _extract_src_dst_layout_map(rank_id, src_strategy_file, dst_strategy_file)
     # src rank => local rank inside pipeline stage
     src_stage_device_num = np.prod(src_strategy_list.get(list(src_strategy_list.keys())[0])[0]) if src_strategy_list \
@@ -211,11 +210,15 @@ def transform_checkpoint_by_rank(rank_id, checkpoint_files_map, save_checkpoint_
                     and _parameter_not_in_local_stage(param_name, origin_dst_strategy_list, dst_strategy_list):
                 continue
             src_rank = rank % src_stage_device_num
+            param_type_dict[param_name][src_rank] = str(param.data.dtype)
+            if param.data.dtype == mstype.bfloat16:
+                param.set_dtype(mstype.float32)
             param_total_dict[param_name][src_rank] = param.data.asnumpy()
             param_attr_dict[param_name][src_rank] = (param.requires_grad, param.layerwise_parallel)
     local_rank_id = rank_id % dst_stage_device_num
     transform_param_list = _transform_parallel_checkpoint(local_rank_id, param_total_dict,
-                                                          param_attr_dict, src_strategy_list, dst_strategy_list)
+                                                          param_attr_dict, src_strategy_list, dst_strategy_list,
+                                                          param_type_dict)
     ms.save_checkpoint(transform_param_list, save_checkpoint_file_name)
 
 
@@ -224,8 +227,7 @@ def transform_checkpoints(src_checkpoints_dir, dst_checkpoints_dir, ckpt_prefix,
     """
     Transform distributed checkpoint from source sharding strategy to destination sharding strategy for a rank.
     For more details about converting distributed Checkpoint, please refer to
-    `Distributed Resilience Training and
-    Inference <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/resilience_train_and_predict.html>`_.
+    `Model Transformation <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/model_transformation.html>`_.
 
     Note:
         The `src_checkpoints_dir` directory structure should be organized like "src_checkpoints_dir/rank_0/a.ckpt", the
@@ -301,11 +303,15 @@ def transform_checkpoints(src_checkpoints_dir, dst_checkpoints_dir, ckpt_prefix,
     for needed_rank_list_key, transform_rank_list in needed_rank_list_map.items():
         param_total_dict = defaultdict(dict)
         param_attr_dict = defaultdict(dict)
+        param_type_dict = defaultdict(dict)
         needed_rank_list = needed_rank_list_key.split("-")
         for needed_rank in needed_rank_list:
             ckpt_dict = ms.load_checkpoint(all_checkpoint_files_map.get(int(needed_rank)))
             for param_name, param in ckpt_dict.items():
                 src_rank = int(needed_rank) % src_stage_device_num
+                param_type_dict[param_name][src_rank] = str(param.data.dtype)
+                if param.data.dtype == mstype.bfloat16:
+                    param.set_dtype(mstype.float32)
                 param_total_dict[param_name][src_rank] = param.data.asnumpy()
                 param_attr_dict[param_name][src_rank] = (param.requires_grad, param.layerwise_parallel)
         for transform_rank in transform_rank_list:
@@ -320,7 +326,8 @@ def transform_checkpoints(src_checkpoints_dir, dst_checkpoints_dir, ckpt_prefix,
 
             local_rank_id = transform_rank % dst_stage_device_num
             transform_param_list = _transform_parallel_checkpoint(local_rank_id, param_total_dict_copy,
-                                                                  param_attr_dict, src_strategy_list, dst_strategy_list)
+                                                                  param_attr_dict, src_strategy_list, dst_strategy_list,
+                                                                  param_type_dict)
             save_checkpoint_file = "{}{}.ckpt".format(ckpt_prefix, transform_rank)
             save_checkpoint_file_dir = os.path.join(dst_checkpoints_dir, "rank_{}".format(transform_rank))
             if not os.path.exists(save_checkpoint_file_dir):

mindspore/parallel/shard.py

@@ -24,7 +24,7 @@ class Shard(Shard_):
 
     def __init__(self):
         """Initialize Shard."""
-        Shard_.__init__(self, 'Shard')
+        super().__init__('Shard')
         self.shard_fn = None
         self.fn = None
         self.in_strategy = None
@@ -36,13 +36,17 @@ class Shard(Shard_):
     def __call__(self, fn, in_strategy, out_strategy=None, parameter_plan=None, device="Ascend", level=0):
         if ms.context.get_context("mode") != ms.context.PYNATIVE_MODE or \
                 ms.context.get_auto_parallel_context("parallel_mode") not in ["auto_parallel"]:
-            raise AssertionError(f"Cell shard only supports auto parallel under PyNative mode.")
+            raise AssertionError(
+                f"Cell shard only supports auto parallel under PyNative mode.")
         if ms.context.get_context("device_target") not in ["Ascend", "GPU"]:
-            raise AssertionError(f"'Shard' now only supports 'Ascend' and 'GPU'")
+            raise AssertionError(
+                f"'Shard' now only supports 'Ascend' and 'GPU'")
         if ms.context.get_auto_parallel_context("search_mode") != "sharding_propagation":
-            raise AssertionError(f"'search_mode' must be 'sharding_propagation' for 'Shard'")
+            raise AssertionError(
+                f"'search_mode' must be 'sharding_propagation' for 'Shard'")
         if not isinstance(in_strategy, tuple):
-            raise TypeError(f"For 'Shard', the 'in_strategy' should be a tuple, but got {type(in_strategy).__name__}")
+            raise TypeError(
+                f"For 'Shard', the 'in_strategy' should be a tuple, but got {type(in_strategy).__name__}")
         if not isinstance(out_strategy, (type(None), tuple)):
             raise TypeError(f"For 'Shard', the 'out_strategy' should be None or tuple, "
                             f"but got {type(out_strategy).__name__}")
@@ -117,7 +121,8 @@ class Shard(Shard_):
             return
         if isinstance(parameter_plan, dict):
             if not isinstance(fn, ms.nn.Cell):
-                raise TypeError(f"If parameter_plan is set, type of fn must be mindspore.nn.Cell, but got {type(fn)}")
+                raise TypeError(
+                    f"If parameter_plan is set, type of fn must be mindspore.nn.Cell, but got {type(fn)}")
             for k in parameter_plan.keys():
                 v = parameter_plan[k]
                 if not isinstance(k, str) or not isinstance(v, tuple):
@@ -131,10 +136,12 @@ class Shard(Shard_):
             param_strategy = parameter_plan[param_name]
             param = self._search_parameter_by_name(param_name, fn)
             if param is None:
-                logger.warning(f"{param_name} is not exist, ignored its setting.")
+                logger.warning(
+                    f"{param_name} is not exist, ignored its setting.")
                 continue
 
-            self._check_layout_is_valid(param_name, param.shape, param_strategy)
+            self._check_layout_is_valid(
+                param_name, param.shape, param_strategy)
             if param.param_info.param_strategy:
                 logger.warning(f"The layout of parameter '{param_name}' "
                                f"has been set to {param.param_info.param_strategy}, "
@@ -143,7 +150,7 @@ class Shard(Shard_):
 
     def _is_attrs_has_been_set(self, fn, in_strategy, out_strategy, device, level):
         return self.shard_fn is not None and self.fn == fn and self.in_strategy == in_strategy and \
-               self.out_strategy == out_strategy and self.device == device and self.level == level
+            self.out_strategy == out_strategy and self.device == device and self.level == level
 
 
 def shard(fn, in_strategy, out_strategy=None, parameter_plan=None, device="Ascend", level=0):
@@ -159,8 +166,8 @@ def shard(fn, in_strategy, out_strategy=None, parameter_plan=None, device="Ascen
 
     Note:
         You need to set the execution mode to PyNative mode,
-        set the parallel mode in `set_auto_parallel_context` to "auto_parallel"
-        and the search mode to "sharding_propagation".
+        set the parallel mode in `set_auto_parallel_context` (parallel_mode) to "auto_parallel"
+        and the search mode (search_mode) to "sharding_propagation".
         If the input contain Parameter, its strategy should be set in `in_strategy`.
 
     Args:
@@ -216,15 +223,15 @@ def shard(fn, in_strategy, out_strategy=None, parameter_plan=None, device="Ascen
         ...                           device_num=2)
         >>> def test_shard(x, y):
         ...     return x + y
-        >>> x = Tensor(np.ones(shape=(32, 10)))
-        >>> y = Tensor(np.ones(shape=(32, 10)))
+        >>> x = Tensor(np.ones(shape=(32, 10)), dtype=ms.float32)
+        >>> y = Tensor(np.ones(shape=(32, 10)), dtype=ms.float32)
         >>> output = ms.shard(test_shard, in_strategy=((2, 1), (2, 1)))(x, y)
         >>> print(output.shape)
         (32, 10)
 
     Tutorial Examples:
         - `Functional Operator Sharding
-          <https://www.mindspore.cn/docs/en/r2.1/api_python/samples/mindspore/pynative_shard_function_parallel.html>`_
+          <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/pynative_shard_function_parallel.html>`_
     """
     if not isinstance(fn, (ms.nn.Cell)):
         logger.warning("'fn' is not a mindspore.nn.Cell, and its definition cannot involve Parameter; "

mindspore/profiler/common/struct_type.py

@@ -72,7 +72,7 @@ class StructType(Enum):
         """
         Parse the binary data to get the unpacked data.
 
-        Args：
+        Args:
             data_struct (dict): Key is the data name, value is StructType.
             binary_data (str): This value should be a binary string.
             special_func (Callable): This is a callable function,
@@ -105,14 +105,14 @@ class StructType(Enum):
         for name, data_type in data_struct.items():
             data_size = StructType.sizeof(data_type)
             if special_process_func:
-                unpack_data, success = special_process_func(binary_data[cursor:cursor+data_size], name,
+                unpack_data, success = special_process_func(binary_data[cursor:cursor + data_size], name,
                                                             data_type, unpacked_data)
                 if success:
                     cursor += data_size
                     unpacked_data[name] = unpack_data
                     continue
 
-            unpack_data = struct.unpack(data_type.value, binary_data[cursor: cursor+data_size])[0]
+            unpack_data = struct.unpack(data_type.value, binary_data[cursor: cursor + data_size])[0]
             cursor += data_size
             unpacked_data[name] = unpack_data
         return unpacked_data

mindspore/profiler/common/util.py

@@ -23,6 +23,8 @@ import os
 import re
 import stat
 
+from mindspore import log as logger
+
 
 def to_int(param, param_name):
     """
@@ -80,9 +82,9 @@ def fwrite_format(output_data_path, data_source=None, is_print=False, is_start=F
                 if isinstance(raw_data, (list, tuple)):
                     raw_data = map(str, raw_data)
                     raw_data = " ".join(raw_data)
-                print(raw_data)
+                logger.info(raw_data)
         else:
-            print(data_source)
+            logger.info(data_source)
 
 
 def get_log_slice_id(file_name):

mindspore/profiler/envprofiling.py

@@ -68,7 +68,7 @@ def construct_profiling_options():
         logger.error(
             "The format of MS_PROFILER_OPTIONS is incorrect. "
             "The MS_PROFILER_OPTIONS parameter configuration may refer to "
-            "'https://www.mindspore.cn/mindinsight/docs/zh-CN/master/performance_profiling_ascend.html'."
+            "'https://www.mindspore.cn/mindinsight/docs/zh-CN/r2.2/performance_profiling_ascend.html'."
         )
         return error_config
     conbine_options = combine_profile_options(profiling_options)

mindspore/profiler/parser/aicpu_data_parser.py

@@ -72,9 +72,11 @@ class DataPreProcessParser:
     _source_file_target_old = 'DATA_PREPROCESS.dev.AICPU.'
     _source_file_target = 'DATA_PREPROCESS.AICPU.'
     _dst_file_title = 'title:DATA_PREPROCESS AICPU'
-    _dst_file_column_title = ['serial_number', 'node_type_name', 'total_time(ms)',
-                              'dispatch_time(ms)', 'execution_time(ms)', 'run_start',
-                              'run_end']
+    _dst_file_column_title = [
+        'serial_number', 'node_type_name', 'total_time(ms)',
+        'dispatch_time(ms)', 'execution_time(ms)', 'run_start',
+        'run_end'
+    ]
     _ms_unit = 1000
     _us_unit = 100  # Convert 10ns to 1us.
     _task_id_threshold = 65536

mindspore/profiler/parser/ascend_flops_generator.py

@@ -78,7 +78,7 @@ class AscendFlopsGenerator:
                 writer.writerows(self.flops.tolist())
         except (IOError, OSError) as err:
             logging.critical('Errot occurred when write flops file: %s', err)
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
         if os.path.exists(flops_path):
             os.chmod(flops_path, stat.S_IREAD | stat.S_IWRITE)
 
@@ -89,6 +89,6 @@ class AscendFlopsGenerator:
                 json.dump(self.flops_summary, json_file)
         except (IOError, OSError) as err:
             logging.critical('Errot occurred when write step trace point info file: %s', err)
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
         if os.path.exists(flops_summary_path):
             os.chmod(flops_summary_path, stat.S_IREAD | stat.S_IWRITE)

mindspore/profiler/parser/ascend_fpbp_generator.py

@@ -71,6 +71,6 @@ class AscendFPBPGenerator:
                 json.dump(self.points, json_file)
         except (IOError, OSError) as err:
             logging.critical('Errot occurred when write step trace point info file: %s', err)
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
         if os.path.exists(step_trace_point_info_path):
             os.chmod(step_trace_point_info_path, stat.S_IREAD | stat.S_IWRITE)

mindspore/profiler/parser/ascend_hccl_generator.py

@@ -82,6 +82,238 @@ def count_average(data):
 class AscendHCCLGenerator:
     """Generate ascend hccl data from files."""
 
+    def __init__(self, source_path, steptrace):
+        self.root_path = source_path
+        self.steptrace = steptrace
+        self.hccl_raw = []
+        self.hccl_data_df = np.dtype(
+            [('model_id', int), ('iteration_id', int), ('name', object), ('pid', int), ('tid', int), ('ts', float),
+             ('te', float), ('dur', float), ('ph', object),
+             ('task_type', object), ('link_info', object), ('transport_type', object), ('size', int), ('tag', object)])
+
+    @staticmethod
+    def _cost_analyse(iteration):
+        """analyse communication cost and wait cost"""
+        communication_cost = np.sum(iteration[iteration['name'] != 'Notify_Wait']['dur'])
+        wait_cost = np.sum(iteration[iteration['name'] == 'Notify_Wait']['dur'])
+        return communication_cost, wait_cost
+
+    @staticmethod
+    def _rdma_analyse(groupby_transport):
+        """rdma analyse"""
+        thread_groups = np.unique(groupby_transport['tid'])
+        thread_information = []
+        for thread_index in thread_groups:
+            groupby_thread = groupby_transport[groupby_transport['tid'] == thread_index]
+            rdma_communication_time = 0
+            rdma_communication_size = 0
+            rdma_communication_wait_time = 0
+            start_index = 0
+            end_index = groupby_thread.size - 2
+            while start_index < end_index:
+                first_task_type = groupby_thread[start_index]['task_type']
+                if first_task_type == 'RDMASend':
+                    second_index = start_index + 1
+                    third_index = start_index + 2
+                    second_task_type = groupby_thread[second_index]['task_type']
+                    third_task_type = groupby_thread[third_index]['task_type']
+                    if second_task_type == 'RDMASend' and third_task_type == 'Notify Wait':
+                        rdma_send_cost = groupby_thread[start_index]['dur']
+                        notify_record_cost = groupby_thread[second_index]['dur']
+                        notify_wait_cost = groupby_thread[third_index]['dur']
+                        rdma_communication_time += rdma_send_cost + notify_record_cost + notify_wait_cost
+                        rdma_communication_wait_time += notify_wait_cost
+                        rdma_communication_size += groupby_thread[start_index]['size'] + groupby_thread[second_index][
+                            'size']
+                        start_index += 2
+                start_index += 1
+            rdma_communication_wait_time = rdma_communication_wait_time / 1e3
+            rdma_communication_size = rdma_communication_size / 1e3
+            rdma_communication_time = rdma_communication_time / 1e3
+            rdma_bandwidth = rdma_communication_size / (rdma_communication_time / 1e3) \
+                if rdma_communication_size else 0
+            thread_information.append(
+                [rdma_communication_time, rdma_communication_size, rdma_bandwidth, rdma_communication_wait_time])
+        if len(thread_information) > 1:
+            thread_information = np.sum(thread_information, axis=0).tolist()
+
+        return thread_information
+
+    def parse(self):
+        """Analyse the original hccl data generator hccl data."""
+        hccl_data = []
+        for hccl_file in find_files(self.root_path, "hccl_*.json"):
+            with open(hccl_file) as fr:
+                hccl_data.append(self._original_data_analyse(json.load(fr)))
+        hccl_data = np.concatenate(hccl_data)
+
+        for model_id in np.unique(hccl_data['model_id']):
+            hccl_data_model = hccl_data[hccl_data['model_id'] == model_id]
+            for iteration_id in np.unique(hccl_data_model['iteration_id']):
+                hccl_data_model_iteration = hccl_data_model[hccl_data_model['iteration_id'] == iteration_id]
+
+                hccl_abstract_data = hccl_data_model_iteration[hccl_data_model_iteration['task_type'] == '']
+                hccl_detail_data = hccl_data_model_iteration[hccl_data_model_iteration['task_type'] != '']
+                hccl_abstract_data = np.sort(hccl_abstract_data, order='ts')
+                hccl_detail_data = np.sort(hccl_detail_data, order='ts')
+
+                tag = np.searchsorted(hccl_abstract_data['ts'], hccl_detail_data['ts'], side='right') - 1
+
+                hccl_detail_data['tag'] = [x[-1] for x in
+                                           np.char.split(hccl_abstract_data[tag]['name'].astype(str), sep='/')]
+
+                self.hccl_raw.append(self._iteration_analyse(hccl_detail_data, iteration_id))
+
+        self.hccl_raw = sorted(self.hccl_raw, key=lambda x: x[0])
+        self.hccl_raw.append(copy.deepcopy(self.hccl_raw[-1]))
+        self.hccl_raw[-1][0] = '-'
+        for _, value in self.hccl_raw[-1][4].items():
+            value[0] = '-'
+
+    def write(self, hccl_raw_path):
+        """
+        Write the flops.csv and flops_summary.json
+
+        Args:
+            hccl_raw_path(str): hccl_raw.csv path.
+        """
+        try:
+            with os.fdopen(os.open(hccl_raw_path,
+                                   os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR), 'w',
+                           newline='') as hccl_row:
+                writer = csv.writer(hccl_row)
+                writer.writerow(
+                    ['step_num', 'communication_cost', 'wait_cost', 'link_info', 'communication_operator_cost'])
+                for row in self.hccl_raw:
+                    row[3] = json.dumps(row[3])
+                    row[4] = json.dumps(row[4])
+                writer.writerows(self.hccl_raw)
+        except (IOError, OSError) as err:
+            logging.critical('Errot occurred when write aicore detail file: %s', err)
+            raise ProfilerIOException() from err
+        if os.path.exists(hccl_raw_path):
+            os.chmod(hccl_raw_path, stat.S_IREAD | stat.S_IWRITE)
+
+    def _original_data_analyse(self, original_data):
+        """analyse original data"""
+
+        groups_steptrace = {model_id: np.sort(self.steptrace[self.steptrace['Model ID'] == model_id],
+                                              order='Iteration ID')
+                            for model_id in np.unique(self.steptrace['Model ID'])}
+
+        target_data = []
+        for row in original_data:
+            model_id = row.get('args', {}).get('model id')
+            if row.get('ph') == 'X' and model_id is not None:
+                name = row.get('name')
+                pid = row.get('pid')
+                tid = row.get('tid')
+                ts = row.get('ts')
+                dur = row.get('dur')
+                te = ts + dur
+                ph = row.get('ph')
+                task_type = row.get('args', {}).get('task type', '')
+                src_rank = row.get('args', {}).get('src rank', 0)
+                dst_rank = row.get('args', {}).get('dst rank', 0)
+                if src_rank == int('0xffffffff', 16):
+                    src_rank = dst_rank
+                if dst_rank == int('0xffffffff', 16):
+                    dst_rank = src_rank
+                transport_type = row.get('args', {}).get('transport type', '')
+                if transport_type == 'LOCAL':
+                    src_rank, dst_rank = dst_rank, src_rank
+                link_info = str(src_rank) + '-' + str(dst_rank)
+                size = row.get('args', {}).get('size(Byte)', 0)
+                size = size if isinstance(size, int) else int(size, 16)
+                steptrace = groups_steptrace.get(model_id, None)
+                if steptrace is None:
+                    logging.warning('Could not find model: %s in hccl json, skip.', model_id)
+                    continue
+                tag = np.searchsorted(steptrace['Iteration End'], te * 1e-3, side='left')
+                iteration_id = steptrace[tag]['Iteration ID']
+                target_data.append(
+                    tuple([model_id, iteration_id, name, pid, tid,
+                           ts, te, dur, ph, task_type,
+                           link_info, transport_type, size, -1]))
+
+        hccl_data = np.array(target_data, dtype=self.hccl_data_df)
+
+        return hccl_data
+
+    def _iteration_analyse(self, hccl_detail_data, iteration):
+        """analyse data by iteration """
+        communication_cost, wait_cost = self._cost_analyse(hccl_detail_data)
+        link_info = self._link_info_analyse(hccl_detail_data)
+        communication_operator_cost = self._communication_operator_cost_analyse(hccl_detail_data, iteration)
+        return [iteration, communication_cost, wait_cost, link_info, communication_operator_cost]
+
+    def _link_info_analyse(self, hccl_detail_data):
+        """analyse link info data"""
+        groupby_iteration = hccl_detail_data[hccl_detail_data['task_type'] != 'Notify Record']
+        link_info_groups = np.unique(groupby_iteration['link_info'])
+        link_info_information = dict()
+        for link_info_index in link_info_groups:
+            groupby_link_info = groupby_iteration[groupby_iteration['link_info'] == link_info_index]
+            transport_groups = np.unique(groupby_iteration['transport_type'])
+            transport_information = dict()
+            for transport_index in transport_groups:
+                groupby_transport = groupby_link_info[groupby_link_info['transport_type'] == transport_index]
+                if transport_index == 'SDMA' and groupby_transport.size > 0:
+                    groupby_sdma = \
+                        groupby_transport[np.isin(groupby_transport['task_type'], ['Memcpy', 'Reduce Inline'])][
+                            ['dur', 'size']]
+                    sdma_communication_time = np.sum(groupby_sdma['dur']) * 1e-3
+                    sdma_communication_size = np.sum(groupby_sdma['size']) * 1e-3
+                    sdma_bandwidth = sdma_communication_size / sdma_communication_time * 1e-3 \
+                        if sdma_communication_time != 0 else 0
+                    transport_information['SDMA'] = [sdma_communication_time, sdma_communication_size, sdma_bandwidth]
+                elif transport_index == 'RDMA' and groupby_transport.size > 0:
+                    transport_information['RDMA'] = self._rdma_analyse(groupby_transport)
+            link_info_information[link_info_index] = transport_information
+        return link_info_information
+
+    def _communication_operator_cost_analyse(self, hccl_detail_data, iteration_index):
+        """analyse communication operator cost"""
+        groupby_iteration = hccl_detail_data[hccl_detail_data['task_type'] != 'Notify Record']
+        tag_groups = np.unique(groupby_iteration['tag'])
+        tag_information = dict()
+        for tag_index in tag_groups:
+            groupby_tag = groupby_iteration[groupby_iteration['tag'] == tag_index]
+            link_groups = np.unique(groupby_iteration['link_info'])
+            link_info_information = dict()
+            for link_info_index in link_groups:
+                groupby_link_info = groupby_tag[groupby_tag['link_info'] == link_info_index]
+                transport_groups = np.unique(groupby_link_info['transport_type'])
+                transport_information = dict()
+                for transport_index in transport_groups:
+                    groupby_transport = groupby_link_info[groupby_link_info['transport_type'] == transport_index]
+                    if transport_index == 'SDMA':
+                        groupby_sdma = \
+                            groupby_transport[np.isin(groupby_transport['task_type'], ['Memcpy', 'Reduce Inline'])][
+                                ['dur', 'size']]
+                        sdma_communication_time = np.sum(groupby_sdma['dur']) * 1e-3
+                        sdma_communication_size = np.sum(groupby_sdma['size']) * 1e-3
+                        sdma_bandwidth = sdma_communication_size / sdma_communication_time * 1e-3 \
+                            if sdma_communication_time != 0 else 0
+                        transport_information['SDMA'] = [
+                            sdma_communication_time, sdma_communication_size,
+                            sdma_bandwidth
+                        ]
+                    elif transport_index == 'RDMA':
+                        transport_information['RDMA'] = self._rdma_analyse(groupby_transport)
+                    link_info_information[link_info_index] = transport_information
+                communication_cost = np.sum(groupby_tag[groupby_tag['name'] != 'Notify_Wait']['dur'])
+                wait_cost = np.sum(groupby_tag[groupby_tag['name'] == 'Notify_Wait']['dur'])
+                tag_information[tag_index] = [
+                    str(iteration_index), communication_cost, wait_cost,
+                    link_info_information
+                ]
+        return tag_information
+
+
+class AscendHCCLGeneratorOld:
+    """Generate ascend hccl data from files."""
+
     def __init__(self, source_path):
         self.root_path = source_path
         self.hccl_raw = []
@@ -99,7 +331,7 @@ class AscendHCCLGenerator:
     @staticmethod
     def _rdma_analyse(groupby_transport):
         """rdma analyse"""
-        thread_groups, _, _, _ = np.unique(groupby_transport['tid'])
+        thread_groups = np.unique(groupby_transport['tid'])
         thread_information = []
         for thread_index in thread_groups:
             groupby_thread = groupby_transport[groupby_transport['tid'] == thread_index]
@@ -107,7 +339,7 @@ class AscendHCCLGenerator:
             rdma_communication_size = 0
             rdma_communication_wait_time = 0
             start_index = 0
-            end_index = groupby_thread.size - 1
+            end_index = groupby_thread.size - 2
             while start_index < end_index:
                 first_task_type = groupby_thread[start_index]['task_type']
                 if first_task_type == 'RDMASend':
@@ -142,7 +374,8 @@ class AscendHCCLGenerator:
         file_list = find_files(self.root_path, "hccl_*.json")
 
         for hccl_file in file_list:
-            iteration_id = int(hccl_file.split('_')[-1].split(('.'))[0])
+            _, relative_path = os.path.split(hccl_file)
+            iteration_id = int(relative_path.split('_')[3])
             with open(hccl_file) as f:
                 _, hccl_detail_data = self._original_data_analyse(json.load(f))
                 raw = self._iteration_analyse(hccl_detail_data, iteration_id)
@@ -173,7 +406,7 @@ class AscendHCCLGenerator:
                 writer.writerows(self.hccl_raw)
         except (IOError, OSError) as err:
             logging.critical('Errot occurred when write aicore detail file: %s', err)
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
         if os.path.exists(hccl_raw_path):
             os.chmod(hccl_raw_path, stat.S_IREAD | stat.S_IWRITE)
 
@@ -206,8 +439,8 @@ class AscendHCCLGenerator:
                     tuple([name, pid, tid, ts, te, dur, ph, task_type, link_info, transport_type, size, -1]))
         hccl_data = np.array(target_data, dtype=self.hccl_data_df)
 
-        hccl_abstract_data = hccl_data[hccl_data['tid'] >= 100]
-        hccl_detail_data = hccl_data[hccl_data['tid'] < 100]
+        hccl_abstract_data = hccl_data[hccl_data['task_type'] == '']
+        hccl_detail_data = hccl_data[hccl_data['task_type'] != '']
 
         hccl_abstract_data = hccl_abstract_data[np.argsort(hccl_abstract_data['ts'])]
         hccl_detail_data = hccl_detail_data[np.argsort(hccl_detail_data['ts'])]
@@ -234,7 +467,7 @@ class AscendHCCLGenerator:
             transport_information = dict()
             for transport_index in transport_groups:
                 groupby_transport = groupby_link_info[groupby_link_info['transport_type'] == transport_index]
-                if transport_index == 'SDMA':
+                if transport_index == 'SDMA' and groupby_transport.size > 0:
                     groupby_sdma = \
                         groupby_transport[np.isin(groupby_transport['task_type'], ['Memcpy', 'Reduce Inline'])][
                             ['dur', 'size']]
@@ -243,7 +476,7 @@ class AscendHCCLGenerator:
                     sdma_bandwidth = sdma_communication_size / sdma_communication_time * 1e-3 \
                         if sdma_communication_time != 0 else 0
                     transport_information['SDMA'] = [sdma_communication_time, sdma_communication_size, sdma_bandwidth]
-                elif transport_index == 'RDMA':
+                elif transport_index == 'RDMA' and groupby_transport.size > 0:
                     transport_information['RDMA'] = self._rdma_analyse(groupby_transport)
             link_info_information[link_info_index] = transport_information
         return link_info_information
@@ -271,13 +504,17 @@ class AscendHCCLGenerator:
                         sdma_communication_size = np.sum(groupby_sdma['size']) * 1e-3
                         sdma_bandwidth = sdma_communication_size / sdma_communication_time * 1e-3 \
                             if sdma_communication_time != 0 else 0
-                        transport_information['SDMA'] = [sdma_communication_time, sdma_communication_size,
-                                                         sdma_bandwidth]
+                        transport_information['SDMA'] = [
+                            sdma_communication_time, sdma_communication_size,
+                            sdma_bandwidth
+                        ]
                     elif transport_index == 'RDMA':
                         transport_information['RDMA'] = self._rdma_analyse(groupby_transport)
                     link_info_information[link_info_index] = transport_information
                 communication_cost = np.sum(groupby_tag[groupby_tag['name'] != 'Notify_Wait']['dur'])
                 wait_cost = np.sum(groupby_tag[groupby_tag['name'] == 'Notify_Wait']['dur'])
-                tag_information[tag_index] = [str(iteration_index), communication_cost, wait_cost,
-                                              link_info_information]
+                tag_information[tag_index] = [
+                    str(iteration_index), communication_cost, wait_cost,
+                    link_info_information
+                ]
         return tag_information