mindspore 1.10.0__cp38-cp38-win_amd64.whl → 2.0.0rc1__cp38-cp38-win_amd64.whl

mindspore/profiler/parser/framework_parser.py

@@ -22,8 +22,9 @@ from pathlib import Path
 from typing import List
 from collections import defaultdict
 from collections import namedtuple
-
+import glob
 import numpy as np
+
 from mindspore import log as logger
 from mindspore.profiler.parser.framework_struct import TASK_DESC_STRUCT, TENSOR_DATA_STRUCT, STEP_INFO_STRUCT
 from mindspore.profiler.parser.framework_enum import VmDataType, VmFormat, FileDataType, MSPROF_DIFFERENCE
@@ -34,6 +35,7 @@ from mindspore.profiler.common.exceptions.exceptions import ProfilerDirNotFoundE
 from mindspore.profiler.common.exceptions.exceptions import ProfilerFileNotFoundException
 from mindspore.profiler.common.exceptions.exceptions import ProfilerParamValueErrorException
 from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
+from mindspore.profiler.parser.profiler_info import ProfilerInfo
 
 FILE_DATA_STRUCT_DICT = {
     FileDataType.STEP_INFO.value: STEP_INFO_STRUCT,
@@ -41,7 +43,8 @@ FILE_DATA_STRUCT_DICT = {
     FileDataType.TASK_DESC_INFO.value: TASK_DESC_STRUCT
 }
 
-COL_NAMES = ['task_id', 'stream_id', 'block_dim', 'full_op_name', 'op_name', 'op_type', 'subgraph', 'op_info']
+COL_NAMES = ['task_id', 'stream_id', 'block_dim', 'full_op_name', 'op_name', 'op_type', 'subgraph', 'op_info',
+             'graph_id']
 OpData = namedtuple('OpData', field_names=COL_NAMES)
 
 
@@ -388,6 +391,7 @@ class FrameworkParser:
     def _construct_op_data_to_file(self, task_desc_info, task_id_op_attr_dict):
         """Build data written to a file."""
         all_op_data = []
+        graph_ids = set()
         for task_desc in task_desc_info:
             task_id = task_desc['taskId']
             full_op_name = task_desc['opName']
@@ -400,8 +404,11 @@ class FrameworkParser:
                              op_name=full_op_name.split('/')[-1],
                              op_type=task_desc['opType'],
                              subgraph=subgraph,
-                             op_info=json.dumps(task_id_op_attr_dict.get(combined_task_id, {})))
+                             op_info=json.dumps(task_id_op_attr_dict.get(combined_task_id, {})),
+                             graph_id=task_desc['modelId'])
+            graph_ids.add(task_desc['modelId'])
             all_op_data.append(op_data)
+        ProfilerInfo.set_graph_ids(list(graph_ids))
         return all_op_data
 
 
@@ -429,7 +436,7 @@ class GpuFrameWorkParser:
         self.activity_info_dir = []
         self.framework_info_dir = []
         self.cpu_detail_info_dir = []
-        self.gpu_detail_info_dir = []
+        self.gpu_op_type_info_dir = []
         self.op_execute_times = {}
         self.op_step_shape_info = defaultdict(list)
         self.one_step_op_time = dict()
@@ -484,23 +491,23 @@ class GpuFrameWorkParser:
 
     def get_execute_times(self):
         """Get gpu operators execute times."""
-        if self.gpu_detail_info_dir:
-            gpu_op_detail_file_path = os.path.join(self._output_path, self.gpu_detail_info_dir[0])
-            gpu_op_detail_file_path = validate_and_normalize_path(gpu_op_detail_file_path)
-            with open(gpu_op_detail_file_path, 'r') as fp:
-                op_detail_info = fp.readlines()
-                for line_info in op_detail_info[1:]:
+        if self.gpu_op_type_info_dir:
+            gpu_op_type_file_path = os.path.join(self._output_path, self.gpu_op_type_info_dir[0])
+            gpu_op_type_file_path = validate_and_normalize_path(gpu_op_type_file_path)
+            with open(gpu_op_type_file_path, 'r') as fp:
+                op_type_info = fp.readlines()
+                for line_info in op_type_info[1:]:
                     line_info = line_info.strip(' ').strip('\n').split(',')
-                    self.op_execute_times[line_info[2]] = line_info[4]
+                    self.op_execute_times[line_info[0]] = line_info[1]
 
     def get_activity_op_info(self):
         """Get op detail data."""
         all_file = os.listdir(self._output_path)
         for file_name in all_file:
-            if file_name.startswith('gpu_op_detail') and file_name.endswith(f'{self._dev_id}.csv'):
-                self.gpu_detail_info_dir.append(file_name)
-        if not self.gpu_detail_info_dir and self.activity_info_dir:
-            raise RuntimeError(f'The output file <%s> is not found.' % self.gpu_detail_info_dir)
+            if file_name.startswith('gpu_op_type') and file_name.endswith(f'{self._dev_id}.csv'):
+                self.gpu_op_type_info_dir.append(file_name)
+        if not self.gpu_op_type_info_dir and self.activity_info_dir:
+            raise RuntimeError(f'The output file <%s> is not found.' % self.gpu_op_type_info_dir)
         self.get_execute_times()
         for filename in self.activity_info_dir:
             op_side = filename.split('_')[0]
@@ -511,7 +518,9 @@ class GpuFrameWorkParser:
             for line_info in activity_info[1:]:
                 line_info = line_info.strip(' ').strip('\n').replace(', ', ';').split(',')
                 op_name = line_info[2].split('/')[-1]
-                op_occurrences = int(self.op_execute_times.get(op_name))
+                # op_name: xxx-opx
+                op_type = op_name.split('-')[0]
+                op_occurrences = int(self.op_execute_times.get(op_type))
                 op_total_time = float(line_info[-4])
                 if not self.op_detail.get(op_name):
                     # line_info[4]: op_occurrences, line_info[5]: op_detail_time(us), line_info[6]: op_avg_time(us);
@@ -642,6 +651,23 @@ class GpuFrameWorkParser:
         with os.fdopen(os.open(dynamic_shape_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o660), 'w') as fp:
             json.dump(result, fp)
 
+    def get_graph_ids(self):
+        """Get gpu graph ids."""
+        gpu_framework_file = list(glob.glob(os.path.join(self._output_path,
+                                                         'gpu_framework_{}.txt'.format(self._dev_id))))
+        if not gpu_framework_file:
+            return []
+        graph_ids = set()
+        with open(gpu_framework_file[0], 'r') as f_obj:
+            framework_info = f_obj.readlines()
+        for line_info in framework_info:
+            if line_info.startswith("InitDataSetQueue"):
+                continue
+            line_info = line_info.strip(' ').strip('\n').split(';')
+            if len(line_info) > 2 and line_info[2].isdigit():
+                graph_ids.add(int(line_info[2]))
+        return list(graph_ids)
+
     def _organize_result(self, step, op_info, args):
         """Organize the results."""
         if args.get("type", "") == "cuLaunchKernel":

mindspore/profiler/parser/hccl_parser.py

@@ -182,164 +182,6 @@ class HcclParser:
         """Parse communication info."""
         self._parse_and_save(self._source_dir)
 
-    def _divide_communication_info_by_src_dst_rank(self, trace_event: list):
-        """Divide information by src rank id and dst rank id"""
-        src_dst_dict = dict()
-        for item in trace_event:
-            src_rank = item.get("args").get("src rank")
-            dst_rank = item.get("args").get("dst rank")
-            if src_rank is None or dst_rank is None:
-                continue
-
-            # When the SDMA operation is in the card,
-            # the source card or destination card is 0xffffffff, and it needs to be converted to localrank.
-            if int(src_rank) == int('0xffffffff', 16):
-                src_rank = dst_rank
-
-            if int(dst_rank) == int('0xffffffff', 16):
-                dst_rank = src_rank
-
-            if item.get("args").get("transport type") == CommunicationInfo.LOCAL.value:
-                item["args"]["src rank"] = dst_rank
-                item["args"]["dst rank"] = src_rank
-                src_dst_key = str(dst_rank) + '-' + str(src_rank)
-            else:
-                src_dst_key = str(src_rank) + '-' + str(dst_rank)
-
-            if src_dst_key not in src_dst_dict.keys():
-                src_dst_dict[src_dst_key] = [item]
-            else:
-                src_dst_dict[src_dst_key].append(item)
-        return src_dst_dict
-
-    def _divide_communication_info_by_link_type(self, trace_event: list):
-        """Divide information by link type."""
-        link_type_dict = dict()
-        for item in trace_event:
-            link_type_key = item.get("args").get("transport type")
-            if link_type_key is None:
-                continue
-            if link_type_key in (CommunicationInfo.RDMA.value, CommunicationInfo.SDMA.value):
-                task_type = item.get("args").get("task type")
-                # Filter out the Notify Record operator in SDMA, because it does not transmit the actual amount of data.
-                if task_type == CommunicationInfo.NOTIFY_RECORD.value:
-                    continue
-                if link_type_dict.get(link_type_key):
-                    link_type_dict[link_type_key].append(item)
-                else:
-                    link_type_dict[link_type_key] = [item]
-            if link_type_key == CommunicationInfo.LOCAL.value:
-                if link_type_dict.get(CommunicationInfo.RDMA.value):
-                    link_type_dict[CommunicationInfo.RDMA.value].append(item)
-        return link_type_dict
-
-    def _calculate_device_communication_cost(self, src_dst_link_info: dict):
-        """Calculate notify wait time."""
-        total_communication_time = 0
-        total_wait_time = 0
-        for src_dst_value in src_dst_link_info.values():
-            for link_type_value in src_dst_value.values():
-                # time_cost:0,size_cost:1,brand_width:2,wait_time:3
-                total_communication_time += link_type_value[0]
-                if len(link_type_value) > 3:
-                    total_wait_time += link_type_value[3]
-        return total_communication_time, total_wait_time
-
-    def _parse_link_cost(self, result_dict, key, link_type_dict):
-        """Parse link cost."""
-        for link_type_key, link_type_value in link_type_dict.items():
-            if link_type_key == CommunicationInfo.RDMA.value:
-                # Divide information by thread.
-                rdma_infos = []
-                threads_dict = self._divide_communication_info_by_thread(link_type_value)
-                for thread_value in threads_dict.values():
-                    rdma_info = self._calculate_adma_link_info(thread_value)
-                    rdma_infos.append(rdma_info)
-                rdma_total_cost = np.sum(rdma_infos, axis=0).tolist()
-                result_dict[key][link_type_key] = rdma_total_cost
-            if link_type_key == CommunicationInfo.SDMA.value:
-                sdma_total_cost = self._calculate_sdma_link_info(link_type_value)
-                result_dict[key][link_type_key] = sdma_total_cost
-
-    def _calculate_src_dst_link_info(self, src_dst_dict: dict):
-        """Calculate src dst link info."""
-        result_dict = dict()
-        for k, v in src_dst_dict.items():
-            # Divide information by link type.
-            link_type_dict = self._divide_communication_info_by_link_type(v)
-            if not link_type_dict:
-                continue
-            result_dict[k] = dict()
-            self._parse_link_cost(result_dict, k, link_type_dict)
-        return result_dict
-
-    def _calculate_sdma_link_info(self, trace_event: list):
-        """
-        Calculate SDMA link info.
-
-        When the link is SDMA, the communication time of the primary link is the sum of the execution time\
-        of Reduce inline and Memcpy operators.
-        """
-        sdma_communication_time = 0
-        sdma_communication_size = 0
-
-        for item in trace_event:
-            task_type = item.get("args").get("task type")
-            if task_type in (CommunicationInfo.REDUCE_INLINE.value, CommunicationInfo.MEMCPY.value):
-                sdma_communication_time += item.get("dur", 0)
-                sdma_size = item.get("args").get("size")
-                if sdma_size:
-                    sdma_size = sdma_size if isinstance(sdma_size, int) else int(sdma_size, 16)
-                else:
-                    sdma_size = 0
-
-                sdma_communication_size += sdma_size
-
-        # The unit of sdma_bandwidth is KB/s.
-        # The unit of sdma_communication_size is k_byte and The unit of sdma_communication_time is ms.
-        sdma_communication_time = sdma_communication_time / 1e3
-        sdma_communication_size = sdma_communication_size / 1e3
-        sdma_bandwidth = sdma_communication_size / (sdma_communication_time / 1e3) \
-            if sdma_communication_size else 0
-        return [sdma_communication_time, sdma_communication_size, sdma_bandwidth]
-
-    def _calculate_communication_average_value(self, communication_info: list):
-        """Calculate communication average value."""
-        communication_info_size = len(communication_info)
-        if communication_info_size == 0:
-            return []
-        # index1: communication_cost,index2:wait_cost,index3:link_info
-        communication_cost_average = sum([i[1] for i in communication_info]) / communication_info_size
-        wait_cost_average = sum([i[2] for i in communication_info]) / communication_info_size
-        link_info = [i[3] for i in communication_info]
-        calculate_type = 'average'
-        link_average_info = HcclParser._calculate_link_value(link_info, calculate_type)
-        return [communication_cost_average, wait_cost_average, link_average_info]
-
-    def _validate_file_path(self, file_path):
-        """Validate file path."""
-        try:
-            file_path = validate_and_normalize_path(file_path)
-        except RuntimeError as err:
-            logger.warning('file path is invalid.')
-            raise ProfilerPathErrorException('file path is invalid.') from err
-        if not os.path.isfile(file_path):
-            logger.warning('The file <%s> not found.', file_path)
-            raise ProfilerFileNotFoundException(file_path)
-        return file_path
-
-    def _validate_dir_path(self, dir_path):
-        """Validate dir path."""
-        try:
-            dir_path = validate_and_normalize_path(dir_path)
-        except RuntimeError as err:
-            logger.warning('dir path is invalid.')
-            raise ProfilerPathErrorException('dir path is invalid.') from err
-        if not os.path.isdir(dir_path):
-            logger.warning('The  dir <%s> not found.', dir_path)
-            raise ProfilerDirNotFoundException(dir_path)
-        return dir_path
-
     def _parse_communication_cost(self, operators_cost_info, info, operators_dict):
         """Parse communication cost."""
         for k, v in operators_cost_info.items():
@@ -567,3 +409,161 @@ class HcclParser:
         return [step_id, mainstream_communication_operator_iter_cost[0],
                 mainstream_communication_operator_iter_cost[1],
                 total_communication_operator_iter_cost[2]]
+
+    def _divide_communication_info_by_src_dst_rank(self, trace_event: list):
+        """Divide information by src rank id and dst rank id"""
+        src_dst_dict = dict()
+        for item in trace_event:
+            src_rank = item.get("args").get("src rank")
+            dst_rank = item.get("args").get("dst rank")
+            if src_rank is None or dst_rank is None:
+                continue
+
+            # When the SDMA operation is in the card,
+            # the source card or destination card is 0xffffffff, and it needs to be converted to localrank.
+            if int(src_rank) == int('0xffffffff', 16):
+                src_rank = dst_rank
+
+            if int(dst_rank) == int('0xffffffff', 16):
+                dst_rank = src_rank
+
+            if item.get("args").get("transport type") == CommunicationInfo.LOCAL.value:
+                item["args"]["src rank"] = dst_rank
+                item["args"]["dst rank"] = src_rank
+                src_dst_key = str(dst_rank) + '-' + str(src_rank)
+            else:
+                src_dst_key = str(src_rank) + '-' + str(dst_rank)
+
+            if src_dst_key not in src_dst_dict.keys():
+                src_dst_dict[src_dst_key] = [item]
+            else:
+                src_dst_dict[src_dst_key].append(item)
+        return src_dst_dict
+
+    def _divide_communication_info_by_link_type(self, trace_event: list):
+        """Divide information by link type."""
+        link_type_dict = dict()
+        for item in trace_event:
+            link_type_key = item.get("args").get("transport type")
+            if link_type_key is None:
+                continue
+            if link_type_key in (CommunicationInfo.RDMA.value, CommunicationInfo.SDMA.value):
+                task_type = item.get("args").get("task type")
+                # Filter out the Notify Record operator in SDMA, because it does not transmit the actual amount of data.
+                if task_type == CommunicationInfo.NOTIFY_RECORD.value:
+                    continue
+                if link_type_dict.get(link_type_key):
+                    link_type_dict[link_type_key].append(item)
+                else:
+                    link_type_dict[link_type_key] = [item]
+            if link_type_key == CommunicationInfo.LOCAL.value:
+                if link_type_dict.get(CommunicationInfo.RDMA.value):
+                    link_type_dict[CommunicationInfo.RDMA.value].append(item)
+        return link_type_dict
+
+    def _calculate_device_communication_cost(self, src_dst_link_info: dict):
+        """Calculate notify wait time."""
+        total_communication_time = 0
+        total_wait_time = 0
+        for src_dst_value in src_dst_link_info.values():
+            for link_type_value in src_dst_value.values():
+                # time_cost:0,size_cost:1,brand_width:2,wait_time:3
+                total_communication_time += link_type_value[0]
+                if len(link_type_value) > 3:
+                    total_wait_time += link_type_value[3]
+        return total_communication_time, total_wait_time
+
+    def _parse_link_cost(self, result_dict, key, link_type_dict):
+        """Parse link cost."""
+        for link_type_key, link_type_value in link_type_dict.items():
+            if link_type_key == CommunicationInfo.RDMA.value:
+                # Divide information by thread.
+                rdma_infos = []
+                threads_dict = self._divide_communication_info_by_thread(link_type_value)
+                for thread_value in threads_dict.values():
+                    rdma_info = self._calculate_adma_link_info(thread_value)
+                    rdma_infos.append(rdma_info)
+                rdma_total_cost = np.sum(rdma_infos, axis=0).tolist()
+                result_dict[key][link_type_key] = rdma_total_cost
+            if link_type_key == CommunicationInfo.SDMA.value:
+                sdma_total_cost = self._calculate_sdma_link_info(link_type_value)
+                result_dict[key][link_type_key] = sdma_total_cost
+
+    def _calculate_src_dst_link_info(self, src_dst_dict: dict):
+        """Calculate src dst link info."""
+        result_dict = dict()
+        for k, v in src_dst_dict.items():
+            # Divide information by link type.
+            link_type_dict = self._divide_communication_info_by_link_type(v)
+            if not link_type_dict:
+                continue
+            result_dict[k] = dict()
+            self._parse_link_cost(result_dict, k, link_type_dict)
+        return result_dict
+
+    def _calculate_sdma_link_info(self, trace_event: list):
+        """
+        Calculate SDMA link info.
+
+        When the link is SDMA, the communication time of the primary link is the sum of the execution time\
+        of Reduce inline and Memcpy operators.
+        """
+        sdma_communication_time = 0
+        sdma_communication_size = 0
+
+        for item in trace_event:
+            task_type = item.get("args").get("task type")
+            if task_type in (CommunicationInfo.REDUCE_INLINE.value, CommunicationInfo.MEMCPY.value):
+                sdma_communication_time += item.get("dur", 0)
+                sdma_size = item.get("args").get("size")
+                if sdma_size:
+                    sdma_size = sdma_size if isinstance(sdma_size, int) else int(sdma_size, 16)
+                else:
+                    sdma_size = 0
+
+                sdma_communication_size += sdma_size
+
+        # The unit of sdma_bandwidth is KB/s.
+        # The unit of sdma_communication_size is k_byte and The unit of sdma_communication_time is ms.
+        sdma_communication_time = sdma_communication_time / 1e3
+        sdma_communication_size = sdma_communication_size / 1e3
+        sdma_bandwidth = sdma_communication_size / (sdma_communication_time / 1e3) \
+            if sdma_communication_size else 0
+        return [sdma_communication_time, sdma_communication_size, sdma_bandwidth]
+
+    def _calculate_communication_average_value(self, communication_info: list):
+        """Calculate communication average value."""
+        communication_info_size = len(communication_info)
+        if communication_info_size == 0:
+            return []
+        # index1: communication_cost,index2:wait_cost,index3:link_info
+        communication_cost_average = sum([i[1] for i in communication_info]) / communication_info_size
+        wait_cost_average = sum([i[2] for i in communication_info]) / communication_info_size
+        link_info = [i[3] for i in communication_info]
+        calculate_type = 'average'
+        link_average_info = HcclParser._calculate_link_value(link_info, calculate_type)
+        return [communication_cost_average, wait_cost_average, link_average_info]
+
+    def _validate_file_path(self, file_path):
+        """Validate file path."""
+        try:
+            file_path = validate_and_normalize_path(file_path)
+        except RuntimeError:
+            logger.warning('file path is invalid.')
+            raise ProfilerPathErrorException('file path is invalid.')
+        if not os.path.isfile(file_path):
+            logger.warning('The file <%s> not found.', file_path)
+            raise ProfilerFileNotFoundException(file_path)
+        return file_path
+
+    def _validate_dir_path(self, dir_path):
+        """Validate dir path."""
+        try:
+            dir_path = validate_and_normalize_path(dir_path)
+        except RuntimeError:
+            logger.warning('dir path is invalid.')
+            raise ProfilerPathErrorException('dir path is invalid.')
+        if not os.path.isdir(dir_path):
+            logger.warning('The  dir <%s> not found.', dir_path)
+            raise ProfilerDirNotFoundException(dir_path)
+        return dir_path

mindspore/profiler/parser/hwts_log_parser.py

@@ -30,8 +30,9 @@ class HWTSLogParser:
          output_filename (str): The output data path and name. Such as: './output_format_data_hwts_0.txt'.
     """
 
+    GRAPH_MODE_MAX_TASKID = 65000
     _source_file_target_old = 'hwts.log.data.45.dev.profiler_default_tag'
-    _source_file_target = 'hwts.data'
+    _source_file_target = 'hwts.data.'
     _dst_file_title = 'title:45 HWTS data'
     _dst_file_column_title = 'Type           cnt  Core_ID  Block_ID  Task_ID  Cycle_counter   Stream_ID'
 
@@ -65,7 +66,7 @@ class HWTSLogParser:
                 if not line.strip():
                     continue
                 if len(line) < 64:
-                    logger.warning("Length of hwts data is less than 64")
+                    logger.error("Length of hwts data is less than 64, it is %s", len(line))
                     continue
                 byte_first_four = struct.unpack('BBHHH', line[0:8])
                 byte_first = bin(byte_first_four[0]).replace('0b', '').zfill(8)
@@ -90,10 +91,10 @@ class HWTSLogParser:
                     logger.info("Profiling: invalid hwts log record type %s", ms_type)
                     continue
 
-                if task_id < task_id_threshold:
-                    if last_task_stream_map.get(stream_id, task_id) > task_id and self._dynamic_status:
-                        flip_times += 1
-                    task_id_str = str(stream_id) + "_" + str(task_id + flip_times * task_id_threshold)
+                if HWTSLogParser.GRAPH_MODE_MAX_TASKID < last_task_stream_map.get(stream_id, task_id) \
+                        and task_id < last_task_stream_map.get(stream_id, task_id):
+                    flip_times += 1
+                task_id_str = ("%s_%s" % (str(stream_id), str(task_id + flip_times * task_id_threshold)))
                 result_data += ("%-14s %-4s %-8s %-9s %-8s %-15s %s\n" % (log_type[int(ms_type, 2)], cnt, core_id,
                                                                           blk_id, task_id_str, syscnt, stream_id))
                 last_task_stream_map[stream_id] = task_id