mindspore 2.2.10__cp37-cp37m-manylinux1_x86_64.whl → 2.2.14__cp37-cp37m-manylinux1_x86_64.whl

mindspore/profiler/parser/ascend_communicate_generator.py

@@ -0,0 +1,315 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""communicate data analyze api file"""
+import json
+import re
+import logging
+import os
+import stat
+from collections import defaultdict
+
+from mindspore.profiler.common.exceptions.exceptions import ProfilerIOException
+
+
+class AscendCommunicationGenerator:
+    """
+    load and split communication info by step
+    """
+    COMMUNICATION_TIME_INFO = "Communication Time Info"
+    START_TIMESTAMP = "Start Timestamp(us)"
+    COMMUNICATION_BANDWIDTH_INFO = "Communication Bandwidth Info"
+    HCOM_SEND = "Send"
+    HCOM_RECEIVE = "Receive"
+    TOTAL = "Total"
+    SYNCHRONIZATION_TIME_RATIO = "Synchronization Time Ratio"
+    SYNCHRONIZATION_TIME_MS = "Synchronization Time(ms)"
+    WAIT_TIME_RATIO = "Wait Time Ratio"
+    TRANSIT_TIME_MS = "Transit Time(ms)"
+    TRANSIT_SIZE_MB = "Transit Size(MB)"
+    SIZE_DISTRIBUTION = "Size Distribution"
+    WAIT_TIME_MS = "Wait Time(ms)"
+    BANDWIDTH_GB_S = "Bandwidth(GB/s)"
+    COMMUNICATION = "communication.json"
+    COMMUNICATION_MATRIX = "communication_matrix.json"
+    P2P = "p2p"
+    COLLECTIVE = "collective"
+    TRANSPORT_TYPE = "Transport Type"
+    PATTERN1 = re.compile(r"receive|send")
+    PATTERN2 = re.compile(r"invalid|broadcast|allreduce|reduce|"
+                          r"allgather|reducescatter|scatter|alltoall|alltoallv|alltoallvc")
+
+    def __init__(self, source_path):
+        super().__init__()
+        self.root_path = source_path
+        self.step_list = [{"step_id": None, "start_ts": 0, "end_ts": float('inf'), "comm_ops": {}}]
+        self.output_communication = {}
+        self.output_matrix_data = {}
+
+    @staticmethod
+    def combine_size_distribution(op_dict: dict, total_dict: dict):
+        """combine size distribution"""
+        for size, size_info in op_dict.items():
+            total_dict[size][0] += size_info[0]
+            total_dict[size][1] += size_info[1]
+
+    @staticmethod
+    def compute_ratio(dividend: float, divisor: float):
+        """compute ratio"""
+        if abs(divisor) < 1e-15:
+            return 0
+        return round(dividend / divisor, 4)
+
+    def parse(self) -> None:
+        """parse"""
+        self.generate_communication()
+        self.generate_matrix()
+
+    def generate_communication(self):
+        """
+        generate communication.json
+        """
+        communication_file = os.path.join(self.root_path, self.COMMUNICATION)
+        with open(communication_file) as file:
+            communication_data = json.load(file)
+        if not communication_data:
+            return
+        self.split_comm_op_by_step(communication_data)
+
+        for step_info in self.step_list:
+            step = "step" + step_info.get("step_id") if step_info.get("step_id") else "step"
+            self.output_communication[step] = self.get_communication_ops_dict(step_info.get("comm_ops"))
+
+    def generate_matrix(self):
+        """generate matrix"""
+        communication_file = os.path.join(self.root_path, self.COMMUNICATION_MATRIX)
+        with open(communication_file) as file:
+            matrix_data = json.load(file)
+        if not matrix_data:
+            return
+        matrix_data_by_step = self.split_matrix_by_step(matrix_data)
+
+        for step, comm_matrix_data in matrix_data_by_step.items():
+            self.output_matrix_data[step] = self.get_matrix_ops_dict(comm_matrix_data)
+
+    def split_comm_op_by_step(self, communication_data: dict):
+        """split comm op by step"""
+        if len(self.step_list) == 1:
+            self.step_list[0]["comm_ops"] = communication_data
+        for communication_op, communication_op_info in communication_data.items():
+            start_time = communication_op_info.get(self.COMMUNICATION_TIME_INFO, {}).get(self.START_TIMESTAMP)
+            for step_info in self.step_list:
+                if step_info.get("start_ts", -1) <= start_time <= step_info.get("end_ts", -1):
+                    step_info.get("comm_ops", {})[communication_op] = communication_op_info
+                    break
+
+    def split_communication_p2p_ops(self, op_data: dict):
+        """
+        split communicate
+        """
+        comm_op_dict = {self.P2P: {}, self.COLLECTIVE: {}}
+        for communication_op, communication_info in op_data.items():
+            if communication_op.find(self.HCOM_SEND) != -1 or communication_op.find(self.HCOM_RECEIVE) != -1:
+                comm_op_dict[self.P2P][communication_op] = communication_info
+            elif communication_op.startswith(self.TOTAL):
+                continue
+            else:
+                comm_op_dict[self.COLLECTIVE][communication_op] = communication_info
+        return comm_op_dict
+
+    def split_matrix_by_step(self, matrix_data: dict) -> dict:
+        """
+        split matrix by step
+        """
+        matrix_data_by_step = {}
+        if self.is_step_list_empty():
+            matrix_data_by_step["step"] = matrix_data
+            return matrix_data_by_step
+
+        for comm_op in matrix_data:
+            for step_info in self.step_list:
+                if comm_op in step_info.get("comm_ops", {}):
+                    step = "step" + step_info.get("step_id") if step_info.get("step_id") else "step"
+                    matrix_data_by_step.setdefault(step, {})[comm_op] = matrix_data.get(comm_op)
+                    break
+        return matrix_data_by_step
+
+    def get_communication_ops_dict(self, op_data: dict) -> dict:
+        """get communication ops dict"""
+        comm_op_dict = self.split_communication_p2p_ops(op_data)
+        self.compute_total_info(comm_op_dict[self.P2P])
+        self.compute_total_info(comm_op_dict[self.COLLECTIVE])
+        return comm_op_dict
+
+    def integrate_matrix_data(self, comm_op_dict_simple):
+        """integrate the matrix data"""
+        comm_op_dict = defaultdict(dict)
+        for new_comm_op_name, data in comm_op_dict_simple.items():
+            data.sort(key=lambda x: x[self.BANDWIDTH_GB_S], reverse=True)
+            t_type = data[0].get(self.TRANSPORT_TYPE, '')
+            t_size = sum(x.get(self.TRANSIT_SIZE_MB, 0) for x in data)
+            t_time = sum(x.get(self.TRANSIT_TIME_MS, 0) for x in data)
+            bandwidth = self.compute_ratio(t_size, t_time)
+
+            link = new_comm_op_name[2]
+
+            comm_op_dict[f'{new_comm_op_name[0]}-top1@{new_comm_op_name[1]}'].update({link: data[0]})
+            comm_op_dict[f'{new_comm_op_name[0]}-middle@{new_comm_op_name[1]}'].update({link: data[len(data) // 2]})
+            comm_op_dict[f'{new_comm_op_name[0]}-bottom1@{new_comm_op_name[1]}'].update({link: data[-1]})
+            index2 = -2
+            index3 = -3
+            if len(data) == 1:
+                index2 = -1
+                index3 = -1
+            elif len(data) == 2:
+                index3 = -2
+            comm_op_dict[f'{new_comm_op_name[0]}-bottom2@{new_comm_op_name[1]}'].update({link: data[index2]})
+            comm_op_dict[f'{new_comm_op_name[0]}-bottom3@{new_comm_op_name[1]}'].update({link: data[index3]})
+            comm_op_dict[f'{new_comm_op_name[0]}-total@{new_comm_op_name[1]}'].update({link: {
+                self.TRANSPORT_TYPE: t_type,
+                self.TRANSIT_SIZE_MB: t_size,
+                self.TRANSIT_TIME_MS: t_time,
+                self.BANDWIDTH_GB_S: bandwidth
+            }})
+        return comm_op_dict
+
+    def get_matrix_ops_dict(self, op_data: dict) -> dict:
+        """parse matrix data"""
+        comm_op_dict_simple_p2p = defaultdict(list)
+        comm_op_dict_simple_collective = defaultdict(list)
+
+        for communication_op, communication_info in op_data.items():
+            if communication_op.find(self.HCOM_SEND) != -1 or communication_op.find(self.HCOM_RECEIVE) != -1:
+
+                match_obj = self.PATTERN1.search(communication_op.lower())
+                comm_op_type = match_obj.group()
+                for link, data in communication_info.items():
+                    new_comm_op_name = (comm_op_type, communication_op.split("@")[-1], link)
+                    data['op_name'] = communication_op.split("@")[0]
+                    comm_op_dict_simple_p2p[new_comm_op_name].append(data)
+
+            elif communication_op.startswith(self.TOTAL):
+                continue
+            else:
+                match_obj = self.PATTERN2.search(communication_op.lower())
+                if not match_obj:
+                    comm_op_type = communication_op.lower().split('/')[-1].split('-op')[0]
+                    logging.warning("Communication operator type not found communication_op: %s, use comm_op_type: %s",
+                                    communication_op, comm_op_type)
+                else:
+                    comm_op_type = match_obj.group()
+
+                for link, data in communication_info.items():
+                    new_comm_op_name = (comm_op_type, communication_op.split("@")[-1], link)
+                    data['op_name'] = communication_op.split("@")[0]
+                    comm_op_dict_simple_collective[new_comm_op_name].append(data)
+
+        comm_op_dict = {self.P2P: self.integrate_matrix_data(comm_op_dict_simple_p2p),
+                        self.COLLECTIVE: self.integrate_matrix_data(comm_op_dict_simple_collective)}
+
+        return comm_op_dict
+
+    def is_step_list_empty(self):
+        """is step list empty"""
+        for step_info in self.step_list:
+            if step_info.get("comm_ops"):
+                return False
+        return True
+
+    def compute_total_info(self, comm_ops: dict):
+        """
+        compute total info
+        """
+        if not comm_ops:
+            return
+        total_time_info_dict = defaultdict(float)
+        total_bandwidth_info_dict = {}
+        for _, communication_op_info in comm_ops.items():
+            for com_info, com_info_dict in communication_op_info.items():
+                if com_info == self.COMMUNICATION_TIME_INFO:
+                    self.combine_time_info(com_info_dict, total_time_info_dict)
+                if com_info == self.COMMUNICATION_BANDWIDTH_INFO:
+                    self.combine_bandwidth_info(com_info_dict, total_bandwidth_info_dict)
+        self.compute_time_ratio(total_time_info_dict)
+        self.compute_bandwidth_ratio(total_bandwidth_info_dict)
+        comm_ops['Total Op Info'] = {
+            self.COMMUNICATION_TIME_INFO: total_time_info_dict,
+            self.COMMUNICATION_BANDWIDTH_INFO: total_bandwidth_info_dict
+        }
+
+    def combine_time_info(self, com_info_dict: dict, total_time_info_dict: dict):
+        """combine time info"""
+        ratio_list = [self.WAIT_TIME_RATIO, self.SYNCHRONIZATION_TIME_RATIO]
+        for time_info in com_info_dict:
+            if time_info not in ratio_list and time_info != self.START_TIMESTAMP:
+                total_time_info_dict[time_info] += com_info_dict.get(time_info)
+
+    def combine_bandwidth_info(self, com_info_dict: dict, total_bandwidth_info_dict: dict):
+        """
+        combine bandwidth info
+        """
+        add_list = [self.TRANSIT_TIME_MS, self.TRANSIT_SIZE_MB]
+        dict_list = [self.SIZE_DISTRIBUTION]
+        for transport_type, part_transport_dict in com_info_dict.items():
+            if transport_type not in total_bandwidth_info_dict:
+                total_bandwidth_info_dict[transport_type] = {
+                    self.TRANSIT_TIME_MS: 0,
+                    self.TRANSIT_SIZE_MB: 0,
+                    self.SIZE_DISTRIBUTION: defaultdict(lambda: [0, 0])
+                }
+            for bandwidth_msg, value in part_transport_dict.items():
+                if bandwidth_msg in add_list:
+                    total_bandwidth_info_dict[transport_type][bandwidth_msg] += value
+                if bandwidth_msg in dict_list:
+                    self.combine_size_distribution(value, total_bandwidth_info_dict[transport_type][bandwidth_msg])
+
+    def compute_time_ratio(self, total_time_info_dict: dict):
+        """compute time ratio"""
+        total_time_info_dict[self.WAIT_TIME_RATIO] = \
+            self.compute_ratio(total_time_info_dict.get(self.WAIT_TIME_MS, 0),
+                               total_time_info_dict.get(self.WAIT_TIME_MS, 0) +
+                               total_time_info_dict.get(self.TRANSIT_TIME_MS, 0))
+        total_time_info_dict[self.SYNCHRONIZATION_TIME_RATIO] = \
+            self.compute_ratio(total_time_info_dict.get(self.SYNCHRONIZATION_TIME_MS, 0),
+                               total_time_info_dict.get(self.TRANSIT_TIME_MS, 0) +
+                               total_time_info_dict.get(self.SYNCHRONIZATION_TIME_MS, 0))
+
+    def compute_bandwidth_ratio(self, total_bandwidth_info_dict: dict):
+        """compute bandwidth ratio"""
+        for _, bandwidth_dict in total_bandwidth_info_dict.items():
+            self.compute_ratio(bandwidth_dict.get(self.TRANSIT_SIZE_MB, 0), bandwidth_dict.get(self.TRANSIT_TIME_MS, 0))
+
+    def write(self, communication_file_path, communication_matrix_file_path):
+        """
+        write communication file and communication matrix file
+        """
+        try:
+            with os.fdopen(os.open(communication_file_path,
+                                   os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as json_file:
+                json.dump(self.output_communication, json_file)
+        except (IOError, OSError) as err:
+            logging.critical('Error occurred when write communication file: %s', err)
+            raise ProfilerIOException() from err
+        if os.path.exists(communication_file_path):
+            os.chmod(communication_file_path, stat.S_IREAD | stat.S_IWRITE)
+
+        try:
+            with os.fdopen(os.open(communication_matrix_file_path,
+                                   os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as json_file:
+                json.dump(self.output_matrix_data, json_file)
+        except (IOError, OSError) as err:
+            logging.critical('Error occurred when write communication matrix file: %s', err)
+            raise ProfilerIOException() from err
+        if os.path.exists(communication_matrix_file_path):
+            os.chmod(communication_matrix_file_path, stat.S_IREAD | stat.S_IWRITE)

mindspore/profiler/parser/ascend_flops_generator.py

@@ -26,13 +26,19 @@ from mindspore.profiler.common.exceptions.exceptions import ProfilerIOException
 class AscendFlopsGenerator:
     """Generate ascend flops data from DataFrame."""
 
-    def __init__(self, op_summary):
+    def __init__(self, op_summary, pretty=False):
         self.op_summary = op_summary
         self.flops_dt = np.dtype(
             [('op_full_name', object), ('MFLOPs(10^6 cube)', float), ('GFLOPS(10^9 cube)', float),
              ('MFLOPs(10^6 vector)', float), ('GFLOPS(10^9 vector)', float)])
         self.flops = None
         self.flops_summary = None
+        self.pretty = pretty
+
+    @property
+    def indent(self):
+        indent = 1 if self.pretty else None
+        return indent
 
     def parse(self):
         """Analyse the op_summary data generate flops data."""
@@ -86,7 +92,7 @@ class AscendFlopsGenerator:
             with os.fdopen(os.open(flops_summary_path,
                                    os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR),
                            'w') as json_file:
-                json.dump(self.flops_summary, json_file)
+                json.dump(self.flops_summary, json_file, indent=self.indent)
         except (IOError, OSError) as err:
             logging.critical('Errot occurred when write step trace point info file: %s', err)
             raise ProfilerIOException() from err

mindspore/profiler/parser/ascend_fpbp_generator.py

@@ -26,10 +26,16 @@ from mindspore.profiler.common.exceptions.exceptions import ProfilerIOException
 class AscendFPBPGenerator:
     """Generate ascend fp bp data from DataFrame."""
 
-    def __init__(self, op_summary, steptrace):
+    def __init__(self, op_summary, steptrace, pretty=False):
         self.op_summary = op_summary
         self.steptrace = steptrace
         self.points = None
+        self.pretty = pretty
+
+    @property
+    def indent(self):
+        indent = 1 if self.pretty else None
+        return indent
 
     def parse(self):
         """Analyse the op_summary and steptrace data generate fpbp data."""
@@ -68,7 +74,7 @@ class AscendFPBPGenerator:
             with os.fdopen(os.open(step_trace_point_info_path,
                                    os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR),
                            'w') as json_file:
-                json.dump(self.points, json_file)
+                json.dump(self.points, json_file, indent=self.indent)
         except (IOError, OSError) as err:
             logging.critical('Errot occurred when write step trace point info file: %s', err)
             raise ProfilerIOException() from err

mindspore/profiler/parser/ascend_hccl_generator.py

@@ -208,8 +208,8 @@ class AscendHCCLGenerator:
                 name = row.get('name')
                 pid = row.get('pid')
                 tid = row.get('tid')
-                ts = row.get('ts')
-                dur = row.get('dur')
+                ts = float(row.get('ts'))
+                dur = float(row.get('dur'))
                 te = ts + dur
                 ph = row.get('ph')
                 task_type = row.get('args', {}).get('task type', '')

mindspore/profiler/parser/ascend_msprof_exporter.py

@@ -60,14 +60,20 @@ class AscendMsprofExporter:
 
     def get_drv_version(self):
         """Get the drv_version for choosing the export mode."""
+        script_path = self._get_msprof_info_path()
+        if not script_path:
+            logger.warning("Can`t find get_msprof_info.py path, use single-export mode instead.")
+            return False
+
+        logger.info("get_msprof_info.py path is : %s", script_path)
         host_dir = os.path.join(self.prof_root_dir, 'host')
         cmd = ['python',
-               '/usr/local/Ascend/latest/tools/profiler/profiler_tool/analysis/interface/get_msprof_info.py',
+               script_path,
                '-dir', host_dir]
         try:
             outs, _ = self._run_cmd(cmd)
             if not outs:
-                logger.warning('Check the drvVersion can`t find the result, use single export mode instead.')
+                logger.warning('Check the drvVersion can`t find the result, use single-export mode instead.')
                 return False
             result = json.loads(outs)
             logger.info('get drv_version result is : %s', result)
@@ -104,6 +110,10 @@ class AscendMsprofExporter:
             self._run_cmd(msprof_export_cmd)
             self._check_export_files(self.source_path)
 
+        msprof_analyze_cmd = [self._msprof_cmd, "--analyze=on", "--rule=communication,communication_matrix",
+                              "--output={}".format(self.prof_root_dir)]
+        self._run_cmd(msprof_analyze_cmd)
+
         return flag
 
     def _run_cmd(self, cmd):
@@ -176,6 +186,18 @@ class AscendMsprofExporter:
 
         logger.info("The msprof command has been added to the path!")
 
+    def _get_msprof_info_path(self):
+        """Check the existence of get_msprof_info.py script"""
+        outs, _ = self._run_cmd(['which', self._msprof_cmd])
+        if not outs:
+            return ""
+        msprof_path = os.path.realpath(outs.strip())
+        sup_path = msprof_path.split('tools')[0]
+        script_path = os.path.join(sup_path, 'tools/profiler/profiler_tool/analysis/interface/get_msprof_info.py')
+        if not os.path.exists(script_path):
+            return ""
+        return script_path
+
     def _generate_step_trace(self, prof_path, device_path):
         """"generate model_id iteration_id dict"""
 
@@ -228,9 +250,10 @@ class AscendMsprofExporter:
                 op_statistic.add(summary_file)
 
         if not op_summary:
-            raise RuntimeError("The op_summary file was not found, perhaps the original data was not collected.")
+            logger.warning("The op_summary file was not found, perhaps the original data was not collected.")
+            return
         if not op_statistic:
-            raise RuntimeError("The op_statistics file was not found, perhaps the original data was not collected.")
+            logger.warning("The op_statistics file was not found, perhaps the original data was not collected.")
 
         logger.info("Finish checking files.")
 
@@ -250,7 +273,8 @@ class AscendMsprofExporter:
                 op_statistic.add(summary_file)
 
         if not op_summary:
-            raise RuntimeError("The op_summary file was not found, perhaps the original data was not collected.")
+            logger.warning("The op_summary file was not found, perhaps the original data was not collected.")
+            return
         if not op_statistic:
-            raise RuntimeError("The op_statistics file was not found, perhaps the original data was not collected.")
+            logger.warning("The op_statistics file was not found, perhaps the original data was not collected.")
         logger.info("Finish checking files.")

mindspore/profiler/parser/ascend_msprof_generator.py

@@ -18,6 +18,7 @@ import fnmatch
 import os
 
 import numpy as np
+from mindspore import log as logger
 
 
 class AscendMsprofDataGeneratorOld:
@@ -128,13 +129,22 @@ class AscendMsprofDataGeneratorOld:
                     self.op_summary_name = self.op_summary_basis_name
                 self.op_summary_name['Iteration ID'] = {'index': -1, 'dtype': ('Iteration ID', object)}
                 for row in reader:
-                    row = [row[index.get('index')] for index in self.op_summary_name.values()]
-                    row[self.op_summary_name['Iteration ID']['index']] = iteration
-                    row = ['0' if i == 'N/A' else i for i in row]
-                    op_summary.append(tuple(row))
+                    try:
+                        row = [row[index.get('index')] for index in self.op_summary_name.values()]
+                        row[self.op_summary_name['Iteration ID']['index']] = iteration
+                        row = ['0' if i == 'N/A' else i for i in row]
+                        row += ['0.000']  # Add one column for Task Start Time(us)
+                        op_summary.append(tuple(row))
+                    except IndexError:
+                        logger.warning(f"Fail to read{file}. Will ignore this file and continue reading")
 
         op_summary_dt = np.dtype([value['dtype'] for value in self.op_summary_name.values()])
 
+        for i in range(0, len(op_summary)):
+            if len(op_summary[i]) < len(op_summary_dt):
+                new_raw = [j for j in op_summary[i]]
+                new_raw.extend([0 for _ in range(len(op_summary_dt) - len(op_summary[i]))])
+                op_summary[i] = tuple(new_raw)
         self.op_summary = np.array(op_summary, dtype=op_summary_dt)
         self.op_summary['Task Start Time'] = self.op_summary['Task Start Time'] * 1e-3
         self.op_summary['Task Duration'] = self.op_summary['Task Duration'] * 1e-3
@@ -348,7 +358,8 @@ class AscendMsprofDataGenerator:
                     new_row = tuple(['0' if d == 'N/A' else d for d in new_row])
                     op_statistic.append(new_row)
             break
-
+        if not op_statistic:
+            return
         op_statistic_dt = np.dtype(self.op_statistic_type)
         self.op_statistic = np.array(op_statistic, dtype=op_statistic_dt)
         self.op_statistic['Total Time'] *= 1e-3

mindspore/profiler/parser/ascend_op_generator.py

@@ -35,6 +35,7 @@ class AscendOPGenerator:
         self.aicpu_detail = None
         self.framework_raw = None
         self.output_timeline_data = None
+        self.has_statistic_file = True
 
         self.op_detail_dt = np.dtype(
             [('full_op_name', object), ('task_duration', float), ('execution_frequency', int), ('task_type', object)])
@@ -61,12 +62,16 @@ class AscendOPGenerator:
 
         # aicore intermediation type
         self.op_type = self._parse_op_type(self.op_statistic)
+        if isinstance(self.op_type, np.ndarray) and not self.op_type.size or not isinstance(self.op_type, np.ndarray) \
+                and not self.op_type:
+            self.has_statistic_file = False
 
         # aicpu_intermediation
         self.aicpu_detail = self._parse_aicpu_detail(self.op_summary)
 
         # framwork_raw
-        self.framework_raw = self._parse_framework_raw(self.op_summary)
+        if self.has_statistic_file:
+            self.framework_raw = self._parse_framework_raw(self.op_summary)
 
         self.output_timeline_data = self.op_summary[self.op_summary['Task Type'] == 'AI_CORE'][
             ['Op Name', 'Stream ID', 'Task Start Time', 'Task Duration']]
@@ -84,7 +89,7 @@ class AscendOPGenerator:
             output_timeline_data_path : output_timeline_data.txt path
         """
         # aicore intermediation detail
-        if self.op_detail.shape[0] != 0:
+        if isinstance(self.op_detail, np.ndarray) and self.op_detail.size and self.op_detail.shape[0] != 0:
             try:
                 with os.fdopen(os.open(aicore_intermediate_detail_path,
                                        os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR),
@@ -99,7 +104,7 @@ class AscendOPGenerator:
                 os.chmod(aicore_intermediate_detail_path, stat.S_IREAD | stat.S_IWRITE)
 
         # aicore intermediation type
-        if self.op_type.shape[0] != 0:
+        if isinstance(self.op_type, np.ndarray) and self.op_type.size and self.op_type.shape[0] != 0:
             try:
                 with os.fdopen(os.open(aicore_intermediate_type_path,
                                        os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR),
@@ -114,7 +119,7 @@ class AscendOPGenerator:
                 os.chmod(aicore_intermediate_type_path, stat.S_IREAD | stat.S_IWRITE)
 
         # aicpu_intermediation
-        if self.aicpu_detail.shape[0] != 0:
+        if isinstance(self.aicpu_detail, np.ndarray) and self.aicpu_detail.size and self.aicpu_detail.shape[0] != 0:
             try:
                 with os.fdopen(os.open(aicpu_intermediate_detail_path,
                                        os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR),
@@ -129,7 +134,7 @@ class AscendOPGenerator:
                 os.chmod(aicpu_intermediate_detail_path, stat.S_IREAD | stat.S_IWRITE)
 
         # framwork_raw
-        if self.framework_raw.shape[0] != 0:
+        if isinstance(self.framework_raw, np.ndarray) and self.framework_raw.size and self.framework_raw.shape[0] != 0:
             try:
                 with os.fdopen(os.open(framework_raw_path,
                                        os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR),
@@ -144,7 +149,8 @@ class AscendOPGenerator:
                 os.chmod(framework_raw_path, stat.S_IREAD | stat.S_IWRITE)
 
         # output_timeline_data
-        if self.output_timeline_data.shape[0] != 0 and output_timeline_data_path:
+        if isinstance(self.output_timeline_data, np.ndarray) and self.output_timeline_data.size and \
+                self.output_timeline_data.shape[0] != 0 and output_timeline_data_path:
             try:
                 with os.fdopen(os.open(output_timeline_data_path,
                                        os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR),
@@ -186,7 +192,9 @@ class AscendOPGenerator:
         Args:
             op_statistic(DataFrame): op statistic data.
         """
-
+        if isinstance(op_statistic, np.ndarray) and not op_statistic.size or not isinstance(op_statistic, np.ndarray) \
+                and not op_statistic:
+            return None
         groups, _, inverse, _ = np.unique(op_statistic['Op Type'], return_index=True, return_inverse=True,
                                           return_counts=True)
 

mindspore/profiler/parser/ascend_timeline_generator.py

@@ -49,17 +49,20 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
             [('Op Name', object), ('Stream ID', int), ('Task Start Time', float), ('Task Duration', float),
              ('pid', int)])
 
-    def init_timeline(self, op_summary, steptrace):
+    def init_timeline(self, op_summary, steptrace, pretty=False):
         """
         Init timeline metadata, adding all collected info.
 
         Args:
             op_summary: op data
             steptrace: step data
+            pretty: whether to format json file
         """
 
         logger.info('Initiating timeline...')
-
+        self._pretty = pretty
+        if op_summary.size < 1:
+            return
         timeline_list = op_summary[~np.isin(op_summary['Task Type'], ['AI_CPU', 'HCCL'])][
             ['Op Name', 'Stream ID', 'Task Start Time', 'Task Duration']]
 

mindspore/profiler/parser/base_timeline_generator.py

@@ -102,6 +102,12 @@ class BaseTimelineGenerator:
         self._model = model
         self._step_start_op_name = ""
         self._step_end_op_name = ""
+        self._pretty = False
+
+    @property
+    def indent(self):
+        indent = 1 if self._pretty else None
+        return indent
 
     @staticmethod
     def get_parallel_context():
@@ -201,7 +207,9 @@ class BaseTimelineGenerator:
             with os.fdopen(os.open(display_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as json_file:
                 json_file.write('[')
                 for _, item in enumerate(self._timeline_meta):
-                    json.dump(item, json_file)
+                    item_json = json.dumps([item], indent=self.indent)
+                    item_json = item_json.lstrip('[').rstrip('\n]')
+                    json_file.write(item_json)
                     if "scope_level" in item.keys():
                         self._max_scope_name_num = max(
                             self._max_scope_name_num, item["scope_level"] + 1)
@@ -209,7 +217,7 @@ class BaseTimelineGenerator:
                     json_file.write(',')
                     if file_size > size_limit:
                         break
-                label_name_json = json.dumps(self.get_thread_label_name())
+                label_name_json = json.dumps(self.get_thread_label_name(), indent=self.indent)
                 label_name_json = label_name_json.lstrip('[')
                 json_file.write(label_name_json)
                 os.chmod(display_file_path, stat.S_IREAD | stat.S_IWRITE)
@@ -230,7 +238,7 @@ class BaseTimelineGenerator:
         try:
             with os.fdopen(os.open(timeline_summary_file_path,
                                    os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as json_file:
-                json.dump(self._timeline_summary, json_file)
+                json.dump(self._timeline_summary, json_file, indent=self.indent)
         except (IOError, OSError) as err:
             logger.critical('Error occurred when write timeline summary file: %s', err)
             raise ProfilerIOException() from err

mindspore/profiler/parser/cpu_gpu_timeline_generator.py

@@ -542,8 +542,9 @@ class CpuTimelineGenerator(GpuTimelineGenerator):
 
         return timeline_list
 
-    def init_timeline(self):
+    def init_timeline(self, pretty=False):
         """Init timeline metadata, adding all collected info."""
+        self._pretty = pretty
         timeline_list = self._load_timeline_data()
 
         # Init a dict for counting the num of streams.