mindspore 2.6.0__cp311-cp311-win_amd64.whl → 2.7.0rc1__cp311-cp311-win_amd64.whl

mindspore/profiler/parser/minddata_parser.py

@@ -1,186 +0,0 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""Minddata aicpu parser."""
-import os
-import glob
-import csv
-
-from mindspore.profiler.common.util import get_file_join_name, fwrite_format
-from mindspore import log as logger
-from mindspore.profiler.common.validator.validate_path import \
-    validate_and_normalize_path
-
-
-class MinddataParser:
-    """Minddata Aicpu Parser."""
-
-    @staticmethod
-    def parse_step_minddata_aicpu_data(one_step, result):
-        """
-        Parse step mind_data ai_cpu data.
-
-        Args:
-            one_step (str): The mind_data step info text, it is one of two structures.
-
-                Type queue: node_name,queue_size,run_start,run_end
-                Type run: node_name,run_start,run_end,queue_size
-
-            result ([[node_name, node_start, node_end, queue_size]]): Step info list.
-        """
-
-        if not one_step:
-            return
-        node_info = one_step.split(", ")
-        node_name, node_start, node_end, queue_size = "", 0, 0, 0
-        if node_info:
-            node_name = node_info[0].replace("Node:", "")
-
-        if len(node_info) > 3:
-            if "queue" in node_info[1]:
-                queue_size = node_info[1].replace("queue size:", "")
-                node_start = node_info[2].replace("Run start:", "")
-                node_end = node_info[3].replace("Run end:", "")
-            elif "Run" in node_info[1]:
-                queue_size = node_info[3].replace("queue size:", "")
-                node_start = node_info[1].replace("Run start:", "")
-                node_end = node_info[2].replace("Run end:", "")
-            queue_size = int(queue_size) if queue_size.isdigit() else queue_size
-            node_start = int(node_start) if node_start.isdigit() else node_start
-            node_end = int(node_end) if node_end.isdigit() else node_end
-
-        one_step_list = [node_name, node_start, node_end, queue_size]
-        result.append(one_step_list)
-
-    @staticmethod
-    def parse_minddata_aicpu_data(minddata_aicpu_source_path):
-        """
-        Parse minddata get_next info which contains queue size and execute time.
-
-        Args:
-            minddata_aicpu_source_path (str): the source file path.
-
-        Returns:
-            list[Union[str, float]], the converted data.
-        """
-        result = list()
-        try:
-            minddata_aicpu_source_path = validate_and_normalize_path(minddata_aicpu_source_path)
-            with open(minddata_aicpu_source_path) as source_data_file:
-                source_data = source_data_file.read()
-                step_data = source_data.split("\x00")
-                for one_step in step_data:
-                    MinddataParser.parse_step_minddata_aicpu_data(one_step, result)
-        except OSError:
-            logger.error("Open get_next profiling file error.")
-
-        return result
-
-    @staticmethod
-    def execute(source_path, output_path, job_id, device_id):
-        """
-        Execute the parser.
-
-        Args:
-            source_path (str): the source file path, eg: profiler.
-            output_path (str): the output file path, eg: profiler.
-            job_id (str): the job id, eg: PROF_XXX/device_*
-            device_id (str): the device id.
-        """
-        if MinddataParser._is_legacy_aicpu_data(source_path, job_id):
-            logger.warning("The aicpu data is legacy, which will be deprecated in the future, please update your "
-                           "CANN and driver version.")
-            MinddataParser._execute_legacy(os.path.join(source_path, job_id), output_path, device_id)
-            return
-
-        MinddataParser._execute(source_path, output_path, job_id, device_id)
-
-    @staticmethod
-    def _is_legacy_aicpu_data(source_path, job_id) -> bool:
-        """
-        Check whether the aicpu data is legacy.
-
-        Args:
-            source_path (str): the source file path, eg: profiler.
-            job_id (str): the job id, eg: PROF_XXX/device_*
-        Returns:
-            bool, True if the aicpu data is legacy, False otherwise.
-        """
-        legacy_files = glob.glob(os.path.join(source_path, job_id, "data", "DATA_PREPROCESS.*"))
-        return len(legacy_files) > 0
-
-    @staticmethod
-    def _execute(source_path, output_path, job_id, device_id):
-        """
-        Execute the parser when using newest CANN and driver version.
-
-        Args:
-            source_path (str): the source file path, eg: profiler.
-            output_path (str): the output file path, eg: profiler.
-            job_id (str): the job id, eg: PROF_XXX/device_*
-            device_id (str): the device id.
-        """
-        minddata_aicpu_data = []
-        prof_path = job_id.split("/")[0]
-        if not prof_path:
-            logger.error("The job_id is invalid: %s", job_id)
-            return
-
-        prof_output_path = os.path.join(source_path, prof_path, "mindstudio_profiler_output")
-        aicpu_file = glob.glob(os.path.join(prof_output_path, "aicpu_mi_*.csv"))
-        if not aicpu_file:
-            return
-
-        # aicpu_file len is 1
-        for file_path in aicpu_file:
-            file_path = validate_and_normalize_path(file_path)
-            with open(file_path, "r", newline='') as f:
-                reader = csv.reader(f)
-                minddata_aicpu_data = [[line[1], line[2][:-2], line[3][:-2], line[4]] for line in reader]
-
-        if minddata_aicpu_data:
-            minddata_aicpu_output_path = os.path.join(output_path, "minddata_aicpu_" + str(device_id) + ".txt")
-            fwrite_format(minddata_aicpu_output_path, minddata_aicpu_data[1:], is_start=True)
-            logger.info("Minddata aicpu data has been saved to %s", minddata_aicpu_output_path)
-
-    @staticmethod
-    def _execute_legacy(source_path, output_path, device_id):
-        """
-        Execute the parser when using legacy CANN and driver version.
-
-        Args:
-            source_path (str): the source file path, eg: profiler/PROF_XXX/device_*.
-            output_path (str): the output file path, eg: profiler.
-            device_id (str): the device id.
-        """
-        col_names = ["node_name", "start_time", "end_time", "queue_size"]
-        source_path = validate_and_normalize_path(source_path)
-        minddata_aicpu_source_path = get_file_join_name(
-            input_path=source_path, file_name='DATA_PREPROCESS.AICPUMI')
-        if not minddata_aicpu_source_path:
-            minddata_aicpu_source_path = get_file_join_name(
-                input_path=source_path, file_name='DATA_PREPROCESS.dev.AICPUMI')
-            if not minddata_aicpu_source_path:
-                minddata_aicpu_source_path = get_file_join_name(
-                    input_path=os.path.join(source_path, "data"), file_name='DATA_PREPROCESS.AICPUMI')
-                if not minddata_aicpu_source_path:
-                    minddata_aicpu_source_path = get_file_join_name(
-                        input_path=os.path.join(source_path, "data"), file_name='DATA_PREPROCESS.dev.AICPUMI')
-                    if not minddata_aicpu_source_path:
-                        return
-        minddata_aicpu_output_path = os.path.join(output_path, "minddata_aicpu_" + str(device_id) + ".txt")
-        minddata_aicpu_data = MinddataParser.parse_minddata_aicpu_data(minddata_aicpu_source_path)
-        if minddata_aicpu_data:
-            fwrite_format(minddata_aicpu_output_path, " ".join(col_names), is_start=True)
-            fwrite_format(minddata_aicpu_output_path, minddata_aicpu_data, is_start=True)

mindspore/profiler/parser/minddata_pipeline_parser.py

@@ -1,299 +0,0 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""Thr parser for parsing minddata pipeline files."""
-import csv
-import json
-import os
-import stat
-from queue import Queue
-
-from mindspore.profiler.common.exceptions.exceptions import \
-    ProfilerPathErrorException, ProfilerRawFileException, \
-    ProfilerDirNotFoundException
-from mindspore import log as logger
-from mindspore.profiler.common.validator.validate_path import \
-    validate_and_normalize_path
-
-
-class MinddataPipelineParser:
-    """
-    Thr parser for parsing minddata pipeline files.
-
-    Args:
-        source_dir (str): The minddata pipeline source dir.
-        device_id (str): The device ID.
-        output_path (str): The directory of the parsed file. Default: `./`.
-
-    Raises:
-        ProfilerPathErrorException: If the minddata pipeline file path or
-            the output path is invalid.
-    """
-    _raw_pipeline_file_name = 'pipeline_profiling_{}.json'
-    _parsed_pipeline_file_name = 'minddata_pipeline_raw_{}.csv'
-    _col_names = [
-        'op_id', 'op_type', 'num_workers', 'output_queue_size',
-        'output_queue_average_size', 'output_queue_length',
-        'output_queue_usage_rate', 'sample_interval', 'parent_id', 'children_id'
-    ]
-
-    def __init__(self, source_dir, device_id, output_path='./'):
-        self._device_id = device_id
-        self._pipeline_path = self._get_pipeline_path(source_dir)
-        self._save_path = self._get_save_path(output_path)
-
-    @property
-    def save_path(self):
-        """
-        The property of save path.
-
-        Returns:
-            str, the save path.
-        """
-        return self._save_path
-
-    def parse(self):
-        """
-        Parse the minddata pipeline files.
-
-        Raises:
-            ProfilerRawFileException: If fails to parse the raw file of
-                minddata pipeline or the file is empty.
-        """
-        if not self._pipeline_path:
-            return
-        with open(self._pipeline_path, 'r') as file:
-            try:
-                pipeline_info = json.load(file)
-            except (json.JSONDecodeError, TypeError) as err:
-                logger.warning(err)
-                raise ProfilerRawFileException(
-                    'Fail to parse minddata pipeline file.'
-                ) from err
-        if not pipeline_info:
-            logger.warning('The minddata pipeline file is empty.')
-            raise ProfilerRawFileException(
-                'The minddata pipeline file is empty.'
-            )
-
-        self._parse_and_save(pipeline_info)
-
-    def _get_pipeline_path(self, source_dir):
-        """
-        Get the minddata pipeline file path.
-
-        Args:
-            source_dir (str): The minddata pipeline source dir.
-
-        Returns:
-            str, the minddata pipeline file path.
-        """
-        pipeline_path = os.path.join(
-            source_dir,
-            self._raw_pipeline_file_name.format(self._device_id)
-        )
-
-        try:
-            pipeline_path = validate_and_normalize_path(pipeline_path)
-        except RuntimeError as err:
-            logger.warning('Minddata pipeline file is invalid.')
-            raise ProfilerPathErrorException('Minddata pipeline file is invalid.') from err
-        if not os.path.isfile(pipeline_path):
-            logger.warning(
-                'The minddata pipeline file <%s> not found.', pipeline_path
-            )
-            pipeline_path = ""
-
-        return pipeline_path
-
-    def _get_save_path(self, output_path):
-        """
-        Get the save path.
-
-        Args:
-            output_path (str): The output dir.
-
-        Returns:
-            str, the save path.
-        """
-        try:
-            output_dir = validate_and_normalize_path(output_path)
-        except RuntimeError as err:
-            logger.warning('Output path is invalid.')
-            raise ProfilerPathErrorException('Output path is invalid.') from err
-        if not os.path.isdir(output_dir):
-            logger.warning('The output dir <%s> not found.', output_dir)
-            raise ProfilerDirNotFoundException(output_dir)
-        return os.path.join(
-            output_dir, self._parsed_pipeline_file_name.format(self._device_id)
-        )
-
-    def _parse_and_save(self, pipeline_info):
-        """
-        Parse and save the parsed minddata pipeline file.
-
-        Args:
-            pipeline_info (dict): The pipeline info reads from the raw file of
-                the minddata pipeline.
-
-        Raises:
-            ProfilerRawFileException: If the format of minddata pipeline raw
-                file is wrong.
-        """
-        sample_interval = pipeline_info.get('sampling_interval')
-        op_info = pipeline_info.get('op_info')
-        if sample_interval is None or not op_info:
-            raise ProfilerRawFileException(
-                'The format of minddata pipeline raw file is wrong.'
-            )
-
-        op_id_info_cache = {}
-        for item in op_info:
-            if not item:
-                raise ProfilerRawFileException(
-                    'The content of minddata pipeline raw file is wrong.'
-                )
-            op_id_info_cache[item.get('op_id')] = item
-
-        with os.fdopen(os.open(self._save_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as save_file:
-            csv_writer = csv.writer(save_file)
-            csv_writer.writerow(self._col_names)
-            self._parse_and_save_op_info(
-                csv_writer, op_id_info_cache, sample_interval
-            )
-        os.chmod(self._save_path, stat.S_IREAD | stat.S_IWRITE)
-
-    def _parse_and_save_op_info(self, csv_writer, op_id_info_cache,
-                                sample_interval):
-        """
-        Parse and save the minddata pipeline operator information.
-
-        Args:
-            csv_writer (csv.writer): The csv writer.
-            op_id_info_cache (dict): The operator id and information cache.
-            sample_interval (int): The sample interval.
-
-        Raises:
-            ProfilerRawFileException: If the operator that id is 0 does not exist.
-        """
-        queue = Queue()
-        root_node = op_id_info_cache.get(0)
-        if not root_node:
-            raise ProfilerRawFileException(
-                'The format of minddata pipeline raw file is wrong, '
-                'the operator that id is 0 does not exist.'
-            )
-        root_node['parent_id'] = None
-        queue.put_nowait(root_node)
-
-        while not queue.empty():
-            node = queue.get_nowait()
-            self._update_child_node(node, op_id_info_cache)
-            csv_writer.writerow(self._get_op_info(node, sample_interval))
-
-            op_id = node.get('op_id')
-            children_ids = node.get('children')
-            if not children_ids:
-                continue
-            for child_op_id in children_ids:
-                sub_node = op_id_info_cache.get(child_op_id)
-                sub_node['parent_id'] = op_id
-                queue.put_nowait(sub_node)
-
-    def _update_child_node(self, node, op_id_info_cache):
-        """
-        Updates the child node information of the operator.
-
-        Args:
-            node (dict): The node represents an operator.
-            op_id_info_cache (dict): The operator id and information cache.
-        """
-        child_op_ids = node.get('children')
-        if not child_op_ids:
-            return
-
-        queue = Queue()
-        self._cp_list_item_to_queue(child_op_ids, queue)
-
-        new_child_op_ids = []
-        while not queue.empty():
-            child_op_id = queue.get_nowait()
-            child_node = op_id_info_cache.get(child_op_id)
-            if child_node is None:
-                continue
-            metrics = child_node.get('metrics')
-            if not metrics or not metrics.get('output_queue'):
-                op_ids = child_node.get('children')
-                if op_ids:
-                    self._cp_list_item_to_queue(op_ids, queue)
-            else:
-                new_child_op_ids.append(child_op_id)
-
-        node['children'] = new_child_op_ids
-
-    def _get_op_info(self, op_node, sample_interval):
-        """
-        Get the operator information.
-
-        Args:
-            op_node (dict): The node represents an operator.
-            sample_interval (int): The sample interval.
-
-        Returns:
-            list[str, int, float], the operator information.
-        """
-        queue_size = None
-        queue_average_size = None
-        queue_length = None
-        queue_usage_rate = None
-        metrics = op_node.get('metrics')
-        if metrics:
-            output_queue = metrics.get('output_queue')
-            if output_queue:
-                queue_size = output_queue.get('size')
-                if queue_size is None:
-                    raise ValueError("The queue can not be None.")
-                if queue_size:
-                    queue_average_size = sum(queue_size) / len(queue_size)
-                queue_length = output_queue.get('length')
-                if queue_length == 0:
-                    raise ValueError("The length of queue can not be 0.")
-                if queue_average_size is not None:
-                    queue_usage_rate = queue_average_size / queue_length
-
-        children_id = op_node.get('children')
-        op_info = [
-            op_node.get('op_id'),
-            op_node.get('op_type'),
-            op_node.get('num_workers'),
-            queue_size,
-            queue_average_size,
-            queue_length,
-            queue_usage_rate,
-            sample_interval,
-            op_node.get('parent_id'),
-            children_id if children_id else None
-        ]
-        return op_info
-
-    def _cp_list_item_to_queue(self, inner_list, queue):
-        """
-        Copy the contents of a list to a queue.
-
-        Args:
-            inner_list (list): The list.
-            queue (Queue): The target queue.
-        """
-        for item in inner_list:
-            queue.put_nowait(item)

mindspore/profiler/parser/op_intermediate_parser.py

@@ -1,149 +0,0 @@
-# Copyright 2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""Op intermediate files parser."""
-import csv
-import os
-import stat
-from mindspore.profiler.common.exceptions.exceptions import ProfilerFileNotFoundException, \
-    ProfilerIOException
-from mindspore import log as logger
-from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
-
-
-class OPIntermediateParser:
-    """
-    Op intermediate files parser.
-
-    Args:
-        profiling_dir (str): The directory where the parsed profiling files are
-            located.
-        rank_id (str): The rank ID.
-    """
-
-    _output_timeline_data_file_path = 'output_timeline_data_{}.txt'
-    _file_name_op_intermediate_type = 'pynative_op_intermediate_{}_type.csv'
-    _file_name_op_intermediate_detail = 'pynative_op_intermediate_{}_detail.csv'
-
-    _op_intermediate_type_header = ['op_type', 'execution_time', 'execution_frequency', 'percent']
-    _op_intermediate_op_header = ['full_op_name', 'execution_time']
-
-    _ms_decimal_digits = 6
-    _percent_decimal_digits = 2
-
-    def __init__(self, profiling_dir, rank_id):
-        self._profiling_dir = profiling_dir
-        self._rank_id = rank_id
-
-    def get_timeline_data(self, all_reduce_names=None):
-        """
-        Load timeline data from file.
-
-        Args:
-            all_reduce_names (list): The communication operator list.
-        """
-        all_reduce_names = all_reduce_names or []
-        file_path = os.path.join(
-            self._profiling_dir,
-            self._output_timeline_data_file_path.format(self._rank_id)
-        )
-        file_path = validate_and_normalize_path(file_path)
-        if not os.path.exists(file_path):
-            logger.critical("Failed to find parsed timeline file.")
-            raise ProfilerFileNotFoundException('parsed timeline file')
-
-        timeline_list = []
-        try:
-            with open(file_path, 'r') as f_obj:
-                for line in f_obj:
-                    # line: op_name, stream_id, start_time(ms), duration(ms)
-                    line_list = line.strip('\n').split(',')
-                    # filter out communication operators
-                    if line_list[0] == 'op_name' or line_list[0] in all_reduce_names:
-                        continue
-                    timeline_list.append(line_list)
-        except (IOError, OSError) as err:
-            logger.critical('Error occurred when read timeline intermediate file: %s', err)
-            raise ProfilerIOException() from err
-        finally:
-            pass
-
-        return timeline_list
-
-    def parser_pynative_op_intermediate_detail(self):
-        """Parse pynative op intermediate detail."""
-        timeline_list = self.get_timeline_data(None)
-        # key:op name, value:[op count, total op execution time]
-        op_intermediate_detail = {}
-        for timeline in timeline_list:
-            op_name = timeline[0].split('/')[-1]
-
-            detail = op_intermediate_detail.get(op_name)
-            if not detail:
-                detail = [0, 0]
-                op_intermediate_detail[op_name] = detail
-            detail[0] = detail[0] + 1
-            detail[1] = detail[1] + float(timeline[3])
-
-        op_op_file_path = os.path.join(self._profiling_dir,
-                                       self._file_name_op_intermediate_detail.format(self._rank_id))
-        with os.fdopen(os.open(op_op_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as op_file:
-            csv_writer = csv.writer(op_file)
-            csv_writer.writerow(self._op_intermediate_op_header)
-
-            for op_name, op_name_time_info in op_intermediate_detail.items():
-                op_info = [
-                    op_name, round(op_name_time_info[1] / op_name_time_info[0], self._ms_decimal_digits)
-                ]
-                csv_writer.writerow(op_info)
-        os.chmod(op_op_file_path, stat.S_IREAD | stat.S_IWRITE)
-
-    def parser_pynative_op_type(self):
-        """Parse pynative op intermediate type."""
-        timeline_list = self.get_timeline_data(None)
-        # key:op type, value:[op count, total op execution time, op execution time percent]
-        op_type_list = {}
-        for timeline in timeline_list:
-            type_name = timeline[0].split('/')[-1].split('-')[0]
-            op_type = op_type_list.get(type_name)
-            if not op_type:
-                op_type = [0, 0, 0]
-                op_type_list[type_name] = op_type
-            op_type[0] = op_type[0] + 1
-            op_type[1] = op_type[1] + float(timeline[3])
-
-        sum_avg_time = 0
-        for _, op_type in op_type_list.items():
-            op_type[1] = op_type[1] / op_type[0]
-            sum_avg_time = sum_avg_time + op_type[1]
-
-        if sum_avg_time <= 0:
-            logger.error("Operator time must be greater than 0.")
-            return
-        for _, op_type in op_type_list.items():
-            op_type[2] = op_type[1] / sum_avg_time
-
-        op_type_file_path = os.path.join(self._profiling_dir,
-                                         self._file_name_op_intermediate_type.format(self._rank_id))
-        with os.fdopen(os.open(op_type_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as type_file:
-            csv_writer = csv.writer(type_file)
-            csv_writer.writerow(self._op_intermediate_type_header)
-
-            for op_type, op_type_time_info in op_type_list.items():
-                type_info = [
-                    op_type, op_type_time_info[1], op_type_time_info[0],
-                    round((op_type_time_info[1] / sum_avg_time) * 100, self._percent_decimal_digits)
-                ]
-                csv_writer.writerow(type_info)
-        os.chmod(op_type_file_path, stat.S_IREAD | stat.S_IWRITE)