mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/profiler/parser/gpu_analysis/profiler_info_parser.py

@@ -0,0 +1,72 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Profiler host information parser"""
+import os
+import json
+from decimal import Decimal
+from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
+from mindspore.profiler.parser.ascend_analysis.constant import Constant
+from mindspore.profiler.parser.profiler_info import ProfilerInfo
+
+
+class GPUProfilerInfoParser:
+    """Parse files that record information, such as profiler_info.json"""
+
+    _freq = 2600000000
+    _system_time = 0
+    _system_cnt = 0
+    _s_to_ns = 1e9
+    # profiler information related files
+    _source_path = None
+    _loaded_frequency = False
+    _rank_id = 0
+
+    @classmethod
+    def init_source_path(cls, source_path: str):
+        """initialize the path of PROF_* directory."""
+        source_path = validate_and_normalize_path(source_path)
+        cls._source_path = source_path
+
+    @classmethod
+    def init_rank_id(cls, rank_id: int):
+        """initialize the rank id."""
+        cls._rank_id = rank_id
+
+    @classmethod
+    def get_local_time(cls, syscnt: int) -> Decimal:
+        """Convert syscnt to local time."""
+        if not cls._loaded_frequency:
+            profiler_info_path = os.path.join(cls._source_path, f"profiler_info_{cls._rank_id}.json")
+            if not os.path.isfile(profiler_info_path):
+                raise RuntimeError(f"Can`t find the file {profiler_info_path}, please check !")
+            with os.fdopen(os.open(profiler_info_path, os.O_RDONLY, 0o600),
+                           'r') as fr:
+                profiler_info_data = json.load(fr)
+            cls._system_cnt = profiler_info_data.get('system_cnt')
+            cls._system_time = profiler_info_data.get('system_time')
+            ProfilerInfo.set_system_time(cls._system_cnt)
+            ProfilerInfo.set_system_cnt(cls._system_time)
+            cls._loaded_frequency = True
+
+        start_ns = cls._get_timestamp(syscnt)
+        return Decimal(start_ns).quantize(Decimal('0.000')) * Decimal(Constant.NS_TO_US).quantize(Decimal('0.000'))
+
+    @classmethod
+    def _get_timestamp(cls, syscnt: int):
+        """Convert syscnt to time stamp."""
+        ratio = cls._freq / cls._s_to_ns
+        # The unit of timestamp is ns
+        timestamp = (syscnt - cls._system_cnt) / ratio + cls._system_time
+        return timestamp

mindspore/profiler/parser/integrator.py

@@ -282,7 +282,7 @@ class Integrator:
             _ = next(csv_reader)
             for info in csv_reader:
                 framework_infos[info[3]] = [
-                    info[3], info[4], info[5], info[6], json.loads(info[7]) if info[7] else None
+                    info[3], info[5], info[6], info[7], json.loads(info[8]) if info[8] else None
                 ]
 
         with open(op_detail_file_path, 'r') as file:
@@ -290,6 +290,8 @@ class Integrator:
             _ = next(csv_reader)
             for info in csv_reader:
                 framework_info = framework_infos.get(info[0])
+                if not framework_info:
+                    continue
                 self._aicore_detail_data.append(
                     [
                         framework_info[1], framework_info[2], float(info[1]),

mindspore/profiler/parser/memory_usage_parser.py

@@ -14,169 +14,15 @@
 # ============================================================================
 """Memory Usage Parser."""
 from collections import OrderedDict
-import json
-import os
-import stat
-
-from google.protobuf.text_format import ParseError
 
 from mindspore import log as logger
-from mindspore.profiler.common.exceptions.exceptions import ProfilerIOException, \
-    ProfilerFileNotFoundException, ProfilerRawFileException
-from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
 from mindspore.profiler.parser.container import MemoryGraph as Graph
 from mindspore.profiler.parser.container import MemoryNode as Node
 from mindspore.profiler.parser.container import MemoryTensor as Tensor
-import mindspore._c_expression as c_expression
-
-if not c_expression.security.enable_security():
-    from mindspore.train.memory_profiling_pb2 import MemoryProto
-else:
-    MemoryProto = None
 
 GIGABYTES = 1024 * 1024 * 1024
 
 
-class MemoryUsageParser:
-    """MemoryUsageParser to parse memory raw data."""
-
-    def __init__(self, profiling_dir, device_id, pretty=False):
-        self._profiling_dir = profiling_dir
-        self._device_id = device_id
-        self._proto_file_path = 'memory_usage_{}.pb'
-        self._summary_filename = 'memory_usage_summary_{}.json'
-        self._details_filename = 'memory_usage_details_{}.json'
-        self._graphs_dict = {}
-        self._peak_mem = 0
-        self._mem_summary = {
-            'capacity': 0,
-            'allocations': 0,
-            'deallocations': 0,
-            'peak_mem': 0,
-            'static_mem': 0
-        }
-        self._framework = {}
-        self._points = {}
-        self._pretty = pretty
-
-    @property
-    def indent(self):
-        indent = 1 if self._pretty else None
-        return indent
-
-    @staticmethod
-    def _process_framework_info(aicore_detail_data):
-        """Process framework info."""
-        framework_info_dict = {}
-        for framework_obj in aicore_detail_data:
-            op_name = framework_obj[0]
-            op_full_name = framework_obj[4]
-            op_info = framework_obj[5]
-            framework_info_dict[op_name] = {
-                'fullname': op_full_name,
-                'name': op_name,
-                'args': op_info
-            }
-
-        return framework_info_dict
-
-    def write_memory_files(self):
-        """Write memory files."""
-        logger.info('Start recording memory data into files...')
-        # write memory summary to json file
-        summary_filename = self._summary_filename.format(self._device_id)
-        self._write_memory_files(summary_filename, self._mem_summary)
-
-        # write memory details to json file
-        details_filename = self._details_filename.format(self._device_id)
-        self._write_memory_files(details_filename, self._graphs_dict)
-        logger.info('Successfully write memory data into files.')
-
-    def init_memory_usage_info(self, aicore_detail_data, points):
-        """Init memory usage information."""
-        logger.info("Start to load memory usage data from pb file")
-        file_path = self._get_file_path()
-        self._framework = self._process_framework_info(aicore_detail_data)
-        self._points = points
-
-        # Open memory protobuf file.
-        try:
-            with open(file_path, 'rb') as f:
-                content = f.read()
-        except (IOError, OSError) as err:
-            logger.critical('Failed to read memory file: %s', err)
-            raise ProfilerIOException from err
-
-        # Parse memory raw data from file.
-        if not c_expression.security.enable_security():
-            if not MemoryProto:
-                raise ProfilerRawFileException("Can not find memory profiling pb file.")
-
-            memory_proto = MemoryProto()
-            try:
-                memory_proto.ParseFromString(content)
-            except ParseError as err:
-                msg = "Fail to parse memory proto file."
-                logger.critical("Cannot parse the memory file. Please check the file schema.\n%s", err)
-                raise ProfilerRawFileException(msg) from err
-
-            # Parse memory details based on graphs in the network.
-            graphs = memory_proto.graph_mem
-            self._parse_graph_memory(graphs)
-            # Update memory summary information.
-            self._mem_summary['capacity'] = memory_proto.total_mem / GIGABYTES
-            self._mem_summary['peak_mem'] = self._peak_mem
-
-            logger.info('Finished processing memory usage data.')
-
-    def _get_file_path(self):
-        """Get the proto file path."""
-        file_path = os.path.join(
-            self._profiling_dir,
-            self._proto_file_path.format(self._device_id)
-        )
-        file_path = validate_and_normalize_path(file_path)
-
-        if not os.path.exists(file_path):
-            logger.warning('The memory file does not exist! Please ignore the warning '
-                           'if you are running heterogeneous training.')
-            raise ProfilerFileNotFoundException(msg=file_path)
-
-        return file_path
-
-    def _parse_graph_memory(self, graphs):
-        """Parse memory usage based on subgraphs."""
-        for graph_proto in graphs:
-            graph_id = graph_proto.graph_id
-            if graph_id is None:
-                logger.info('Graph id is missing, skipped the graph.')
-                continue
-
-            graph_parser = GraphMemoryParser(graph_proto, self._points, self._framework)
-            graph, model_id = graph_parser.parse_graph()
-            if graph and model_id != -1:
-                self._graphs_dict[model_id] = graph
-
-            # update global memory usage data
-            self._peak_mem = max(self._peak_mem, graph_parser.peak_mem)
-            self._mem_summary['static_mem'] += graph_parser.static_mem
-            self._mem_summary['allocations'] += graph_parser.allocations
-            self._mem_summary['deallocations'] += graph_parser.deallocations
-
-    def _write_memory_files(self, filename, content):
-        """Write the summary and top breakdowns of memory usage."""
-        file_path = os.path.join(self._profiling_dir, filename)
-        file_path = validate_and_normalize_path(file_path)
-
-        try:
-            with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as json_file:
-                json.dump(content, json_file, indent=self.indent)
-            os.chmod(file_path, stat.S_IREAD | stat.S_IWRITE)
-        except (IOError, OSError) as err:
-            logger.critical('Fail to write memory file.\n%s', err)
-            raise ProfilerIOException from err
-
-
 class GraphMemoryParser:
     """Parse memory usage data for each graph."""
 

mindspore/profiler/parser/minddata_parser.py

@@ -14,6 +14,8 @@
 # ============================================================================
 """Minddata aicpu parser."""
 import os
+import glob
+import csv
 
 from mindspore.profiler.common.util import get_file_join_name, fwrite_format
 from mindspore import log as logger
@@ -86,13 +88,80 @@ class MinddataParser:
         return result
 
     @staticmethod
-    def execute(source_path, output_path, device_id):
+    def execute(source_path, output_path, job_id, device_id):
         """
         Execute the parser.
 
         Args:
-            source_path (str): the source file path.
-            output_path (str): the output file path.
+            source_path (str): the source file path, eg: profiler.
+            output_path (str): the output file path, eg: profiler.
+            job_id (str): the job id, eg: PROF_XXX/device_*
+            device_id (str): the device id.
+        """
+        if MinddataParser._is_legacy_aicpu_data(source_path, job_id):
+            logger.warning("The aicpu data is legacy, which will be deprecated in the future, please update your "
+                           "CANN and driver version.")
+            MinddataParser._execute_legacy(os.path.join(source_path, job_id), output_path, device_id)
+            return
+
+        MinddataParser._execute(source_path, output_path, job_id, device_id)
+
+    @staticmethod
+    def _is_legacy_aicpu_data(source_path, job_id) -> bool:
+        """
+        Check whether the aicpu data is legacy.
+
+        Args:
+            source_path (str): the source file path, eg: profiler.
+            job_id (str): the job id, eg: PROF_XXX/device_*
+        Returns:
+            bool, True if the aicpu data is legacy, False otherwise.
+        """
+        legacy_files = glob.glob(os.path.join(source_path, job_id, "data", "DATA_PREPROCESS.*"))
+        return len(legacy_files) > 0
+
+    @staticmethod
+    def _execute(source_path, output_path, job_id, device_id):
+        """
+        Execute the parser when using newest CANN and driver version.
+
+        Args:
+            source_path (str): the source file path, eg: profiler.
+            output_path (str): the output file path, eg: profiler.
+            job_id (str): the job id, eg: PROF_XXX/device_*
+            device_id (str): the device id.
+        """
+        minddata_aicpu_data = []
+        prof_path = job_id.split("/")[0]
+        if not prof_path:
+            logger.error("The job_id is invalid: %s", job_id)
+            return
+
+        prof_output_path = os.path.join(source_path, prof_path, "mindstudio_profiler_output")
+        aicpu_file = glob.glob(os.path.join(prof_output_path, "aicpu_mi_*.csv"))
+        if not aicpu_file:
+            return
+
+        # aicpu_file len is 1
+        for file_path in aicpu_file:
+            file_path = validate_and_normalize_path(file_path)
+            with open(file_path, "r", newline='') as f:
+                reader = csv.reader(f)
+                minddata_aicpu_data = [[line[1], line[2][:-2], line[3][:-2], line[4]] for line in reader]
+
+        if minddata_aicpu_data:
+            minddata_aicpu_output_path = os.path.join(output_path, "minddata_aicpu_" + str(device_id) + ".txt")
+            fwrite_format(minddata_aicpu_output_path, minddata_aicpu_data[1:], is_start=True)
+            logger.info("Minddata aicpu data has been saved to %s", minddata_aicpu_output_path)
+
+    @staticmethod
+    def _execute_legacy(source_path, output_path, device_id):
+        """
+        Execute the parser when using legacy CANN and driver version.
+
+        Args:
+            source_path (str): the source file path, eg: profiler/PROF_XXX/device_*.
+            output_path (str): the output file path, eg: profiler.
             device_id (str): the device id.
         """
         col_names = ["node_name", "start_time", "end_time", "queue_size"]

mindspore/profiler/parser/profiler_info.py

@@ -13,12 +13,13 @@
 # limitations under the License.
 # ============================================================================
 """Record profiler information"""
-import json
+import glob
 import os
 import stat
 
 from mindspore.version import __version__ as ms_version
-from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
+from mindspore.profiler.parser.ascend_analysis.file_manager import FileManager
+from mindspore import log as logger
 
 
 class ProfilerInfo:
@@ -31,6 +32,7 @@ class ProfilerInfo:
     _file_name = "profiler_info_{}.json"
     _file_path = ""
     _profiler_info_dict = dict()
+    JIT_LEVEL = "jit_level"
 
     @staticmethod
     def init_info(context_mode, rank_id):
@@ -92,9 +94,24 @@ class ProfilerInfo:
 
     @staticmethod
     def set_export_flag(flag):
-        """Set the graph id list."""
+        """Set whether all-export or not."""
         ProfilerInfo._profiler_info_dict["all_export"] = flag
 
+    @staticmethod
+    def set_system_time(sys_time):
+        """Set system time."""
+        ProfilerInfo._profiler_info_dict["system_time"] = sys_time
+
+    @staticmethod
+    def set_system_cnt(sys_cnt):
+        """Set system cnt."""
+        ProfilerInfo._profiler_info_dict["system_cnt"] = sys_cnt
+
+    @staticmethod
+    def set_diff_time(diff_time):
+        """synchronize timestamps between different devices"""
+        ProfilerInfo._profiler_info_dict["diff_time"] = diff_time
+
     @staticmethod
     def set_graph_ids(graph_ids):
         """Set the graph id list."""
@@ -115,12 +132,82 @@ class ProfilerInfo:
         """Get the profiler info."""
         return ProfilerInfo._profiler_info_dict
 
+    @staticmethod
+    def set_profiling_options(profiling_options):
+        """Set profiling options to profiler info dict"""
+        ProfilerInfo._profiler_info_dict["profiling_options"] = profiling_options
+
+    @staticmethod
+    def set_jit_level(jit_level):
+        """Set jit_level to profiler info dict"""
+        ProfilerInfo._profiler_info_dict[ProfilerInfo.JIT_LEVEL] = jit_level
+
+    @staticmethod
+    def set_data_simplification(data_simplification):
+        """
+        Function Description:
+            Set the data simplification to profiler info dict
+        Parameter:
+            data_simplification: Whether data simplification is enabled
+        """
+        ProfilerInfo._profiler_info_dict["data_simplification"] = data_simplification
+
     @staticmethod
     def save(output_path):
         """Save the profiler info to file."""
         ProfilerInfo._file_path = os.path.join(output_path, ProfilerInfo._file_name)
-        ProfilerInfo._file_path = validate_and_normalize_path(ProfilerInfo._file_path)
-        with os.fdopen(os.open(ProfilerInfo._file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600),
-                       'w') as json_file:
-            json.dump(ProfilerInfo._profiler_info_dict, json_file)
+        FileManager.create_json_file(output_path, ProfilerInfo._profiler_info_dict, ProfilerInfo._file_name, indent=4)
+        os.chmod(ProfilerInfo._file_path, stat.S_IREAD | stat.S_IWRITE)
+
+    @staticmethod
+    def load_profiler_info_dict(input_path):
+        """Load the profiler info from input path."""
+        ProfilerInfo._file_path = os.path.join(input_path, ProfilerInfo._file_name)
+        try:
+            load_info_dict = FileManager.read_json_file(ProfilerInfo._file_path)
+        except RuntimeError as err:
+            logger.warning(f"Cannot read file: {ProfilerInfo._file_path}, Error: {err}")
+            return
+        if not load_info_dict:
+            msg = f"Offline analysis failed load the ProfilerInfo._profiler_info_dict from: {ProfilerInfo._file_path}"
+            logger.warning(msg)
+            return
+        ProfilerInfo._profiler_info_dict = load_info_dict
         os.chmod(ProfilerInfo._file_path, stat.S_IREAD | stat.S_IWRITE)
+
+    @staticmethod
+    def get_rank_id(profiler_dir: str):
+        """
+        Function Description:
+            Get rank id from profiler_info_*.json
+        Parameter:
+            profiler_dir: the directory path of profiler data, eg: rank_0/profiler
+        Return:
+            str type rank id
+        """
+        prof_info_path = os.path.join(profiler_dir, "profiler_info_*.json")
+        prof_info_path = glob.glob(prof_info_path)
+        if not prof_info_path:
+            logger.warning("Cannot find profiler_info.json in the profiler directory.")
+            return "-1"
+
+        info_data = FileManager.read_json_file(prof_info_path[0])
+        return info_data.get("rank_id", "-1")
+
+    @staticmethod
+    def get_device_id(prof_dir: str):
+        """
+        Function Description:
+            Get device id from PROF_XXX dir
+        Parameter:
+            prof_dir: the directory path of PROF_XXX
+        Return:
+            str type device id
+        """
+        device_dir = os.path.join(prof_dir, "device_*")
+        device_dir = glob.glob(device_dir)
+        if not device_dir:
+            logger.warning("Cannot find device_XXX in the %s.", prof_dir)
+            return "-1"
+
+        return device_dir[0].split("device_")[-1]

mindspore/profiler/profiler.py

@@ -0,0 +1,153 @@
+# Copyright 2020-2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Profiling api file."""
+from mindspore.profiler.common.registry import PROFILERS
+from mindspore.profiler.common.constant import DeviceTarget
+from mindspore.profiler.common.constant import ProfilerLevel
+from mindspore.profiler.platform_profiler.prof_context import ProfContext
+
+
+class NewProfiler:
+    """
+    Refactor profiler
+    """
+
+    def __init__(
+            self,
+            output_path: str = "./data",
+            profiler_level: ProfilerLevel = None,
+            op_time: bool = True,
+            profile_communication: bool = False,
+            profile_memory: bool = False,
+            parallel_strategy: bool = False,
+            start_profile: bool = True,
+            aicore_metrics: int = 0,
+            l2_cache: bool = False,
+            hbm_ddr: bool = False,
+            pcie: bool = False,
+            sync_enable: bool = True,
+            data_process: bool = False,
+            timeline_limit: int = 500,
+            profile_framework: str = None,
+            with_stack: bool = False,
+            data_simplification: bool = True,
+            **kwargs) -> None:
+
+        self._prof_context = ProfContext(
+            output_path=output_path,
+            profiler_level=profiler_level,
+            op_time=op_time,
+            profile_communication=profile_communication,
+            profile_memory=profile_memory,
+            parallel_strategy=parallel_strategy,
+            start_profile=start_profile,
+            aicore_metrics=aicore_metrics,
+            l2_cache=l2_cache,
+            hbm_ddr=hbm_ddr,
+            pcie=pcie,
+            sync_enable=sync_enable,
+            data_process=data_process,
+            timeline_limit=timeline_limit,
+            profile_framework=profile_framework,
+            with_stack=with_stack,
+            data_simplification=data_simplification
+        )
+
+        self._has_started = False
+
+        self._cpu_profiler = PROFILERS.get_modules().get(DeviceTarget.CPU.value)(
+            op_time=self._prof_context.op_time,
+            with_stack=self._prof_context.with_stack,
+            data_process=self._prof_context.data_process,
+            output_path=self._prof_context.output_path,
+            profile_memory=self._prof_context.profile_memory,
+            profile_framework=self._prof_context.profile_framework
+        )
+
+        self._device_target = self._prof_context.device_target
+        self._device_profiler = PROFILERS.get_modules().get(self._device_target)(
+            self._prof_context.get_args()
+        )
+
+    def start(self) -> None:
+        """
+        Used for Ascend, GPU, start profiling. Profiling can be turned on based on step and epoch.
+        """
+        if not self._has_started:
+            self._has_started = True
+        else:
+            raise RuntimeError("The profiler has already started. Do not turn on again in the open state.")
+
+        self._cpu_profiler.start()
+        self._device_profiler.start()
+
+    def stop(self) -> None:
+        """
+        Used for Ascend, GPU, stop profiling. Profiling can be turned off based on step and epoch.
+        """
+        if self._has_started:
+            self._has_started = False
+        else:
+            raise RuntimeError("The profiler has not started, so can not stop. Please call the start() method "
+                               "before calling the stop() method.")
+
+        self._cpu_profiler.stop()
+        self._device_profiler.stop()
+
+    def analyse(self, offline_path=None, pretty=False, step_list=None, mode="sync") -> None:
+        """
+        Collect and analyze training performance data, support calls during and after training. The example shows above.
+
+        Args:
+            offline_path (Union[str, None], optional): The data path which need to be analyzed with offline mode.
+                Offline mode isused in abnormal exit scenario. This parameter should be set to ``None``
+                for online mode. Default: ``None``.
+            pretty (bool, optional): Whether to pretty json files. Default: ``False``.
+            step_list (list, optional): A list of steps that need to be analyzed. Default: ``None``.
+                By default, all steps will be analyzed.
+            mode (str, optional): Analysis mode, it must be one of ["sync", "async"]. Default: ``sync``.
+
+                - sync: analyse data in current process, it will block the current process.
+                - async: analyse data in subprocess, it will not the current process.Since the parsing process
+                  will take up extra CPU resources, please enable this mode according to the actual resource situation.
+
+        """
+        self._cpu_profiler.stop(offline_path, pretty, step_list)
+        self._device_profiler.stop(offline_path, pretty, step_list, mode)
+
+    def op_analyse(self, op_name, device_id=None) -> None:
+        """
+        Profiler users can use this interface to obtain operator performance data.
+
+        Args:
+            op_name (str or list): The primitive operator name to query.
+            device_id (int, optional): ID of the target device. This parameter is optional during network training or
+                inference, and users can use device_id parameter to specify which card operator performance data to
+                parse. If this interface is used for offline data parsing, Default: ``0`` .
+        """
+
+    @classmethod
+    def offline_analyse(cls, path: str, pretty=False, step_list=None) -> None:
+        """
+        Analyze training performance data offline, which is invoked after performance data collection is completed.
+
+        Args:
+            path (str): The profiling data path which need to be analyzed offline.
+                There needs to be a profiler directory in this path.
+            pretty (bool, optional): Whether to pretty json files. Default: ``False``.
+            step_list (list, optional): A list of steps that need to be analyzed. Default: ``None``.
+                By default, all steps will be analyzed.
+        """
+        return