mindspore 2.3.0__cp310-cp310-win_amd64.whl → 2.4.0__cp310-cp310-win_amd64.whl

mindspore/profiler/parser/ascend_analysis/path_manager.py

@@ -0,0 +1,313 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Profiler path manager"""
+import os
+import re
+import shutil
+import warnings
+
+from mindspore import log as logger
+from mindspore.profiler.parser.ascend_analysis.constant import Constant
+
+
+__all__ = ['PathManager']
+
+
+class PathManager:
+    """
+    Path common operations manager
+    """
+    MAX_PATH_LENGTH = 4096
+    MAX_FILE_NAME_LENGTH = 255
+    DATA_FILE_AUTHORITY = 0o640
+    DATA_DIR_AUTHORITY = 0o750
+
+    @classmethod
+    def check_input_directory_path(cls, path: str):
+        """
+        Function Description:
+            check whether the path is valid, some businesses can accept a path that does not exist,
+            so the function do not verify whether the path exists
+        Parameter:
+            path: the path to check, whether the incoming path is absolute or relative depends on the business
+        Exception Description:
+            when invalid data throw exception
+        """
+        cls._input_path_common_check(path)
+
+        if os.path.isfile(path):
+            msg = "Invalid input path is a file path: {path}"
+            raise RuntimeError(msg)
+
+    @classmethod
+    def check_input_file_path(cls, path: str):
+        """
+        Function Description:
+            check whether the file path is valid, some businesses can accept a path that does not exist,
+            so the function do not verify whether the path exists
+        Parameter:
+            path: the file path to check, whether the incoming path is absolute or relative depends on the business
+        Exception Description:
+            when invalid data throw exception
+        """
+        cls._input_path_common_check(path)
+
+        if os.path.isdir(path):
+            msg = "Invalid input path is a directory path: {path}"
+            raise RuntimeError(msg)
+
+    @classmethod
+    def copy_file(cls, src_path: str, dst_path: str):
+        """
+        Function Description:
+            copy file safety
+        Parameter:
+            src_path: file source path
+            dst_path: file destination path
+        Exception Description:
+            when src_path is link throw exception
+        """
+        if not os.path.exists(src_path):
+            logger.warning("The source file does not exist: %s", src_path)
+            return
+
+        cls.check_input_file_path(src_path)
+        dst_dir = os.path.dirname(dst_path)
+        cls.check_directory_path_writeable(dst_dir)
+
+        try:
+            shutil.copy2(src_path, dst_path)
+        except Exception as err:
+            msg = f"Failed to copy path: {src_path}"
+            raise RuntimeError(msg) from err
+
+    @classmethod
+    def check_path_owner_consistent(cls, path: str):
+        """
+        Function Description:
+            check whether the path belong to process owner
+        Parameter:
+            path: the path to check
+        Exception Description:
+            when invalid path, prompt the user
+        """
+
+        if not os.path.exists(path):
+            msg = f"The path does not exist: {path}"
+            raise RuntimeError(msg)
+        if os.stat(path).st_uid != os.getuid():
+            warnings.warn(f"Warning: The {path} owner does not match the current user.")
+
+    @classmethod
+    def check_directory_path_writeable(cls, path):
+        """
+        Function Description:
+            check whether the path is writable
+        Parameter:
+            path: the path to check
+        Exception Description:
+            when invalid data throw exception
+        """
+        cls.check_path_owner_consistent(path)
+        if os.path.islink(path):
+            msg = f"Invalid path is a soft chain: {path}"
+            raise RuntimeError(msg)
+        if not os.access(path, os.W_OK):
+            msg = f"The path permission check failed: {path}"
+            raise RuntimeError(msg)
+
+    @classmethod
+    def check_directory_path_readable(cls, path):
+        """
+        Function Description:
+            check whether the path is writable
+        Parameter:
+            path: the path to check
+        Exception Description:
+            when invalid data throw exception
+        """
+        cls.check_path_owner_consistent(path)
+        if os.path.islink(path):
+            msg = f"Invalid path is a soft chain: {path}"
+            raise RuntimeError(msg)
+        if not os.access(path, os.R_OK):
+            msg = f"The path permission check failed: {path}"
+            raise RuntimeError(msg)
+
+    @classmethod
+    def remove_path_safety(cls, path: str):
+        """
+        Function Description:
+            remove path safety
+        Parameter:
+            path: the path to remove
+        Exception Description:
+            when invalid data throw exception
+        """
+        msg = f"Failed to remove path: {path}"
+        if os.path.islink(path):
+            raise RuntimeError(msg)
+        if not os.path.exists(path):
+            return
+        try:
+            shutil.rmtree(path)
+        except FileNotFoundError:
+            return
+        except Exception as err:
+            raise RuntimeError(msg) from err
+
+    @classmethod
+    def remove_file_safety(cls, file: str):
+        """
+        Function Description:
+            remove file safety
+        Parameter:
+            path: the file to remove
+        Exception Description:
+            when invalid data throw exception
+        """
+        msg = f"Failed to remove file: {file}"
+        if os.path.islink(file):
+            raise RuntimeError(msg)
+        if not os.path.exists(file):
+            return
+        try:
+            os.remove(file)
+        except FileExistsError:
+            return
+        except Exception as err:
+            raise RuntimeError(msg) from err
+
+    @classmethod
+    def make_dir_safety(cls, path: str):
+        """
+        Function Description:
+            make directory safety
+        Parameter:
+            path: the directory to remove
+        Exception Description:
+            when invalid data throw exception
+        """
+        msg = f"Failed to make directory: {path}"
+        if os.path.islink(path):
+            raise RuntimeError(msg)
+        if os.path.exists(path):
+            return
+        try:
+            os.makedirs(path, mode=cls.DATA_DIR_AUTHORITY, exist_ok=True)
+        except Exception as err:
+            raise RuntimeError(msg) from err
+
+    @classmethod
+    def create_file_safety(cls, path: str):
+        """
+        Function Description:
+            create file safety
+        Parameter:
+            path: the file to remove
+        Exception Description:
+            when invalid data throw exception
+        """
+        msg = f"Failed to create file: {path}"
+        if os.path.islink(path):
+            raise RuntimeError(msg)
+        if os.path.exists(path):
+            return
+        try:
+            os.close(os.open(path, os.O_WRONLY | os.O_CREAT, cls.DATA_FILE_AUTHORITY))
+        except Exception as err:
+            raise RuntimeError(msg) from err
+
+    @classmethod
+    def _input_path_common_check(cls, path: str):
+        """
+        Function Description:
+            input path check common function
+        Parameter:
+            path: the file path to check
+        Exception Description:
+            when invalid data throw exception
+        """
+        if len(path) > cls.MAX_PATH_LENGTH:
+            raise RuntimeError("Length of input path exceeds the limit.")
+
+        if os.path.islink(path):
+            msg = f"Invalid input path is a soft chain: {path}"
+            raise RuntimeError(msg)
+
+        pattern = r'(\.|/|_|-|\s|[~0-9a-zA-Z])+'
+        if not re.fullmatch(pattern, path):
+            msg = f"Invalid input path: {path}"
+            raise RuntimeError(msg)
+
+        path_split_list = path.split("/")
+        for name in path_split_list:
+            if len(name) > cls.MAX_FILE_NAME_LENGTH:
+                raise RuntimeError("Length of input path exceeds the limit.")
+
+    @classmethod
+    def get_profiler_parent_path_list(cls, input_path: str):
+        """
+        Function Description:
+            get valid profiler parent path list from input_path
+        Parameter:
+            input_path: The directory path from which to extract profiler parent paths.
+        Return:
+            A list containing the input path or its subdirectories that are valid profiler parents.
+        """
+        profiler_path = os.path.join(input_path, Constant.PROFILER_DIR)
+        if os.path.isdir(profiler_path) and (cls.get_fwk_path(profiler_path) or cls.get_cann_path(profiler_path)):
+            return [input_path]
+        sub_dirs = os.listdir(os.path.realpath(input_path))
+        profiler_parent_path_list = []
+        for sub_dir in sub_dirs:
+            sub_path = os.path.join(input_path, sub_dir, Constant.PROFILER_DIR)
+            if not os.path.isdir(sub_path):
+                continue
+            if cls.get_fwk_path(sub_path) or cls.get_cann_path(sub_path):
+                profiler_parent_path_list.append(os.path.join(input_path, sub_dir))
+        return profiler_parent_path_list
+
+    @classmethod
+    def get_fwk_path(cls, input_path: str):
+        """
+        Function Description:
+            get valid framework path from input_path
+        Parameter:
+            input_path: the directory path to check whether exist valid FRAMEWORK path
+        Return:
+            The path to the FRAMEWORK directory if found, otherwise an empty string.
+        """
+        fwk_path = os.path.join(input_path, Constant.FRAMEWORK_DIR)
+        if os.path.isdir(fwk_path):
+            return fwk_path
+        return ""
+
+    @classmethod
+    def get_cann_path(cls, input_path: str):
+        """
+        Function Description:
+            get valid PROF_XXX path from input_path
+        Parameter:
+            input_path: the directory path to check valid PROF_XXX path
+        Return:
+            The path to the PROF_XXX directory if it matches the pattern and exists, otherwise an empty string.
+        """
+        sub_dirs = os.listdir(os.path.realpath(input_path))
+        for sub_dir in sub_dirs:
+            sub_path = os.path.join(input_path, sub_dir)
+            if os.path.isdir(sub_path) and re.match(r"^PROF_\d+_\d+_[0-9a-zA-Z]+", sub_dir):
+                return sub_path
+        return ""

mindspore/profiler/parser/ascend_analysis/profiler_info_parser.py

@@ -18,25 +18,28 @@ import json
 from decimal import Decimal
 from subprocess import CalledProcessError, TimeoutExpired
 from subprocess import Popen, PIPE
+from configparser import ConfigParser
 
 from mindspore import log as logger
 from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
 from mindspore.profiler.parser.ascend_analysis.constant import Constant
-from mindspore.profiler.parser.profiler_info import ProfilerInfo
 
 
 class ProfilerInfoParser:
     """Parse files that record information, such as profiler_info.json"""
 
     _freq = 100.0
-    _time_offset = 0
-    _start_cnt = 0
     _msprof_cmd = "msprof"
     _time_out = 1
     # profiler information related files
     _source_prof_path = None
     _loaded_frequency = False
     _rank_id = 0
+    _clock_monotonic_raw = 0
+    _cntvct = 0
+    _collectionTimeBegin = 0
+    _clockMonotonicRaw = 0
+    _get_localtime_diff = 0
 
     @classmethod
     def init_source_path(cls, source_path: str):
@@ -73,31 +76,35 @@ class ProfilerInfoParser:
                 cls._freq = float(cpu_info.get("Frequency", cls._freq))
             except ValueError:
                 pass
-            profiler_info_path = os.path.join(cls._source_prof_path, os.path.pardir,
-                                              f"profiler_info_{cls._rank_id}.json")
-            if not os.path.isfile(profiler_info_path):
-                raise RuntimeError(f"Can`t find the file {profiler_info_path}, please check !")
-            with os.fdopen(os.open(profiler_info_path, os.O_RDONLY, 0o600),
-                           'r') as fr:
-                profiler_info_data = json.load(fr)
-            cls._start_cnt = profiler_info_data.get('system_cnt')
-            cls._time_offset = profiler_info_data.get('system_time')
-            ProfilerInfo.set_system_time(cls._time_offset)
-            ProfilerInfo.set_system_cnt(cls._start_cnt)
+            cls._get_msprof_timestamp(cls._source_prof_path)
             cls._loaded_frequency = True
         start_ns = cls.__get_timestamp(syscnt)
-        start_us = Decimal(start_ns * Constant.NS_TO_US)
-        return start_us
+        return Decimal(start_ns).quantize(Decimal('0.000')) * Decimal(Constant.NS_TO_US).quantize(Decimal('0.000'))
+
+    @classmethod
+    def _get_msprof_timestamp(cls, source_path):
+        """get msprof timestamp info"""
+        start_log = ConfigParser()
+        start_log.read(os.path.join(source_path, "host", "host_start.log"))
+        cls._clock_monotonic_raw = int(start_log.get("Host", "clock_monotonic_raw"))
+        cls._cntvct = int(start_log.get("Host", "cntvct"))
+
+        with open(os.path.join(source_path, "host", "start_info"), "r") as f:
+            info_dict = json.load(f)
+            cls._collectionTimeBegin = int(info_dict.get("collectionTimeBegin", 0))  # us
+            cls._clockMonotonicRaw = int(info_dict.get("clockMonotonicRaw", 0))
+            us_to_ns = 1000
+            cls._get_localtime_diff = cls._clock_monotonic_raw + (cls._collectionTimeBegin * us_to_ns -
+                                                                  cls._clockMonotonicRaw)
 
     @classmethod
     def __run_cmd(cls, cmd):
         """run shell command"""
         try:
-            proc = Popen(cmd, stdout=PIPE, stderr=PIPE, text=True)
+            with Popen(cmd, stdout=PIPE, stderr=PIPE, text=True) as proc:
+                outs, errs = proc.communicate(timeout=cls._time_out)
         except (FileNotFoundError, PermissionError, CalledProcessError) as exc:
             raise RuntimeError(exc) from exc
-        try:
-            outs, errs = proc.communicate(timeout=cls._time_out)
         except TimeoutExpired as err:
             proc.kill()
             msg = "The possible cause is that too much data is collected " \
@@ -112,5 +119,5 @@ class ProfilerInfoParser:
         """Convert syscnt to time stamp."""
         ratio = time_fmt / cls._freq
         # The unit of timestamp is ns
-        timestamp = round((syscnt - cls._start_cnt) * ratio) + cls._time_offset
+        timestamp = round((syscnt - cls._cntvct) * ratio) + cls._get_localtime_diff
         return timestamp

mindspore/profiler/parser/ascend_analysis/trace_event_manager.py

@@ -33,14 +33,21 @@ class TraceEventManager:
         return x_event
 
     @classmethod
-    def create_m_event(cls, pid: int, tid_list: set) -> List:
+    def create_i_event(cls, event: BaseEvent) -> Dict:
+        """Create a i event."""
+        event = {"name": event.name, "ph": "i", "ts": str(event.ts),
+                 "pid": event.pid, "tid": event.tid, "args": event.args}
+        return event
+
+    @classmethod
+    def create_m_event(cls, pid: int, tid_list: set, process_index: int) -> List:
         """Create some metadata event."""
         # framework sidee trace information display format: MindSpore(pid pid_value): CPU
         event_list = [
             # process information
             {"ph": "M", "name": Constant.PROCESS_NAME, "pid": pid, "tid": 0, "args": {"name": "MindSpore"}},
             {"ph": "M", "name": Constant.PROCESS_LABEL, "pid": pid, "tid": 0, "args": {"labels": "CPU"}},
-            {"ph": "M", "name": Constant.PROCESS_SORT, "pid": pid, "tid": 0, "args": {"sort_index": 0}},
+            {"ph": "M", "name": Constant.PROCESS_SORT, "pid": pid, "tid": 0, "args": {"sort_index": process_index}},
         ]
         for tid in tid_list:
             sort_index = tid

mindspore/profiler/parser/ascend_msprof_exporter.py

@@ -24,6 +24,7 @@ from subprocess import Popen, PIPE
 import csv
 from mindspore import log as logger
 from mindspore.profiler.common.util import get_newest_file
+from mindspore.profiler.common.util import timeit
 
 
 class AscendMsprofExporter:
@@ -54,7 +55,7 @@ class AscendMsprofExporter:
         self._time_out = time_out
         self.mindstudio_profiler_output = mindstudio_profiler_output  # mindstudio_profiler_output dir
         # PROF* dir
-        self.prof_root_dir = os.path.abspath(os.path.join(self.mindstudio_profiler_output, os.path.pardir))
+        self.prof_root_dir = os.path.realpath(os.path.join(self.mindstudio_profiler_output, os.path.pardir))
 
         AscendMsprofExporter.check_msprof_env()
 
@@ -127,11 +128,10 @@ class AscendMsprofExporter:
     def run_cmd(cls, cmd, timeout=300):
         """run shell command"""
         try:
-            proc = Popen(cmd, stdout=PIPE, stderr=PIPE, text=True)
+            with Popen(cmd, stdout=PIPE, stderr=PIPE, text=True) as proc:
+                outs, errs = proc.communicate(timeout=timeout)
         except (FileNotFoundError, PermissionError, CalledProcessError) as exc:
             raise RuntimeError(exc) from exc
-        try:
-            outs, errs = proc.communicate(timeout=timeout)
         except TimeoutExpired as err:
             proc.kill()
             msg = "The possible cause is that too much data is collected " \
@@ -168,6 +168,7 @@ class AscendMsprofExporter:
             logger.warning('Get the drvVersion error, use single-export mode instead. detail : %s', err)
             return False
 
+    @timeit("CANN export done")
     def export(self, model_iteration_dict=None):
         """start_time is the time to collect PROF data"""
 

mindspore/profiler/parser/ascend_timeline_generator.py

@@ -26,6 +26,7 @@ from mindspore.profiler.parser.base_timeline_generator import BaseTimelineGenera
 from mindspore.profiler.parser.integrator import DeviceTarget
 from mindspore.profiler.parser.ascend_analysis.fwk_cann_parser import FwkCANNParser
 from mindspore.profiler.common.util import get_newest_file
+from mindspore.profiler.parser.ascend_analysis.constant import Constant
 
 
 class AscendTimelineGenerator(BaseTimelineGenerator):
@@ -33,9 +34,6 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
     _timeline_display_filename = 'ascend_timeline_display_{}.json'
     _timeline_summary_filename = 'ascend_timeline_summary_{}.json'
     _cluster_analyse_filename = 'ascend_cluster_analyse_{}_{}_{}_{}.csv'
-    top_scope_name = ('Default', 'Gradients', 'recompute_Default')
-    scope_index = 1
-    cpu_index = 2
 
     def __init__(self, profiling_dir, source_path, mindstudio_profiler_output, rank_id, rank_size, mode,
                  step_list=None):
@@ -161,10 +159,6 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
         else:
             msprof_timeline = self._parse_msprof_data(get_newest_file(file_list_msprof))
 
-        # get Ascend Hardware for scope
-        scope_data = self._parse_ascend_hardware_scope(msprof_timeline)
-        all_scope_data.extend(scope_data)
-
         # get cpu op
         cpu_op_file_name = fr'{self._profiling_dir}/cpu_op_execute_timestamp_{self._rank_id}.txt'
         file_list = glob.glob(cpu_op_file_name)
@@ -175,20 +169,24 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
             timeline_data.extend(cpu_timeline)
             all_scope_data.extend(scope_data)
 
-        # parse scope info
-        scope_timeline = self._parse_scope_info(all_scope_data)
-        timeline_data.extend(scope_timeline)
-
         oprange_name = self._op_range_name.format(self._rank_id)
         fwk_file_path = fr'{self._profiling_dir}/{self._framework_dir}/{oprange_name}'
         if os.path.exists(fwk_file_path):
             # It is faster not to submit to the pool
             result = self._parse_fwk_device_data(msprof_timeline)
+            all_scope_data.extend(result.get('scope_data', []))
             timeline_data.extend(result.get("trace_data", []))
             self._kernel_events = result.get("kernels", [])
         else:
+            # get Ascend Hardware for scope
+            scope_data = self._parse_ascend_hardware_scope(msprof_timeline)
+            all_scope_data.extend(scope_data)
             timeline_data.extend(msprof_timeline)
 
+        # parse scope info
+        scope_timeline = self._parse_scope_info(all_scope_data)
+        timeline_data.extend(scope_timeline)
+
         logger.info("All timeline data parse complete.")
         self._timeline_data = timeline_data
         return timeline_data
@@ -196,16 +194,16 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
     def parse_cpu_timeline(self, file_list):
         """Load cpu operator data from file"""
         ms_to_us = 1e3
-        ps_to_ns = 1e-3
-        new_pid = int(f'{self.cpu_index}{self._rank_id}')
+        ns_to_us = 1e-3
+        new_pid = Constant.CPU_OP
         process_list = [{"name": "process_name",
                          "pid": new_pid,
                          "args": {
-                             "name": f"CPU OP Rank{self._rank_id}"
+                             "name": f"CPU OP"
                          },
                          "ph": "M"
                          }, {"name": "process_sort_index", "pid": new_pid,
-                             "args": {"sort_index": self.cpu_index}, "ph": "M"}
+                             "args": {"sort_index": new_pid}, "ph": "M"}
                         ]
         tid_set = set()
         thread_list = []
@@ -221,11 +219,14 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
                         time_arr = op_list[-1]
                         time_arr = time_arr.split(" ")
                         for time in time_arr:
-                            ts, dur, tid = time.split(",")
-                            ts = Decimal(ts).quantize(Decimal('0.000')) * Decimal(ps_to_ns).quantize(Decimal('0.000'))
-
-                            if op_full_name and op_full_name.startswith(self.top_scope_name):
-                                te = ts + Decimal(dur).quantize(Decimal('0.000'))
+                            ts, dur, tid = time.split(",")   # origin unit of ts is ns and dur is ms.
+                            ts = Decimal(ts).quantize(Decimal('0.000')) * Decimal(ns_to_us).quantize(
+                                Decimal('0.000'))  # cast to us
+                            dur = Decimal(dur).quantize(Decimal('0.000')) * Decimal(ms_to_us).quantize(
+                                Decimal('0.000'))  # cast to us
+
+                            if op_full_name and op_full_name.startswith(Constant.TOP_SCOPE_NAMES):
+                                te = ts + dur
                                 scope_data.append((op_full_name.split('/')[:-1], ts, te))
 
                             if int(tid) not in tid_set:
@@ -261,7 +262,8 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
         fwkcann_parser = FwkCANNParser(self._source_path, cann_kernel_data, self._rank_id, self._step_list)
         fwk_link_data = fwkcann_parser.generate_trace_data()
         kernels = fwkcann_parser.kernels
-        result = {"trace_data": fwk_link_data, "kernels": kernels}
+        scope_data = fwkcann_parser.scope_data_with_flow + fwkcann_parser.scope_data_without_flow
+        result = {"trace_data": fwk_link_data, "kernels": kernels, "scope_data": scope_data}
         return result
 
     def _parse_msprof_data(self, file_list):
@@ -298,7 +300,7 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
                     continue
 
                 op_full_name = event.get('name')
-                if op_full_name and op_full_name.startswith(self.top_scope_name):
+                if op_full_name and op_full_name.startswith(Constant.TOP_SCOPE_NAMES):
                     ts = Decimal(event.get('ts')).quantize(Decimal('0.000'))
                     te = ts + Decimal(event.get('dur')).quantize(Decimal('0.000'))
                     scope_data.append((op_full_name.split('/')[:-1], ts, te))
@@ -313,18 +315,18 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
         """Parse scope info of op"""
         if not scope_data:
             return []
-        new_pid = int(f'{self.scope_index}{self._rank_id}')
+        new_pid = Constant.SCOPE_LAYLER
         scope_data.sort(key=lambda x: x[1])
         process_list = [
             {"name": "process_name",
              "pid": new_pid,
              "args": {
-                 "name": f"Scope Layer Rank{self._rank_id}"
+                 "name": f"Scope Layer"
              },
              "ph": "M"},
             {"name": "process_sort_index",
              "pid": new_pid,
-             "args": {"sort_index": self.scope_index},
+             "args": {"sort_index": new_pid},
              "ph": "M"}
         ]