mindspore 2.7.0rc1__cp310-cp310-win_amd64.whl → 2.7.1__cp310-cp310-win_amd64.whl

mindspore/profiler/common/file_manager.py

@@ -206,3 +206,12 @@ class FileManager:
             if file_name.startswith(start_name) and file_name.endswith(".csv"):
                 file_list.append(os.path.join(source_path, file_name))
         return file_list
+
+    @classmethod
+    def check_file_owner(cls, path):
+        """Check whether the file owner is the current user or root."""
+        stat_info = os.stat(path)
+        if stat_info.st_uid == 0:
+            return True
+        current_uid = os.geteuid()
+        return current_uid == stat_info.st_uid

mindspore/profiler/common/msprof_cmd_tool.py

@@ -22,6 +22,7 @@ from typing import Dict, List, Optional
 from mindspore import log as logger
 from mindspore.profiler.common.command_executor import CommandExecutor
 from mindspore.profiler.common.constant import ExportType
+from mindspore.profiler.common.path_manager import PathManager
 
 
 class MsprofCmdTool:
@@ -120,6 +121,7 @@ class MsprofCmdTool:
         Raises:
             FileNotFoundError: If msprof or python3 command is not found.
         """
+        self._check_msprof_profile_path_is_valid()
         if not shutil.which(self._MSPROF_CMD):
             logger.warning(
                 "The msprof command is not found in PATH. Searching in environment variables..."
@@ -131,11 +133,44 @@ class MsprofCmdTool:
                 logger.info("Successfully added msprof command to PATH.")
             else:
                 raise FileNotFoundError("Failed to find msprof command in environment.")
-
+        else:
+            msprof_path = shutil.which(self._MSPROF_CMD)
+        self._check_msprof_permission(msprof_path)
         if not shutil.which("python3"):
             logger.warning("Failed to find python3 command in environment.")
             raise FileNotFoundError("Failed to find python3 command in environment.")
 
+    def _check_msprof_profile_path_is_valid(self):
+        """Check msprof profiler path is invalid."""
+        PathManager.check_directory_path_readable(self._msprof_profile_path)
+        PathManager.check_directory_path_writeable(self._msprof_profile_path)
+        PathManager.check_path_owner_consistent(self._msprof_profile_path)
+        PathManager.check_path_is_other_writable(self._msprof_profile_path)
+        if not PathManager.check_path_is_executable(self._msprof_profile_path):
+            raise PermissionError(f"The '{self._msprof_profile_path}' path is not executable."
+                                  f"Please execute chmod -R 755 {self._msprof_profile_path}")
+
+    def _check_msprof_permission(self, msprof_path):
+        """Check msprof path permissions."""
+        msprof_script_path = self._get_msprof_script_path(self._MSPROF_PY_PATH)
+        if not msprof_script_path:
+            raise FileNotFoundError(
+                "Failed to find msprof.py path. Perhaps the permission of the 'msprof' tool is unexecutable. "
+                "Please check the CANN environment. You can modify the 'msprof' file to an executable permission "
+                "through the chmod method."
+            )
+        if not PathManager.check_path_is_owner_or_root(msprof_script_path) or \
+                not PathManager.check_path_is_owner_or_root(msprof_path):
+            raise PermissionError(f"PermissionError, CANN package user id: {os.stat(msprof_path).st_uid}, "
+                                  f"current user id: {os.getuid()}. "
+                                  f"Ensure CANN package user id and current user id consistency")
+        if not PathManager.check_path_is_executable(msprof_script_path) or \
+                not PathManager.check_path_is_executable(msprof_path):
+            raise PermissionError(f"The '{msprof_script_path}' path or '{msprof_path}' path is not executable."
+                                  f"Please execute chmod u+x {msprof_script_path} and "
+                                  f"chmod u+x {msprof_path}")
+        PathManager.check_path_is_other_writable(msprof_script_path)
+
     def _find_msprof_path(self) -> Optional[str]:
         """Find msprof path in environment variables.
 
@@ -145,14 +180,14 @@ class MsprofCmdTool:
         if os.environ.get("ASCEND_TOOLKIT_HOME"):
             temp_path = os.path.join(os.environ.get("ASCEND_TOOLKIT_HOME"), "bin")
             if os.path.isdir(temp_path) and self._MSPROF_CMD in os.listdir(temp_path):
-                return temp_path
+                return os.path.abspath(temp_path)
 
         for path in os.environ.get("PATH", "").split(":"):
             if self._ASCEND_MARK in path:
                 prefix = path.split(self._ASCEND_MARK)[0]
                 temp_path = os.path.join(prefix, self._HIAI_MSPROF_TAIL)
                 if os.path.isdir(temp_path) and self._MSPROF_CMD in os.listdir(temp_path):
-                    return temp_path
+                    return os.path.abspath(temp_path)
 
         return None
 
@@ -166,7 +201,8 @@ class MsprofCmdTool:
         if not script_path:
             logger.error("Failed to find get_msprof_info.py path.")
             return {}
-
+        if not PathManager.check_path_is_executable(script_path):
+            raise PermissionError(f"The '{script_path}' path is not executable. Please execute chmod u+x {script_path}")
         host_dir = os.path.join(self._msprof_profile_path, "host")
         cmd = ["python3", script_path, "-dir", host_dir]
         command_outs = CommandExecutor.execute(cmd)[0]

mindspore/profiler/common/path_manager.py

@@ -73,6 +73,15 @@ class PathManager:
             msg = f"file size exceeds the limit: {cls.MAX_FILE_SIZE}, file size: {file_size}"
             raise ProfilerPathErrorException(msg)
 
+        file_stat = os.stat(path)
+        if file_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH):
+            msg = f"File path {path} has group or others writable permissions, which is not allowed."
+            raise ProfilerPathErrorException(msg)
+
+        if stat.S_ISCHR(file_stat.st_mode) or stat.S_ISBLK(file_stat.st_mode):
+            msg = f"Invalid input path is a character or block device path: {path}"
+            raise ProfilerPathErrorException(msg)
+
     @classmethod
     def get_directory_size(cls, directory: str, unit: str = 'MB') -> float:
         """
@@ -126,7 +135,8 @@ class PathManager:
             msg = f"The path does not exist: {path}"
             raise ProfilerPathErrorException(msg)
         if os.name != 'nt' and os.stat(path).st_uid != os.getuid():
-            msg = f"Path {path} owner[{os.stat(path).st_uid}] does not match the current user[{os.getuid()}]."
+            msg = (f"Path {path} owner[{os.stat(path).st_uid}] does not match the current user[{os.getuid()}]."
+                   f"Please execute chown -R $(id -un) {path}")
             raise ProfilerPathErrorException(msg)
 
     @classmethod
@@ -144,7 +154,7 @@ class PathManager:
             msg = f"Invalid path is a soft link: {path}"
             raise ProfilerPathErrorException(msg)
         if not os.access(path, os.W_OK):
-            msg = f"The path writeable permission check failed: {path}"
+            msg = f"The path writeable permission check failed: {path}. Please execute chmod -R 755 {path}"
             raise ProfilerPathErrorException(msg)
 
     @classmethod
@@ -162,7 +172,7 @@ class PathManager:
             msg = f"Invalid path is a soft link: {path}"
             raise ProfilerPathErrorException(msg)
         if not os.access(path, os.R_OK):
-            msg = f"The path readable permission check failed: {path}"
+            msg = f"The path readable permission check failed: {path}. Please execute chmod -R 755 {path}"
             raise ProfilerPathErrorException(msg)
 
     @classmethod
@@ -237,26 +247,6 @@ class PathManager:
         except Exception as err:
             raise ProfilerPathErrorException(f"Failed to make directory: {path}, err: {err}") from err
 
-    @classmethod
-    def create_file_safety(cls, path: str):
-        """
-        Function Description:
-            create file safety
-        Parameter:
-            path: the file to remove
-        Exception Description:
-            when invalid data throw exception
-        """
-        if os.path.islink(path):
-            raise RuntimeError(f"Failed to create file: {path}, is a soft link")
-        if os.path.exists(path):
-            logger.warning("File already exists: %s", path)
-            return
-        try:
-            os.close(os.open(path, os.O_WRONLY | os.O_CREAT, cls.DATA_FILE_AUTHORITY))
-        except Exception as err:
-            raise RuntimeError(f"Failed to create file: {path}, err: {err}") from err
-
     @classmethod
     def _input_path_common_check(cls, path: str):
         """
@@ -390,6 +380,57 @@ class PathManager:
             return False
         if os.name == 'nt':
             return False
-        if os.stat(path).st_uid == 0 or os.stat(path).st_uid == os.getuid():
+        if os.stat(lib_path).st_uid == 0 or os.stat(lib_path).st_uid == os.getuid():
             return True
         return False
+
+    @classmethod
+    def check_path_is_other_writable(cls, path):
+        """Check whether the file or directory in the specified path has writable permissions for others."""
+        file_stat = os.stat(path)
+        if file_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH):
+            msg = (f"File path {path} has group or others writable permissions, which is not allowed."
+                   f"Please execute chmod -R 755 {path}")
+            raise ProfilerPathErrorException(msg)
+
+    @classmethod
+    def check_path_is_owner_or_root(cls, path):
+        """Check path is owner or root."""
+        if not os.path.exists(path):
+            msg = f"The path does not exist: {path}"
+            raise ProfilerPathErrorException(msg)
+        file_stat = os.stat(path)
+        current_uid = os.getuid()
+        file_uid = file_stat.st_uid
+        if file_uid not in (0, current_uid):
+            return False
+        return True
+
+    @classmethod
+    def check_path_is_executable(cls, path):
+        """Check path is executable"""
+        return os.access(path, os.X_OK)
+
+    @classmethod
+    def check_path_is_readable(cls, path):
+        """Check path is readable"""
+        if os.path.islink(path):
+            msg = f"Invalid path is a soft link: {path}"
+            raise ProfilerPathErrorException(msg)
+        if not os.access(path, os.R_OK):
+            msg = f"The path readable permission check failed: {path}."
+            raise ProfilerPathErrorException(msg)
+
+    @classmethod
+    def walk_with_depth(cls, path, *args, max_depth=10, **kwargs):
+        """walk path depth"""
+        if not isinstance(path, str):
+            return
+        base_depth = path.count(os.sep)
+        if path.endswith(os.sep):
+            base_depth -= 1
+        for root, dirs, files in os.walk(path, *args, **kwargs):
+            if root.count(os.sep) - base_depth > max_depth:
+                dirs.clear()
+                continue
+            yield root, dirs, files

mindspore/profiler/common/profiler_context.py

@@ -24,7 +24,6 @@ from typing import (
 )
 
 from mindspore.communication.management import GlobalComm
-from mindspore.communication.management import get_local_rank
 from mindspore.communication.management import get_rank
 from mindspore.profiler.common.constant import (
     DeviceTarget,
@@ -43,6 +42,8 @@ from mindspore.profiler.schedule import Schedule
 from mindspore import context
 from mindspore import log as logger
 from mindspore.profiler.common.profiler_info import ProfilerInfo
+from mindspore.profiler.experimental_config import _ExperimentalConfig
+from mindspore.profiler.common.util import get_device_id
 
 
 @Singleton
@@ -81,12 +82,34 @@ class ProfilerContext:
                 logger.warning(f"Both on_trace_ready path and output_path are provided. "
                                f"The on_trace_ready path takes effect. Final path is {final_path}")
             kwargs["output_path"] = final_path
-
+        if kwargs.get("experimental_config"):
+            self._check_and_set_experimental_params(kwargs)
         self._profiler_params_mgr: ProfilerParameters = ProfilerParameters(**kwargs)
         self._profiler_path_mgr: ProfilerOutputPath = ProfilerOutputPath(rank_id=int(self._rank_id))
-
         self._profiler_path_mgr.output_path = self._profiler_params_mgr.output_path
 
+    @staticmethod
+    def _check_and_set_experimental_params(kwargs):
+        """
+        Set experimental parameters
+        """
+        if not isinstance(kwargs.get("experimental_config"), _ExperimentalConfig):
+            logger.warning("For Profiler, experimental_config value must be the "
+                           "'mindspore.profiler._ExperimentalConfig' class, "
+                           "reset to default value.")
+            return
+        kwargs["profiler_level"] = kwargs.get("experimental_config").profiler_level
+        kwargs["aic_metrics"] = kwargs.get("experimental_config").aic_metrics
+        kwargs["l2_cache"] = kwargs.get("experimental_config").l2_cache
+        kwargs["mstx"] = kwargs.get("experimental_config").mstx
+        kwargs["data_simplification"] = kwargs.get("experimental_config").data_simplification
+        kwargs["export_type"] = kwargs.get("experimental_config").export_type
+        kwargs["mstx_domain_include"] = kwargs.get("experimental_config").mstx_domain_include
+        kwargs["mstx_domain_exclude"] = kwargs.get("experimental_config").mstx_domain_exclude
+        kwargs["sys_io"] = kwargs.get("experimental_config").sys_io
+        kwargs["sys_interconnection"] = kwargs.get("experimental_config").sys_interconnection
+        kwargs["host_sys"] = kwargs.get("experimental_config").host_sys
+
     @property
     def on_trace_ready_output_path(self) -> str:
         """Get the on trace ready output path."""
@@ -465,17 +488,7 @@ class ProfilerContext:
         """
         Initialize the device ID.
         """
-        self._device_id = str(context.get_context("device_id"))
-
-        if not self._device_id or not self._device_id.isdigit():
-            if GlobalComm.INITED and self._device_target == DeviceTarget.NPU.value:
-                self._device_id = str(get_local_rank())
-            else:
-                self._device_id = os.getenv("DEVICE_ID")
-
-        if not self._device_id or not self._device_id.isdigit():
-            self._device_id = "0"
-            logger.warning("Fail to get DEVICE_ID, use 0 instead.")
+        self._device_id = get_device_id()
 
     def _init_rank_id(self) -> None:
         """

mindspore/profiler/common/profiler_info.py

@@ -97,13 +97,14 @@ class ProfilerInfo:
         Load time parameters from msprof profile and host start log.
         This method should be called before TimeConverter.init_parameters.
         """
+        msprof_info = MsprofCmdTool(msprof_profile_path).get_msprof_info()
         if not msprof_profile_path or not msprof_profile_host_path:
             raise ValueError(
                 "msprof_profile_path and msprof_profile_host_path must be provided"
             )
         self._read_host_start_log(msprof_profile_host_path)
         self._read_start_info(msprof_profile_host_path)
-        self._get_freq_from_msprof(msprof_profile_path)
+        self._get_freq_from_msprof(msprof_info)
 
     @property
     def time_parameters(self) -> Dict[str, Any]:
@@ -237,7 +238,7 @@ class ProfilerInfo:
             self._collection_time_begin * self.US_TO_NS - self._clock_monotonic_raw_info
         )
 
-    def _get_freq_from_msprof(self, msprof_profile_path: str) -> None:
+    def _get_freq_from_msprof(self, msprof_info: str) -> None:
         """
         Get frequency from get_msprof_info.py script
 
@@ -250,7 +251,6 @@ class ProfilerInfo:
             }
         }
         """
-        msprof_info = MsprofCmdTool(msprof_profile_path).get_msprof_info()
 
         if not isinstance(msprof_info, dict):
             raise RuntimeError("msprof_info must be a dictionary")

mindspore/profiler/common/profiler_meta_data.py

@@ -30,6 +30,7 @@ class ProfilerMetaData:
         This class is used to handle metadata.
     """
     metadata: Dict[str, str] = {}
+    MAX_META_SIZE = 100 * 1024 * 1024  # 100MB
 
     @classmethod
     def get_metadata(cls) -> Dict[str, str]:

mindspore/profiler/common/profiler_op_analyse.py

@@ -16,7 +16,7 @@
 import json
 import os
 from collections import defaultdict
-from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
+from mindspore.profiler.common.path_manager import PathManager
 
 
 class OpAnalyser:
@@ -34,7 +34,7 @@ class OpAnalyser:
     def __init__(self, output_path, dev_id, op_names=None):
         """The parser for parsing framework files."""
         self._dev_id = dev_id
-        self._output_path = output_path
+        self._output_path = PathManager.get_real_path(output_path)
         self.op_names = op_names
         self.op_name = ''
         self.framework_list = []
@@ -66,7 +66,8 @@ class OpAnalyser:
         for filename in self.framework_info_dir:
             op_side = filename.split('_')[0]
             framework_file_path = os.path.join(self._output_path, filename)
-            framework_file_path = validate_and_normalize_path(framework_file_path)
+            PathManager.check_input_file_path(framework_file_path)
+            PathManager.check_directory_path_readable(framework_file_path)
             with open(framework_file_path, 'r') as f_obj:
                 framework_info = f_obj.readlines()
             for line_info in framework_info:
@@ -85,7 +86,8 @@ class OpAnalyser:
         for filename in self.cpu_detail_info_dir:
             op_side = filename.split('_')[0]
             op_detail_file_path = os.path.join(self._output_path, filename)
-            op_detail_file_path = validate_and_normalize_path(op_detail_file_path)
+            PathManager.check_input_file_path(op_detail_file_path)
+            PathManager.check_directory_path_readable(op_detail_file_path)
             with open(op_detail_file_path, 'r') as f_obj:
                 op_detail_info = f_obj.readlines()
             for line_info in op_detail_info[1:]:
@@ -99,7 +101,8 @@ class OpAnalyser:
         """Get gpu operators execute times."""
         if self.gpu_op_type_info_dir:
             gpu_op_type_file_path = os.path.join(self._output_path, self.gpu_op_type_info_dir[0])
-            gpu_op_type_file_path = validate_and_normalize_path(gpu_op_type_file_path)
+            PathManager.check_input_file_path(gpu_op_type_file_path)
+            PathManager.check_directory_path_readable(gpu_op_type_file_path)
             with open(gpu_op_type_file_path, 'r') as fp:
                 op_type_info = fp.readlines()
                 for line_info in op_type_info[1:]:
@@ -118,7 +121,8 @@ class OpAnalyser:
         for filename in self.activity_info_dir:
             op_side = filename.split('_')[0]
             activity_file_path = os.path.join(self._output_path, filename)
-            activity_file_path = validate_and_normalize_path(activity_file_path)
+            PathManager.check_input_file_path(activity_file_path)
+            PathManager.check_directory_path_readable(activity_file_path)
             with open(activity_file_path, 'r') as file:
                 activity_info = file.readlines()
             for line_info in activity_info[1:]:

mindspore/profiler/common/profiler_path_manager.py

@@ -143,6 +143,19 @@ class ProfilerPathManager:
                     new_file_path = os.path.join(self._prof_ctx.ascend_profiler_output_path, new_file_name)
                     shutil.move(db_file, new_file_path)
 
+    def remove_db_file(self):
+        """
+        Remove the db file in the output path.
+        """
+        if not self._prof_ctx.msprof_profile_output_path:
+            return
+        db_files = glob.glob(os.path.join(
+            os.path.dirname(self._prof_ctx.msprof_profile_output_path),
+            'msprof*.db'
+        ))
+        for db_file in db_files:
+            if os.path.isfile(db_file):
+                os.remove(db_file)
 
     def create_output_path(self):
         """

mindspore/profiler/common/util.py

@@ -25,10 +25,15 @@ import re
 import shutil
 import stat
 
+from mindspore.communication.management import GlobalComm
+from mindspore.communication.management import get_local_rank
+from mindspore import context
 from mindspore import log as logger
 from mindspore.profiler.common.path_manager import PathManager
 from mindspore.profiler.common.exceptions.exceptions import ProfilerPathErrorException
 
+from mindspore.profiler.common.constant import DeviceTarget
+
 
 def no_exception_func(
         default_ret: Any = None,
@@ -75,12 +80,16 @@ def get_cann_version():
     ascend_home_path = os.environ.get("ASCEND_HOME_PATH", "")
     cann_version = "not known"
     try:
-        PathManager.check_directory_path_readable(os.path.realpath(ascend_home_path))
-        for dirpath, _, filenames in os.walk(os.path.realpath(ascend_home_path)):
+        if not PathManager.check_path_is_owner_or_root(ascend_home_path):
+            raise PermissionError(f"PermissionError, CANN package user id: {os.stat(ascend_home_path).st_uid}, "
+                                  f"current user id: {os.getuid()}. "
+                                  f"Ensure CANN package user id and current user id consistency")
+        PathManager.check_path_is_readable(os.path.realpath(ascend_home_path))
+        for dirpath, _, filenames in PathManager.walk_with_depth(os.path.realpath(ascend_home_path)):
             install_files = [file for file in filenames if re.match(r"ascend_.{1,20}_install\.info", file)]
             if install_files:
                 filepath = os.path.realpath(os.path.join(dirpath, install_files[0]))
-                PathManager.check_directory_path_readable(filepath)
+                PathManager.check_path_is_readable(filepath)
                 with open(filepath, "r") as f:
                     for line in f:
                         if line.find("version") != -1:
@@ -441,6 +450,24 @@ def get_newest_file(file_list):
     return newest_file_list
 
 
+def get_device_id():
+    """
+    Get device ID.
+    """
+    device_id = str(context.get_context("device_id"))
+
+    if not device_id or not device_id.isdigit():
+        if GlobalComm.INITED and context.get_context("device_target") == DeviceTarget.NPU.value:
+            device_id = str(get_local_rank())
+        else:
+            device_id = os.getenv("DEVICE_ID")
+
+    if not device_id or not device_id.isdigit():
+        logger.warning("Fail to get DEVICE_ID, use 0 instead.")
+        device_id = "0"
+    return device_id
+
+
 class ProfilerPathManager:
     """A path manager to manage profiler path"""