mindspore 2.4.1__cp310-cp310-manylinux1_x86_64.whl → 2.5.0__cp310-cp310-manylinux1_x86_64.whl

mindspore/profiler/common/log.py

@@ -0,0 +1,130 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Profiler Log Module"""
+import os
+import logging
+from logging.handlers import RotatingFileHandler
+from datetime import datetime
+from typing import Optional
+
+from mindspore.profiler.common.path_manager import PathManager
+
+
+class ProfilerLogger:
+    """
+    Profiler Logger class for managing log operations.
+
+    This class provides a centralized logging mechanism for the profiler,
+    writing logs to file with rotation support.
+
+    Attributes:
+        LOG_FORMAT: The format string for log messages
+        DATE_FORMAT: The format string for timestamps in log messages
+        DEFAULT_LOGGER_NAME: Default name for the logger instance
+        DEFAULT_LOG_DIR: Default directory name for log files
+        MAX_BYTES: Maximum size of each log file
+        BACKUP_COUNT: Number of backup files to keep
+    """
+
+    LOG_FORMAT = "[%(asctime)s] [%(levelname)s] [%(filename)s:%(lineno)d] %(message)s"
+    DATE_FORMAT = "%Y-%m-%d-%H:%M:%S"
+    DEFAULT_LOGGER_NAME = "MindSporeProfiler"
+    DEFAULT_LOG_LEVEL = logging.INFO
+    DEFAULT_LOG_DIR = "logs"
+    # 10MB per file
+    MAX_BYTES = 10 * 1024 * 1024
+    # Keep 3 backup files
+    BACKUP_COUNT = 3
+
+    _instance: Optional[logging.Logger] = None
+
+    @classmethod
+    def get_instance(cls) -> logging.Logger:
+        """Get the singleton logger instance."""
+        if cls._instance is None:
+            raise RuntimeError("Logger not initialized. Call init first.")
+        return cls._instance
+
+    @classmethod
+    def init(cls, output_dir: str, custom_name: Optional[str] = None) -> None:
+        """
+        Initialize the logger with rotating file handler.
+
+        Args:
+            output_dir (str): Directory where log files will be stored
+
+        Raises:
+            RuntimeError: If logger initialization fails
+        """
+        if cls._instance is not None:
+            return
+
+        # Create logs directory
+        log_dir = os.path.join(output_dir, cls.DEFAULT_LOG_DIR)
+        PathManager.make_dir_safety(log_dir)
+        PathManager.check_directory_path_writeable(log_dir)
+
+        # Create logger
+        logger = logging.getLogger(cls.DEFAULT_LOGGER_NAME)
+        logger.setLevel(cls.DEFAULT_LOG_LEVEL)
+        logger.propagate = False
+
+        # Create formatters
+        formatter = logging.Formatter(fmt=cls.LOG_FORMAT, datefmt=cls.DATE_FORMAT)
+
+        # Add rotating file handler
+        timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
+        log_file = os.path.join(
+            log_dir,
+            (
+                f"profiler_{timestamp}_{os.getpid()}_{custom_name}.log"
+                if custom_name
+                else f"profiler_{timestamp}_{os.getpid()}.log"
+            ),
+        )
+        file_handler = RotatingFileHandler(
+            filename=log_file,
+            maxBytes=cls.MAX_BYTES,
+            backupCount=cls.BACKUP_COUNT,
+            encoding="utf-8",
+        )
+        file_handler.setFormatter(formatter)
+        file_handler.setLevel(cls.DEFAULT_LOG_LEVEL)
+        logger.addHandler(file_handler)
+
+        cls._instance = logger
+        logger.info("Profiler logger initialized at: %s", log_file)
+
+    @classmethod
+    def set_level(cls, level: int) -> None:
+        """
+        Set the logging level for both file and console handlers.
+
+        Args:
+            level (int): Logging level (e.g., logging.DEBUG, logging.INFO)
+        """
+        logger = cls.get_instance()
+        logger.setLevel(level)
+        for handler in logger.handlers:
+            handler.setLevel(level)
+
+    @classmethod
+    def destroy(cls) -> None:
+        """Close and cleanup the logger."""
+        if cls._instance:
+            for handler in cls._instance.handlers[:]:
+                handler.close()
+                cls._instance.removeHandler(handler)
+            cls._instance = None

mindspore/profiler/common/msprof_cmd_tool.py

@@ -0,0 +1,202 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Tool for managing Ascend msprof profiling commands and environment."""
+import os
+import json
+import shutil
+from functools import lru_cache
+from typing import Dict, List, Optional
+
+from mindspore import log as logger
+from mindspore.profiler.common.command_executor import CommandExecutor
+
+
+class MsprofCmdTool:
+    """Tool for managing Ascend msprof profiling commands and environment."""
+
+    _MSPROF_CMD = "msprof"
+    _ASCEND_MARK = "Ascend"
+    _HIAI_MSPROF_TAIL = "Ascend/latest/tools/profiler/bin"
+    _MSPROF_PY_PATH = "tools/profiler/profiler_tool/analysis/msprof/msprof.py"
+    _MSPROF_INFO_PATH = "tools/profiler/profiler_tool/analysis/interface/get_msprof_info.py"
+
+    def __init__(self, msprof_profile_path: str):
+        """Initialize MsprofTool."""
+        self._msprof_profile_path = msprof_profile_path
+        self._check_environment()
+
+    def run_ms_export_cmd(self) -> None:
+        """Run msprof export command."""
+        CommandExecutor.execute(
+            [
+                self._MSPROF_CMD,
+                "--export=on",
+                f"--output={self._msprof_profile_path}"
+            ]
+        )
+
+    def run_ms_py_export_cmd(self, model_id: int, iter_list: List[int]) -> None:
+        """Export timeline and summary data for the specified model and iterations.
+
+        Args:
+            model_id (int): The ID of the model to export data for.
+            iter_list (List[int]): A list of iteration IDs to export data for.
+
+        Raises:
+            FileNotFoundError: If msprof.py path cannot be found.
+        """
+        script_path = self._get_msprof_script_path(self._MSPROF_PY_PATH)
+        if not script_path:
+            raise FileNotFoundError(
+                "Failed to find msprof.py path. Please check the CANN environment."
+            )
+
+        export_cmd = ["python3", script_path]
+        iter_param = self._get_iteration_params(model_id, iter_list)
+
+        for export_type in ("timeline", "summary"):
+            cmd = (
+                export_cmd
+                + ["export", export_type, "-dir", self._msprof_profile_path]
+                + iter_param
+            )
+            CommandExecutor.execute(cmd)
+
+    def run_ms_analyze_cmd(self) -> None:
+        """Run msprof analyze command."""
+        CommandExecutor.execute(
+            [
+                self._MSPROF_CMD,
+                "--analyze=on",
+                "--rule=communication,communication_matrix",
+                f"--output={self._msprof_profile_path}",
+            ]
+        )
+
+    @lru_cache(maxsize=1)
+    def get_msprof_info(self):
+        """Get the msprof info.
+
+        Returns:
+            Dict: Msprof information dictionary.
+        """
+        msprof_info = self._setup_msprof_info()
+        return msprof_info
+
+    def _check_environment(self) -> None:
+        """Check if required commands are available in the environment.
+
+        Raises:
+            FileNotFoundError: If msprof or python3 command is not found.
+        """
+        if not shutil.which(self._MSPROF_CMD):
+            logger.warning(
+                "The msprof command is not found in PATH. Searching in environment variables..."
+            )
+            msprof_path = self._find_msprof_path()
+
+            if msprof_path:
+                os.environ["PATH"] = f"{msprof_path}:{os.environ.get('PATH', '')}"
+                logger.info("Successfully added msprof command to PATH.")
+            else:
+                raise FileNotFoundError("Failed to find msprof command in environment.")
+
+        if not shutil.which("python3"):
+            logger.warning("Failed to find python3 command in environment.")
+            raise FileNotFoundError("Failed to find python3 command in environment.")
+
+    def _find_msprof_path(self) -> Optional[str]:
+        """Find msprof path in environment variables.
+
+        Returns:
+            Optional[str]: Path to msprof if found, None otherwise.
+        """
+        if os.environ.get("ASCEND_TOOLKIT_HOME"):
+            temp_path = os.path.join(os.environ.get("ASCEND_TOOLKIT_HOME"), "bin")
+            if os.path.isdir(temp_path) and self._MSPROF_CMD in os.listdir(temp_path):
+                return temp_path
+
+        for path in os.environ.get("PATH", "").split(":"):
+            if self._ASCEND_MARK in path:
+                prefix = path.split(self._ASCEND_MARK)[0]
+                temp_path = os.path.join(prefix, self._HIAI_MSPROF_TAIL)
+                if os.path.isdir(temp_path) and self._MSPROF_CMD in os.listdir(temp_path):
+                    return temp_path
+
+        return None
+
+    def _setup_msprof_info(self) -> Dict:
+        """Get msprof information.
+
+        Returns:
+            Dict: Msprof information dictionary.
+        """
+        script_path = self._get_msprof_script_path(self._MSPROF_INFO_PATH)
+        if not script_path:
+            logger.error("Failed to find get_msprof_info.py path.")
+            return {}
+
+        host_dir = os.path.join(self._msprof_profile_path, "host")
+        cmd = ["python3", script_path, "-dir", host_dir]
+        command_outs = CommandExecutor.execute(cmd)[0]
+
+        try:
+            return json.loads(command_outs)
+        except json.JSONDecodeError as err:
+            logger.error(f"Failed to decode msprof info. Error: {str(err)}")
+            return {}
+
+    def _get_msprof_script_path(self, script_path: str) -> str:
+        """Get the full path of a msprof script.
+
+        Args:
+            script_path (str): Relative path to the script.
+
+        Returns:
+            str: Full path to the script if found, empty string otherwise.
+        """
+        msprof_path = shutil.which(self._MSPROF_CMD)
+        if not msprof_path:
+            return ""
+        msprof_path = os.path.realpath(msprof_path.strip())
+        pre_path = msprof_path.split("tools")[0]
+        full_script_path = os.path.join(pre_path, script_path)
+        return full_script_path if os.path.exists(full_script_path) else ""
+
+    @staticmethod
+    def _get_iteration_params(model_id: int, iter_list: List[int]) -> List[str]:
+        """Get iteration parameters for msprof.py script.
+
+        Args:
+            model_id (int): Model ID.
+            iter_list (List[int]): List of iteration IDs.
+
+        Returns:
+            List[str]: List of parameter strings.
+        """
+        iter_param = []
+        if isinstance(model_id, int) and model_id >= 0:
+            iter_param.extend(["--model-id", str(model_id)])
+        if iter_list and isinstance(iter_list, list):
+            iter_list.sort()
+            iter_param.extend(
+                [
+                    "--iteration-id",
+                    str(iter_list[0]),
+                    "--iteration-count",
+                    str(len(iter_list)),
+                ]
+            )
+        return iter_param

mindspore/profiler/common/path_manager.py

@@ -0,0 +1,371 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Profiler path manager"""
+import os
+import re
+import shutil
+import glob
+
+from mindspore import log as logger
+from mindspore.profiler.common.constant import FileConstant
+from mindspore.profiler.common.exceptions.exceptions import ProfilerPathErrorException
+
+
+class PathManager:
+    """
+    Path common operations manager
+    """
+    MAX_PATH_LENGTH = 4096
+    MAX_FILE_NAME_LENGTH = 255
+    DATA_FILE_AUTHORITY = 0o640
+    DATA_DIR_AUTHORITY = 0o750
+    MAX_FILE_SIZE = 1024 * 1024 * 1024 * 10
+
+    @classmethod
+    def check_input_directory_path(cls, path: str):
+        """
+        Function Description:
+            check whether the path is valid, some businesses can accept a path that does not exist,
+            so the function do not verify whether the path exists
+        Parameter:
+            path: the path to check, whether the incoming path is absolute or relative depends on the business
+        Exception Description:
+            when invalid data throw exception
+        """
+        cls._input_path_common_check(path)
+
+        if os.path.isfile(path):
+            msg = f"Invalid input path is a file path: {path}"
+            raise ProfilerPathErrorException(msg)
+
+    @classmethod
+    def check_input_file_path(cls, path: str):
+        """
+        Function Description:
+            check whether the file path is valid, some businesses can accept a path that does not exist,
+            so the function do not verify whether the path exists
+        Parameter:
+            path: the file path to check, whether the incoming path is absolute or relative depends on the business
+        Exception Description:
+            when invalid data throw exception
+        """
+        cls._input_path_common_check(path)
+
+        if os.path.isdir(path):
+            msg = f"Invalid input path is a directory path: {path}"
+            raise ProfilerPathErrorException(msg)
+
+        file_size = os.path.getsize(path)
+        if file_size >= cls.MAX_FILE_SIZE:
+            msg = f"file size exceeds the limit: {cls.MAX_FILE_SIZE}, file size: {file_size}"
+            raise ProfilerPathErrorException(msg)
+
+    @classmethod
+    def get_directory_size(cls, directory: str, unit: str = 'MB') -> float:
+        """
+        Function Description:
+            Get the size of the directory
+        Parameter:
+            directory: the directory path
+            unit: the unit of the size, default is MB
+        Return:
+            float: the size of the directory
+        """
+        if not os.path.exists(directory):
+            logger.warning("Get directory size failed, %s not exists", directory)
+            return 0.0
+
+        cls.check_input_directory_path(directory)
+        unit_map = {
+            'B': 1,
+            'KB': 1024,
+            'MB': 1024 * 1024,
+            'GB': 1024 * 1024 * 1024
+        }
+
+        if unit not in unit_map:
+            logger.error("Invalid unit: %s", unit)
+            return 0.0
+
+        total_size = 0
+        for dirpath, _, filenames in os.walk(directory):
+            for filename in filenames:
+                file_path = os.path.join(dirpath, filename)
+                try:
+                    total_size += os.path.getsize(file_path)
+                except (OSError, FileNotFoundError):
+                    continue
+
+        return total_size / unit_map[unit]
+
+    @classmethod
+    def check_path_owner_consistent(cls, path: str):
+        """
+        Function Description:
+            check whether the path belong to process owner
+        Parameter:
+            path: the path to check
+        Exception Description:
+            when invalid path, prompt the user
+        """
+
+        if not os.path.exists(path):
+            msg = f"The path does not exist: {path}"
+            raise ProfilerPathErrorException(msg)
+        if os.name != 'nt' and os.stat(path).st_uid != os.getuid():
+            msg = f"Path {path} owner[{os.stat(path).st_uid}] does not match the current user[{os.getuid()}]."
+            raise ProfilerPathErrorException(msg)
+
+    @classmethod
+    def check_directory_path_writeable(cls, path):
+        """
+        Function Description:
+            check whether the path is writable
+        Parameter:
+            path: the path to check
+        Exception Description:
+            when invalid data throw exception
+        """
+        cls.check_path_owner_consistent(path)
+        if os.path.islink(path):
+            msg = f"Invalid path is a soft link: {path}"
+            raise ProfilerPathErrorException(msg)
+        if not os.access(path, os.W_OK):
+            msg = f"The path writeable permission check failed: {path}"
+            raise ProfilerPathErrorException(msg)
+
+    @classmethod
+    def check_directory_path_readable(cls, path):
+        """
+        Function Description:
+            check whether the path is writable
+        Parameter:
+            path: the path to check
+        Exception Description:
+            when invalid data throw exception
+        """
+        cls.check_path_owner_consistent(path)
+        if os.path.islink(path):
+            msg = f"Invalid path is a soft link: {path}"
+            raise ProfilerPathErrorException(msg)
+        if not os.access(path, os.R_OK):
+            msg = f"The path readable permission check failed: {path}"
+            raise ProfilerPathErrorException(msg)
+
+    @classmethod
+    def remove_path_safety(cls, path: str):
+        """
+        Function Description:
+            remove path safety
+        Parameter:
+            path: the path to remove
+        Exception Description:
+            when invalid data throw exception
+        """
+        if not os.path.exists(path):
+            logger.warning("The path does not exist: %s", path)
+            return
+
+        if os.path.islink(path):
+            msg = f"Failed to remove path: {path}, is a soft link"
+            raise ProfilerPathErrorException(msg)
+
+        try:
+            shutil.rmtree(path)
+        except PermissionError as err:
+            raise ProfilerPathErrorException(f"Permission denied while removing path: {path}") from err
+        except Exception as err:
+            raise ProfilerPathErrorException(f"Failed to remove path: {path}, err: {err}") from err
+
+    @classmethod
+    def remove_file_safety(cls, file: str):
+        """
+        Function Description:
+            remove file safety
+        Parameter:
+            path: the file to remove
+        Exception Description:
+            when invalid data throw exception
+        """
+        if not os.path.exists(file):
+            logger.warning("The file does not exist: %s", file)
+            return
+
+        if os.path.islink(file):
+            msg = f"Failed to remove file: {file}, is a soft link"
+            raise ProfilerPathErrorException(msg)
+
+        try:
+            os.remove(file)
+        except PermissionError as err:
+            raise ProfilerPathErrorException(f"Permission denied while removing file: {file}") from err
+        except Exception as err:
+            raise ProfilerPathErrorException(f"Failed to remove file: {file}, err: {err}") from err
+
+    @classmethod
+    def make_dir_safety(cls, path: str):
+        """
+        Function Description:
+            make directory safety
+        Parameter:
+            path: the directory to remove
+        Exception Description:
+            when invalid data throw exception
+        """
+        if os.path.exists(path):
+            return
+
+        if os.path.islink(path):
+            msg = f"Failed to make directory: {path}, is a soft link"
+            raise ProfilerPathErrorException(msg)
+
+        try:
+            os.makedirs(path, mode=cls.DATA_DIR_AUTHORITY, exist_ok=True)
+        except Exception as err:
+            raise ProfilerPathErrorException(f"Failed to make directory: {path}, err: {err}") from err
+
+    @classmethod
+    def create_file_safety(cls, path: str):
+        """
+        Function Description:
+            create file safety
+        Parameter:
+            path: the file to remove
+        Exception Description:
+            when invalid data throw exception
+        """
+        if os.path.islink(path):
+            raise RuntimeError(f"Failed to create file: {path}, is a soft link")
+        if os.path.exists(path):
+            logger.warning("File already exists: %s", path)
+            return
+        try:
+            os.close(os.open(path, os.O_WRONLY | os.O_CREAT, cls.DATA_FILE_AUTHORITY))
+        except Exception as err:
+            raise RuntimeError(f"Failed to create file: {path}, err: {err}") from err
+
+    @classmethod
+    def _input_path_common_check(cls, path: str):
+        """
+        Function Description:
+            input path check common function
+        Parameter:
+            path: the file path to check
+        Exception Description:
+            when invalid data throw exception
+        """
+        if len(path) > cls.MAX_PATH_LENGTH:
+            msg = f"Path {path} length {len(path)} exceeds the limit {cls.MAX_PATH_LENGTH}."
+            raise ProfilerPathErrorException(msg)
+
+        if os.path.islink(path):
+            msg = f"Invalid input path is a soft link: {path}"
+            raise ProfilerPathErrorException(msg)
+
+        pattern = r"(\.|/|_|-|\s|[~0-9a-zA-Z]|[\u4e00-\u9fa5])+"
+        if not re.fullmatch(pattern, path):
+            msg = f"Invalid input path: {path}, contains invalid characters."
+            raise ProfilerPathErrorException(msg)
+
+        path_split_list = path.split("/")
+        for name in path_split_list:
+            if len(name) > cls.MAX_FILE_NAME_LENGTH:
+                msg = f"Length of input path {path} file name {name} exceeds the limit {cls.MAX_FILE_NAME_LENGTH}."
+                raise ProfilerPathErrorException(msg)
+
+    @classmethod
+    def get_ascend_ms_path_list(cls, input_path: str):
+        """
+        Function Description:
+            get valid profiler {}_ascend_ms_dir path list from input_path
+        Parameter:
+            input_path: The directory path from which to extract profiler parent paths.
+        Return:
+            A list containing the input path or its subdirectories that are valid profiler parents.
+        """
+        if os.path.isdir(input_path) and (cls.get_fwk_path(input_path) or cls.get_cann_path(input_path)):
+            return [input_path]
+        sub_dirs = os.listdir(os.path.realpath(input_path))
+        profiler_ascend_ms_path_list = []
+        for sub_dir in sub_dirs:
+            sub_path = os.path.join(input_path, sub_dir)
+            if not os.path.isdir(sub_path):
+                continue
+            if cls.get_fwk_path(sub_path) or cls.get_cann_path(sub_path):
+                profiler_ascend_ms_path_list.append(os.path.join(input_path, sub_dir))
+        return profiler_ascend_ms_path_list
+
+    @classmethod
+    def get_fwk_path(cls, input_path: str):
+        """
+        Function Description:
+            get valid framework path from input_path
+        Parameter:
+            input_path: the directory path to check whether exist valid FRAMEWORK path
+        Return:
+            The path to the FRAMEWORK directory if found, otherwise an empty string.
+        """
+        fwk_path = os.path.join(input_path, FileConstant.FRAMEWORK_DIR)
+        if os.path.isdir(fwk_path):
+            return fwk_path
+        return ""
+
+    @classmethod
+    def get_cann_path(cls, input_path: str):
+        """
+        Function Description:
+            get valid PROF_XXX path from input_path
+        Parameter:
+            input_path: the directory path to check valid PROF_XXX path
+        Return:
+            The path to the PROF_XXX directory if it matches the pattern and exists, otherwise an empty string.
+        """
+        sub_dirs = os.listdir(os.path.realpath(input_path))
+        for sub_dir in sub_dirs:
+            sub_path = os.path.join(input_path, sub_dir)
+            if os.path.isdir(sub_path) and re.match(FileConstant.CANN_FILE_REGEX, sub_dir):
+                return sub_path
+        return ""
+
+    @classmethod
+    def get_profiler_info_path(cls, ascend_ms_dir: str) -> str:
+        """
+        Function Description:
+            Get profiler_info_*.json path from ascend_ms_dir
+        Parameter:
+            ascend_ms_dir: the directory path of profiler data, eg: xxx_ascend_ms
+        Return:
+            str type profiler_info_*.json path
+        """
+        prof_info_path_pattern = os.path.join(ascend_ms_dir, "profiler_info_*.json")
+        prof_info_paths = glob.glob(prof_info_path_pattern)
+
+        if not prof_info_paths:
+            raise ValueError(f"Cannot find profiler_info.json in the {ascend_ms_dir}")
+
+        if len(prof_info_paths) > 1:
+            logger.warning(
+                f"There are more than one profiler_info.json in the {ascend_ms_dir}, "
+                f"use the first one: {prof_info_paths[0]}"
+            )
+        return prof_info_paths[0]
+
+    @classmethod
+    def get_real_path(cls, path: str):
+        expanded_path = os.path.expanduser(path)
+        if os.path.islink(path):
+            msg = f"Invalid input path is a soft link: {path}"
+            raise ProfilerPathErrorException(msg)
+        return os.path.realpath(expanded_path)