mindspore 2.4.0__cp310-cp310-macosx_10_15_x86_64.whl

mindspore/profiler/__init__.py

@@ -0,0 +1,28 @@
+# Copyright 2020-2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+This module provides Python APIs to enable the profiling of MindSpore neural networks.
+Users can import the mindspore.profiler.Profiler, initialize the Profiler object to start profiling,
+and use Profiler.analyse() to stop profiling and analyse the results.
+Users can visualize the results using the MindInsight tool.
+Now, Profiler supports AICORE operator, AICPU operator, HostCPU operator, memory,
+correspondence, cluster, etc data analysis.
+"""
+__all__ = ["Profiler", "EnvProfiler", "ProfilerLevel", "DynamicProfilerMonitor"]
+
+from mindspore.profiler.profiling import Profiler
+from mindspore.profiler.envprofiling import EnvProfiler
+from mindspore.profiler.profiling import ProfilerLevel
+from mindspore.profiler.dynamic_profiler import DynamicProfilerMonitor

mindspore/profiler/common/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================

mindspore/profiler/common/constant.py

@@ -0,0 +1,29 @@
+# Copyright 2020-2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Constant values"""
+from enum import Enum
+
+
+class DeviceTarget(Enum):
+    """The device target enum."""
+    CPU = 'cpu'
+    GPU = 'gpu'
+    ASCEND = 'ascend'
+
+
+class ProfilerLevel(Enum):
+    Level0 = "Level0"
+    Level1 = "Level1"
+    Level2 = "Level2"

mindspore/profiler/common/exceptions/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================

mindspore/profiler/common/exceptions/error_code.py

@@ -0,0 +1,83 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Profiler error code and messages."""
+from enum import unique, Enum
+
+_GENERAL_MASK = 0b00001 << 7
+_PARSER_MASK = 0b00010 << 7
+_ANALYSER_MASK = 0b00011 << 7
+
+
+class ProfilerMgrErrors(Enum):
+    """Enum definition for profiler errors"""
+
+
+@unique
+class ProfilerErrors(ProfilerMgrErrors):
+    """Profiler error codes."""
+    # general error code
+    PARAM_VALUE_ERROR = 0 | _GENERAL_MASK
+    PATH_ERROR = 1 | _GENERAL_MASK
+    PARAM_TYPE_ERROR = 2 | _GENERAL_MASK
+    DIR_NOT_FOUND_ERROR = 3 | _GENERAL_MASK
+    FILE_NOT_FOUND_ERROR = 4 | _GENERAL_MASK
+    IO_ERROR = 5 | _GENERAL_MASK
+
+    # parser error code
+    DEVICE_ID_MISMATCH_ERROR = 0 | _PARSER_MASK
+    RAW_FILE_ERROR = 1 | _PARSER_MASK
+    STEP_NUM_NOT_SUPPORTED_ERROR = 2 | _PARSER_MASK
+    JOB_ID_MISMATCH_ERROR = 3 | _PARSER_MASK
+
+    # analyser error code
+    COLUMN_NOT_EXIST_ERROR = 0 | _ANALYSER_MASK
+    ANALYSER_NOT_EXIST_ERROR = 1 | _ANALYSER_MASK
+    DEVICE_ID_ERROR = 2 | _ANALYSER_MASK
+    OP_TYPE_ERROR = 3 | _ANALYSER_MASK
+    GROUP_CONDITION_ERROR = 4 | _ANALYSER_MASK
+    SORT_CONDITION_ERROR = 5 | _ANALYSER_MASK
+    FILTER_CONDITION_ERROR = 6 | _ANALYSER_MASK
+    COLUMN_NOT_SUPPORT_SORT_ERROR = 7 | _ANALYSER_MASK
+    PIPELINE_OP_NOT_EXIST_ERROR = 8 | _ANALYSER_MASK
+
+
+@unique
+class ProfilerErrorMsg(Enum):
+    """Profiler error messages."""
+    # general error msg
+    PARAM_VALUE_ERROR = 'Param value error. {}'
+    PATH_ERROR = 'Path error. {}'
+    PARAM_TYPE_ERROR = 'Param type error. {}'
+    DIR_NOT_FOUND_ERROR = 'The dir <{}> not found.'
+    FILE_NOT_FOUND_ERROR = 'The file <{}> not found.'
+    IO_ERROR = 'Read or write file fail.'
+
+    # parser error msg
+    DEVICE_ID_MISMATCH_ERROR = 'The device ID mismatch.'
+    RAW_FILE_ERROR = 'Raw file error. {}'
+    STEP_NUM_NOT_SUPPORTED_ERROR = 'The step num must be in {}'
+    JOB_ID_MISMATCH_ERROR = 'The job id in the parameter is not the same as ' \
+                            'in the training trace file. '
+
+    # analyser error msg
+    COLUMN_NOT_EXIST_ERROR = 'The column {} does not exist.'
+    ANALYSER_NOT_EXIST_ERROR = 'The analyser {} does not exist.'
+    DEIVICE_ID_ERROR = 'The device_id in search_condition error, {}'
+    FILTER_CONDITION_ERROR = 'The filter_condition in search_condition error, {}'
+    OP_TYPE_ERROR = 'The op_type in search_condition error, {}'
+    GROUP_CONDITION_ERROR = 'The group_condition in search_condition error, {}'
+    SORT_CONDITION_ERROR = 'The sort_condition in search_condition error, {}'
+    COLUMN_NOT_SUPPORT_SORT_ERROR = 'The column {} does not support to sort.'
+    PIPELINE_OP_NOT_EXIST_ERROR = 'The minddata pipeline operator {} does not exist.'

mindspore/profiler/common/exceptions/exceptions.py

@@ -0,0 +1,286 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Definition of error code and relative messages in profiler module."""
+from mindspore.profiler.common.exceptions.error_code import ProfilerErrors, \
+    ProfilerErrorMsg
+
+
+class ProfilerException(Exception):
+    """
+    Base class for Profilier exception.
+
+    Examples:
+        >>> raise ProfilerException(GeneralErrors.PATH_NOT_EXISTS_ERROR, 'path not exists')
+    """
+
+    RUNTIME = 1
+    TYPE = 1
+    LEVEL = 0
+    SYSID = 42
+
+    def __init__(self, error, message, http_code=500):
+        """
+        Initialization of ProfilerException.
+
+        Args:
+            error (Enum): Error value for specified case.
+            message (str): Description for exception.
+            http_code (int): Http code for exception. Default is 500.
+        """
+        if isinstance(message, str):
+            message = ' '.join(message.split())
+        super(ProfilerException, self).__init__(message)
+        self.error = error
+        self.message = message
+        self.http_code = http_code
+
+    def __str__(self):
+        return '[{}] code: {}, msg: {}'.format(self.__class__.__name__, self.error_code, self.message)
+
+    @property
+    def error_code(self):
+        """
+        Transform exception no to Profiler error code.
+
+        code compose(4bytes):
+        runtime 2bits, type 2bits, level 3bits, sysid 8bits, modid 5bits, value 12bits.
+
+        num = ((0xFF & runtime) << 30) \
+                | ((0xFF & type) << 28) \
+                | ((0xFF & level) << 25) \
+                | ((0xFF & sysid) << 17) \
+                | ((0xFF & modid) << 12) \
+                | (0x0FFF & value)
+
+        Returns:
+            str, Hex string representing the composed Profiler error code.
+        """
+        num = (((0xFF & self.RUNTIME) << 30)
+               | ((0xFF & self.TYPE) << 28)
+               | ((0xFF & self.LEVEL) << 25)
+               | ((0xFF & self.SYSID) << 17)
+               | ((0xFF & 6) << 12)
+               | (0x0FFF & self.error.value))
+
+        return hex(num)[2:].zfill(8).upper()
+
+
+class ProfilerParamValueErrorException(ProfilerException):
+    """The parameter value error in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerParamValueErrorException, self).__init__(
+            error=ProfilerErrors.PARAM_VALUE_ERROR,
+            message=ProfilerErrorMsg.PARAM_VALUE_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerPathErrorException(ProfilerException):
+    """The path error in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerPathErrorException, self).__init__(
+            error=ProfilerErrors.PATH_ERROR,
+            message=ProfilerErrorMsg.PATH_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerParamTypeErrorException(ProfilerException):
+    """The parameter type error in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerParamTypeErrorException, self).__init__(
+            error=ProfilerErrors.PARAM_TYPE_ERROR,
+            message=ProfilerErrorMsg.PARAM_TYPE_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerDirNotFoundException(ProfilerException):
+    """The dir not found exception in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerDirNotFoundException, self).__init__(
+            error=ProfilerErrors.DIR_NOT_FOUND_ERROR,
+            message=ProfilerErrorMsg.DIR_NOT_FOUND_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerFileNotFoundException(ProfilerException):
+    """The file not found exception in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerFileNotFoundException, self).__init__(
+            error=ProfilerErrors.FILE_NOT_FOUND_ERROR,
+            message=ProfilerErrorMsg.FILE_NOT_FOUND_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerIOException(ProfilerException):
+    """The IO exception in profiler module."""
+
+    def __init__(self):
+        super(ProfilerIOException, self).__init__(
+            error=ProfilerErrors.IO_ERROR,
+            message=ProfilerErrorMsg.IO_ERROR.value,
+            http_code=400
+        )
+
+
+class ProfilerDeviceIdMismatchException(ProfilerException):
+    """The device id mismatch exception in profiler module."""
+
+    def __init__(self):
+        super(ProfilerDeviceIdMismatchException, self).__init__(
+            error=ProfilerErrors.DEVICE_ID_MISMATCH_ERROR,
+            message=ProfilerErrorMsg.DEVICE_ID_MISMATCH_ERROR.value,
+            http_code=400
+        )
+
+
+class ProfilerRawFileException(ProfilerException):
+    """The raw file exception in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerRawFileException, self).__init__(
+            error=ProfilerErrors.RAW_FILE_ERROR,
+            message=ProfilerErrorMsg.RAW_FILE_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerColumnNotExistException(ProfilerException):
+    """The column does not exist exception in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerColumnNotExistException, self).__init__(
+            error=ProfilerErrors.COLUMN_NOT_EXIST_ERROR,
+            message=ProfilerErrorMsg.COLUMN_NOT_EXIST_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerAnalyserNotExistException(ProfilerException):
+    """The analyser in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerAnalyserNotExistException, self).__init__(
+            error=ProfilerErrors.ANALYSER_NOT_EXIST_ERROR,
+            message=ProfilerErrorMsg.ANALYSER_NOT_EXIST_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerDeviceIdException(ProfilerException):
+    """The parameter device_id error in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerDeviceIdException, self).__init__(
+            error=ProfilerErrors.DEVICE_ID_ERROR,
+            message=ProfilerErrorMsg.DEIVICE_ID_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerOpTypeException(ProfilerException):
+    """The parameter op_type error in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerOpTypeException, self).__init__(
+            error=ProfilerErrors.OP_TYPE_ERROR,
+            message=ProfilerErrorMsg.OP_TYPE_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerSortConditionException(ProfilerException):
+    """The parameter sort_condition error in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerSortConditionException, self).__init__(
+            error=ProfilerErrors.SORT_CONDITION_ERROR,
+            message=ProfilerErrorMsg.SORT_CONDITION_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerFilterConditionException(ProfilerException):
+    """The parameter filer_condition error in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerFilterConditionException, self).__init__(
+            error=ProfilerErrors.FILTER_CONDITION_ERROR,
+            message=ProfilerErrorMsg.FILTER_CONDITION_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerGroupConditionException(ProfilerException):
+    """The parameter group_condition error in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerGroupConditionException, self).__init__(
+            error=ProfilerErrors.GROUP_CONDITION_ERROR,
+            message=ProfilerErrorMsg.GROUP_CONDITION_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerColumnNotSupportSortException(ProfilerException):
+    """The column does not support to sort error in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerColumnNotSupportSortException, self).__init__(
+            error=ProfilerErrors.COLUMN_NOT_SUPPORT_SORT_ERROR,
+            message=ProfilerErrorMsg.COLUMN_NOT_SUPPORT_SORT_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class StepNumNotSupportedException(ProfilerException):
+    """The step number error in profiler module."""
+
+    def __init__(self, msg):
+        super(StepNumNotSupportedException, self).__init__(
+            error=ProfilerErrors.STEP_NUM_NOT_SUPPORTED_ERROR,
+            message=ProfilerErrorMsg.STEP_NUM_NOT_SUPPORTED_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class JobIdMismatchException(ProfilerException):
+    """The Job ID mismatch error in profiler module."""
+
+    def __init__(self):
+        super(JobIdMismatchException, self).__init__(
+            error=ProfilerErrors.JOB_ID_MISMATCH_ERROR,
+            message=ProfilerErrorMsg.JOB_ID_MISMATCH_ERROR.value,
+            http_code=400
+        )
+
+
+class ProfilerPipelineOpNotExistException(ProfilerException):
+    """The minddata pipeline operator does not exist error in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerPipelineOpNotExistException, self).__init__(
+            error=ProfilerErrors.PIPELINE_OP_NOT_EXIST_ERROR,
+            message=ProfilerErrorMsg.PIPELINE_OP_NOT_EXIST_ERROR.value.format(msg),
+            http_code=400
+        )

mindspore/profiler/common/process_pool.py

@@ -0,0 +1,41 @@
+# Copyright 2022-2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Analyze profiling data asynchronously by asynchronous process"""
+import atexit
+from typing import List
+from multiprocessing import Process
+
+from mindspore.profiler.common.singleton import Singleton
+
+
+@Singleton
+class MultiProcessPool:
+    """A ProcessPool to run task asynchronously"""
+
+    def __init__(self) -> None:
+        self.porcess_list: List[Process] = []
+        atexit.register(self.wait_all_job_finished)
+
+    def add_async_job(self, func):
+        """Add job and run in subprocess"""
+        process = Process(target=func)
+        process.start()
+        self.porcess_list.append(process)
+
+    def wait_all_job_finished(self):
+        """Wait all subprocess finished"""
+        for process in self.porcess_list:
+            process.join()
+        self.porcess_list = []

mindspore/profiler/common/registry.py

@@ -0,0 +1,47 @@
+# Copyright 2020-2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""CPU platform profiler."""
+from typing import Any, Dict
+from mindspore import log as logger
+
+
+class Register:
+    """
+    Common register class
+    """
+    def __init__(self, name) -> None:
+        self.module_dict = dict()
+        self.name = name
+
+    def register_module(self, name):
+        """ registers module decorator."""
+        def decorator(target):
+            model_name = target.__name__
+            if name in self.module_dict:
+                raise Exception(f'{model_name} already registered with name: {name}')
+
+            self.module_dict[name] = target
+            logger.info(f'registered module: {model_name} with name: {name}')
+            return target
+
+        return decorator
+
+    @property
+    def modules(self) -> Dict[str, Any]:
+        """ get all registered modules """
+        return self.module_dict
+
+
+PROFILERS = Register('PROFILERS')

mindspore/profiler/common/singleton.py

@@ -0,0 +1,28 @@
+# Copyright 2022-2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""A singleton implement in python"""
+
+
+class Singleton:
+    """A singleton implement"""
+
+    def __init__(self, cls):
+        self._cls = cls
+        self._instance = {}
+
+    def __call__(self):
+        if self._cls not in self._instance:
+            self._instance[self._cls] = self._cls()
+        return self._instance[self._cls]