mindspore 2.6.0__cp311-cp311-win_amd64.whl → 2.7.0rc1__cp311-cp311-win_amd64.whl

mindspore/profiler/parser/ascend_analysis/path_manager.py

@@ -1,313 +0,0 @@
-# Copyright 2023 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""Profiler path manager"""
-import os
-import re
-import shutil
-import warnings
-
-from mindspore import log as logger
-from mindspore.profiler.parser.ascend_analysis.constant import Constant
-
-
-__all__ = ['PathManager']
-
-
-class PathManager:
-    """
-    Path common operations manager
-    """
-    MAX_PATH_LENGTH = 4096
-    MAX_FILE_NAME_LENGTH = 255
-    DATA_FILE_AUTHORITY = 0o640
-    DATA_DIR_AUTHORITY = 0o750
-
-    @classmethod
-    def check_input_directory_path(cls, path: str):
-        """
-        Function Description:
-            check whether the path is valid, some businesses can accept a path that does not exist,
-            so the function do not verify whether the path exists
-        Parameter:
-            path: the path to check, whether the incoming path is absolute or relative depends on the business
-        Exception Description:
-            when invalid data throw exception
-        """
-        cls._input_path_common_check(path)
-
-        if os.path.isfile(path):
-            msg = "Invalid input path is a file path: {path}"
-            raise RuntimeError(msg)
-
-    @classmethod
-    def check_input_file_path(cls, path: str):
-        """
-        Function Description:
-            check whether the file path is valid, some businesses can accept a path that does not exist,
-            so the function do not verify whether the path exists
-        Parameter:
-            path: the file path to check, whether the incoming path is absolute or relative depends on the business
-        Exception Description:
-            when invalid data throw exception
-        """
-        cls._input_path_common_check(path)
-
-        if os.path.isdir(path):
-            msg = "Invalid input path is a directory path: {path}"
-            raise RuntimeError(msg)
-
-    @classmethod
-    def copy_file(cls, src_path: str, dst_path: str):
-        """
-        Function Description:
-            copy file safety
-        Parameter:
-            src_path: file source path
-            dst_path: file destination path
-        Exception Description:
-            when src_path is link throw exception
-        """
-        if not os.path.exists(src_path):
-            logger.warning("The source file does not exist: %s", src_path)
-            return
-
-        cls.check_input_file_path(src_path)
-        dst_dir = os.path.dirname(dst_path)
-        cls.check_directory_path_writeable(dst_dir)
-
-        try:
-            shutil.copy2(src_path, dst_path)
-        except Exception as err:
-            msg = f"Failed to copy path: {src_path}"
-            raise RuntimeError(msg) from err
-
-    @classmethod
-    def check_path_owner_consistent(cls, path: str):
-        """
-        Function Description:
-            check whether the path belong to process owner
-        Parameter:
-            path: the path to check
-        Exception Description:
-            when invalid path, prompt the user
-        """
-
-        if not os.path.exists(path):
-            msg = f"The path does not exist: {path}"
-            raise RuntimeError(msg)
-        if os.stat(path).st_uid != os.getuid():
-            warnings.warn(f"Warning: The {path} owner does not match the current user.")
-
-    @classmethod
-    def check_directory_path_writeable(cls, path):
-        """
-        Function Description:
-            check whether the path is writable
-        Parameter:
-            path: the path to check
-        Exception Description:
-            when invalid data throw exception
-        """
-        cls.check_path_owner_consistent(path)
-        if os.path.islink(path):
-            msg = f"Invalid path is a soft chain: {path}"
-            raise RuntimeError(msg)
-        if not os.access(path, os.W_OK):
-            msg = f"The path permission check failed: {path}"
-            raise RuntimeError(msg)
-
-    @classmethod
-    def check_directory_path_readable(cls, path):
-        """
-        Function Description:
-            check whether the path is writable
-        Parameter:
-            path: the path to check
-        Exception Description:
-            when invalid data throw exception
-        """
-        cls.check_path_owner_consistent(path)
-        if os.path.islink(path):
-            msg = f"Invalid path is a soft chain: {path}"
-            raise RuntimeError(msg)
-        if not os.access(path, os.R_OK):
-            msg = f"The path permission check failed: {path}"
-            raise RuntimeError(msg)
-
-    @classmethod
-    def remove_path_safety(cls, path: str):
-        """
-        Function Description:
-            remove path safety
-        Parameter:
-            path: the path to remove
-        Exception Description:
-            when invalid data throw exception
-        """
-        msg = f"Failed to remove path: {path}"
-        if os.path.islink(path):
-            raise RuntimeError(msg)
-        if not os.path.exists(path):
-            return
-        try:
-            shutil.rmtree(path)
-        except FileNotFoundError:
-            return
-        except Exception as err:
-            raise RuntimeError(msg) from err
-
-    @classmethod
-    def remove_file_safety(cls, file: str):
-        """
-        Function Description:
-            remove file safety
-        Parameter:
-            path: the file to remove
-        Exception Description:
-            when invalid data throw exception
-        """
-        msg = f"Failed to remove file: {file}"
-        if os.path.islink(file):
-            raise RuntimeError(msg)
-        if not os.path.exists(file):
-            return
-        try:
-            os.remove(file)
-        except FileExistsError:
-            return
-        except Exception as err:
-            raise RuntimeError(msg) from err
-
-    @classmethod
-    def make_dir_safety(cls, path: str):
-        """
-        Function Description:
-            make directory safety
-        Parameter:
-            path: the directory to remove
-        Exception Description:
-            when invalid data throw exception
-        """
-        msg = f"Failed to make directory: {path}"
-        if os.path.islink(path):
-            raise RuntimeError(msg)
-        if os.path.exists(path):
-            return
-        try:
-            os.makedirs(path, mode=cls.DATA_DIR_AUTHORITY, exist_ok=True)
-        except Exception as err:
-            raise RuntimeError(msg) from err
-
-    @classmethod
-    def create_file_safety(cls, path: str):
-        """
-        Function Description:
-            create file safety
-        Parameter:
-            path: the file to remove
-        Exception Description:
-            when invalid data throw exception
-        """
-        msg = f"Failed to create file: {path}"
-        if os.path.islink(path):
-            raise RuntimeError(msg)
-        if os.path.exists(path):
-            return
-        try:
-            os.close(os.open(path, os.O_WRONLY | os.O_CREAT, cls.DATA_FILE_AUTHORITY))
-        except Exception as err:
-            raise RuntimeError(msg) from err
-
-    @classmethod
-    def _input_path_common_check(cls, path: str):
-        """
-        Function Description:
-            input path check common function
-        Parameter:
-            path: the file path to check
-        Exception Description:
-            when invalid data throw exception
-        """
-        if len(path) > cls.MAX_PATH_LENGTH:
-            raise RuntimeError("Length of input path exceeds the limit.")
-
-        if os.path.islink(path):
-            msg = f"Invalid input path is a soft chain: {path}"
-            raise RuntimeError(msg)
-
-        pattern = r'(\.|/|_|-|\s|[~0-9a-zA-Z])+'
-        if not re.fullmatch(pattern, path):
-            msg = f"Invalid input path: {path}"
-            raise RuntimeError(msg)
-
-        path_split_list = path.split("/")
-        for name in path_split_list:
-            if len(name) > cls.MAX_FILE_NAME_LENGTH:
-                raise RuntimeError("Length of input path exceeds the limit.")
-
-    @classmethod
-    def get_profiler_parent_path_list(cls, input_path: str):
-        """
-        Function Description:
-            get valid profiler parent path list from input_path
-        Parameter:
-            input_path: The directory path from which to extract profiler parent paths.
-        Return:
-            A list containing the input path or its subdirectories that are valid profiler parents.
-        """
-        profiler_path = os.path.join(input_path, Constant.PROFILER_DIR)
-        if os.path.isdir(profiler_path) and (cls.get_fwk_path(profiler_path) or cls.get_cann_path(profiler_path)):
-            return [input_path]
-        sub_dirs = os.listdir(os.path.realpath(input_path))
-        profiler_parent_path_list = []
-        for sub_dir in sub_dirs:
-            sub_path = os.path.join(input_path, sub_dir, Constant.PROFILER_DIR)
-            if not os.path.isdir(sub_path):
-                continue
-            if cls.get_fwk_path(sub_path) or cls.get_cann_path(sub_path):
-                profiler_parent_path_list.append(os.path.join(input_path, sub_dir))
-        return profiler_parent_path_list
-
-    @classmethod
-    def get_fwk_path(cls, input_path: str):
-        """
-        Function Description:
-            get valid framework path from input_path
-        Parameter:
-            input_path: the directory path to check whether exist valid FRAMEWORK path
-        Return:
-            The path to the FRAMEWORK directory if found, otherwise an empty string.
-        """
-        fwk_path = os.path.join(input_path, Constant.FRAMEWORK_DIR)
-        if os.path.isdir(fwk_path):
-            return fwk_path
-        return ""
-
-    @classmethod
-    def get_cann_path(cls, input_path: str):
-        """
-        Function Description:
-            get valid PROF_XXX path from input_path
-        Parameter:
-            input_path: the directory path to check valid PROF_XXX path
-        Return:
-            The path to the PROF_XXX directory if it matches the pattern and exists, otherwise an empty string.
-        """
-        sub_dirs = os.listdir(os.path.realpath(input_path))
-        for sub_dir in sub_dirs:
-            sub_path = os.path.join(input_path, sub_dir)
-            if os.path.isdir(sub_path) and re.match(r"^PROF_\d+_\d+_[0-9a-zA-Z]+", sub_dir):
-                return sub_path
-        return ""

mindspore/profiler/parser/ascend_analysis/profiler_info_parser.py

@@ -1,123 +0,0 @@
-# Copyright 2023 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""Profiler host information parser"""
-import os
-import json
-from decimal import Decimal
-from subprocess import CalledProcessError, TimeoutExpired
-from subprocess import Popen, PIPE
-from configparser import ConfigParser
-
-from mindspore import log as logger
-from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
-from mindspore.profiler.parser.ascend_analysis.constant import Constant
-
-
-class ProfilerInfoParser:
-    """Parse files that record information, such as profiler_info.json"""
-
-    _freq = 100.0
-    _msprof_cmd = "msprof"
-    _time_out = 1
-    # profiler information related files
-    _source_prof_path = None
-    _loaded_frequency = False
-    _rank_id = 0
-    _clock_monotonic_raw = 0
-    _cntvct = 0
-    _collectionTimeBegin = 0
-    _clockMonotonicRaw = 0
-    _get_localtime_diff = 0
-
-    @classmethod
-    def init_source_path(cls, source_path: str):
-        """initialize the path of PROF_* directory."""
-        source_path = validate_and_normalize_path(source_path)
-        prof_path = os.path.dirname(source_path)
-        dir_name = os.path.basename(source_path)
-        if not dir_name.startswith("device") or not os.path.exists(source_path):
-            raise RuntimeError("Input source path is invalid!")
-        cls._source_prof_path = prof_path
-
-    @classmethod
-    def init_rank_id(cls, rank_id: int):
-        """initialize the rank id."""
-        cls._rank_id = rank_id
-
-    @classmethod
-    def get_local_time(cls, syscnt: int) -> Decimal:
-        """Convert syscnt to local time."""
-        if not cls._loaded_frequency:
-            outs, _ = cls.__run_cmd(['which', cls._msprof_cmd])
-            if not outs:
-                raise FileNotFoundError("Failed to find msprof command!")
-            msprof_path = os.path.realpath(outs.strip())
-            sup_path = msprof_path.split('tools')[0]
-            script_path = os.path.join(sup_path, 'tools/profiler/profiler_tool/analysis/interface/get_msprof_info.py')
-            py_cmd = ['python', script_path, '-dir', os.path.join(cls._source_prof_path, 'host')]
-            outs, _ = cls.__run_cmd(py_cmd)
-            if not outs:
-                raise RuntimeError("Failed to get msprof information!")
-            result = json.loads(outs)
-            cpu_info = result.get('data', {}).get('host_info', {}).get('cpu_info', [{}])[0]
-            try:
-                cls._freq = float(cpu_info.get("Frequency", cls._freq))
-            except ValueError:
-                pass
-            cls._get_msprof_timestamp(cls._source_prof_path)
-            cls._loaded_frequency = True
-        start_ns = cls.__get_timestamp(syscnt)
-        return Decimal(start_ns).quantize(Decimal('0.000')) * Decimal(Constant.NS_TO_US).quantize(Decimal('0.000'))
-
-    @classmethod
-    def _get_msprof_timestamp(cls, source_path):
-        """get msprof timestamp info"""
-        start_log = ConfigParser()
-        start_log.read(os.path.join(source_path, "host", "host_start.log"))
-        cls._clock_monotonic_raw = int(start_log.get("Host", "clock_monotonic_raw"))
-        cls._cntvct = int(start_log.get("Host", "cntvct"))
-
-        with open(os.path.join(source_path, "host", "start_info"), "r") as f:
-            info_dict = json.load(f)
-            cls._collectionTimeBegin = int(info_dict.get("collectionTimeBegin", 0))  # us
-            cls._clockMonotonicRaw = int(info_dict.get("clockMonotonicRaw", 0))
-            us_to_ns = 1000
-            cls._get_localtime_diff = cls._clock_monotonic_raw + (cls._collectionTimeBegin * us_to_ns -
-                                                                  cls._clockMonotonicRaw)
-
-    @classmethod
-    def __run_cmd(cls, cmd):
-        """run shell command"""
-        try:
-            with Popen(cmd, stdout=PIPE, stderr=PIPE, text=True) as proc:
-                outs, errs = proc.communicate(timeout=cls._time_out)
-        except (FileNotFoundError, PermissionError, CalledProcessError) as exc:
-            raise RuntimeError(exc) from exc
-        except TimeoutExpired as err:
-            proc.kill()
-            msg = "The possible cause is that too much data is collected " \
-                  "and the export time is too long."
-            logger.error(msg)
-            raise TimeoutError(msg) from err
-        logger.info(outs)
-        return outs, errs
-
-    @classmethod
-    def __get_timestamp(cls, syscnt: int, time_fmt: int = 1000):
-        """Convert syscnt to time stamp."""
-        ratio = time_fmt / cls._freq
-        # The unit of timestamp is ns
-        timestamp = round((syscnt - cls._cntvct) * ratio) + cls._get_localtime_diff
-        return timestamp

mindspore/profiler/parser/ascend_analysis/tlv_decoder.py

@@ -1,86 +0,0 @@
-# Copyright 2023 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""TLV format file decoder"""
-
-import struct
-from typing import List, Dict, Union
-
-from mindspore import log as logger
-
-from mindspore.profiler.parser.ascend_analysis.constant import Constant
-from mindspore.profiler.parser.ascend_analysis.function_event import MindSporeOpEvent
-
-
-class TLVDecoder:
-    """
-    The class of TLV format data decoder.
-
-    Args:
-        all_bytes(bytes): all the bytes data of tlv format binary file.
-        event_class(MindSporeOpEvent): class of events to decode.
-        fix_data_struct_size(int): a constant value.
-
-    Return:
-        List[MindSporeOpEvent]: event list of input event class.
-    """
-    _type_len = 2
-    _length_len = 4
-
-    @classmethod
-    def decode(cls, all_bytes: bytes, event_class: MindSporeOpEvent, fix_data_struct_size: int):
-        """Decode all the data."""
-        result_data = []
-        records = cls.tlv_list_decode(all_bytes)
-        for record in records:
-            if fix_data_struct_size > len(record):
-                logger.warning("The collected data has been lost")
-                continue
-            fix_data_bytes = record[0: fix_data_struct_size]
-            tlv_fields = cls.tlv_list_decode(record[fix_data_struct_size:], is_field=True)
-            tlv_fields[Constant.FIX_SIZE_BYTES] = fix_data_bytes
-            result_data.append(event_class(tlv_fields))
-        return result_data
-
-    @classmethod
-    def tlv_list_decode(cls, tlv_bytes: bytes, is_field: bool = False) -> Union[Dict, List]:
-        """Decode TLV format data."""
-        result_data = {} if is_field else []
-        index = 0
-        all_bytes_len = len(tlv_bytes)
-        while index < all_bytes_len:
-            if index + cls._type_len > all_bytes_len:
-                logger.warning("The collected data has been lost")
-                break
-            type_id = struct.unpack("<H", tlv_bytes[index: index + cls._type_len])[0]
-            index += cls._type_len
-            if index + cls._length_len > all_bytes_len:
-                logger.warning("The collected data has been lost")
-                break
-            value_len = struct.unpack("<I", tlv_bytes[index: index + cls._length_len])[0]
-            index += cls._length_len
-            if index + value_len > all_bytes_len:
-                logger.warning("The collected data has been lost")
-                break
-            value = tlv_bytes[index: index + value_len]
-            index += value_len
-            if is_field:
-                try:
-                    result_data[type_id] = bytes.decode(value)
-                except UnicodeDecodeError:
-                    logger.warning(f"The collected data can't decode by bytes.decode: {value}")
-                    result_data[type_id] = 'N/A'
-            else:
-                result_data.append(value)
-        return result_data

mindspore/profiler/parser/ascend_analysis/trace_event_manager.py

@@ -1,75 +0,0 @@
-# Copyright 2023 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""Json trace event manager"""
-
-from typing import Dict, List
-
-from mindspore.profiler.parser.ascend_analysis.constant import Constant
-from mindspore.profiler.parser.ascend_analysis.function_event import BaseEvent
-
-
-class TraceEventManager:
-    """Chrome trace format json object manager."""
-
-    @classmethod
-    def create_x_event(cls, event: BaseEvent, cat: str) -> Dict:
-        """Create a X event."""
-        x_event = {
-            "ph": "X", "name": event.name, "pid": event.pid, "tid": event.tid,
-            "ts": str(event.ts), "dur": str(event.dur), "cat": cat, "args": event.args
-        }
-        return x_event
-
-    @classmethod
-    def create_i_event(cls, event: BaseEvent) -> Dict:
-        """Create a i event."""
-        event = {"name": event.name, "ph": "i", "ts": str(event.ts),
-                 "pid": event.pid, "tid": event.tid, "args": event.args}
-        return event
-
-    @classmethod
-    def create_m_event(cls, pid: int, tid_list: set, process_index: int) -> List:
-        """Create some metadata event."""
-        # framework sidee trace information display format: MindSpore(pid pid_value): CPU
-        event_list = [
-            # process information
-            {"ph": "M", "name": Constant.PROCESS_NAME, "pid": pid, "tid": 0, "args": {"name": "MindSpore"}},
-            {"ph": "M", "name": Constant.PROCESS_LABEL, "pid": pid, "tid": 0, "args": {"labels": "CPU"}},
-            {"ph": "M", "name": Constant.PROCESS_SORT, "pid": pid, "tid": 0, "args": {"sort_index": process_index}},
-        ]
-        for tid in tid_list:
-            sort_index = tid
-            event_list.extend(
-                [{"ph": "M", "name": Constant.THREAD_NAME, "pid": pid, "tid": tid, "args": {"name": f"Thread {tid}"}},
-                 {"ph": "M", "name": Constant.THREAD_SORT, "pid": pid, "tid": tid, "args": {"sort_index": sort_index}}])
-        return event_list
-
-    @classmethod
-    def create_mindspore_to_npu_flow(cls, start_event: BaseEvent, end_event: BaseEvent) -> List:
-        """Create flow events link mindspore operator and npu kernel."""
-        flow_id = str(end_event.ts)
-        return [{"ph": "s", "bp": "e", "name": "mindspore_to_npu", "id": flow_id, "pid": start_event.pid,
-                 "tid": start_event.tid, "ts": str(start_event.ts), "cat": "async_npu"},
-                {"ph": "f", "bp": "e", "name": "mindspore_to_npu", "id": flow_id, "pid": end_event.pid,
-                 "tid": end_event.tid, "ts": str(end_event.ts), "cat": "async_npu"}]
-
-    @classmethod
-    def create_mindspore_to_self_flow(cls, start_event: BaseEvent, end_event: BaseEvent) -> List:
-        """Create flow events link mindspore operator and npu kernel."""
-        flow_id = start_event.flow_id
-        return [{"ph": "s", "bp": "e", "name": "mindspore_to_self", "id": flow_id, "pid": start_event.pid,
-                 "tid": start_event.tid, "ts": str(start_event.ts), "cat": "async_mindspore"},
-                {"ph": "f", "bp": "e", "name": "mindspore_to_self", "id": flow_id, "pid": end_event.pid,
-                 "tid": end_event.tid, "ts": str(end_event.ts), "cat": "async_mindspore"}]