mindspore 2.1.0__cp39-none-any.whl → 2.2.10__cp39-none-any.whl

mindspore/_akg/akg/utils/ascend_profilier/file_manager.py

@@ -0,0 +1,56 @@
+import csv
+import json
+import os.path
+import shutil
+from warnings import warn
+
+MAX_FILE_SIZE = 1024 * 1024 * 1024 * 10
+MAX_CSV_SIZE = 1024 * 1024 * 1024 * 5
+
+class FileManager:
+    @classmethod
+    def file_read_all(cls, file_path: str, mode: str = "r") -> any:
+        if not os.path.isfile(file_path):
+            return ''
+        file_size = os.path.getsize(file_path)
+        if file_size <= 0:
+            return ''
+        if file_size > MAX_FILE_SIZE:
+            warn(f"The file size exceeds the preset value {MAX_FILE_SIZE / 1024 / 1024}MB, "
+                 f"please check the file: {file_path}")
+            return ''
+        try:
+            with open(file_path, mode) as file:
+                return file.read()
+        except Exception:
+            raise RuntimeError(f"Can't read file: {file_path}")
+
+    @classmethod
+    def read_csv_file(cls, file_path: str, class_bean: any) -> list:
+        if not os.path.isfile(file_path):
+            return []
+        file_size = os.path.getsize(file_path)
+        if file_size <= 0:
+            return []
+        if file_size > MAX_CSV_SIZE:
+            warn(f"The file size exceeds the preset value {MAX_CSV_SIZE / 1024 / 1024}MB, "
+                 f"please check the file: {file_path}")
+            return []
+        result_data = []
+        try:
+            with open(file_path, newline="") as csv_file:
+                reader = csv.DictReader(csv_file)
+                for row in reader:
+                    result_data.append(class_bean(row))
+        except Exception:
+            raise RuntimeError(f"Failed to read the file: {file_path}")
+        return result_data
+
+    @classmethod
+    def remove_file_safety(cls, path: str):
+        if os.path.exists(path):
+            try:
+                shutil.rmtree(path)
+            except Exception:
+                print(f"[WARNING] [{os.getpid()}] profiler.py: Can't remove the directory: {path}")
+

mindspore/_akg/akg/utils/ascend_profilier/op_summary_bean.py

@@ -0,0 +1,23 @@
+from .op_summary_headers import OpSummaryHeaders
+
+class OpSummaryBean:
+    headers = []
+
+    def __init__(self, data: list):
+        self._data = data
+
+    @property
+    def row(self) -> list:
+        row = []
+        read_headers = OpSummaryBean.headers if OpSummaryBean.headers else self._data.keys()
+        for field_name in read_headers:
+            row.append(self._data.get(field_name, ""))
+        return row
+
+    @property
+    def ts(self) -> float:
+        return float(self._data.get(OpSummaryHeaders.TASK_START_TIME, 0))
+
+    @property
+    def all_headers(self) -> list:
+        return list(self._data.keys())

mindspore/_akg/akg/utils/ascend_profilier/op_summary_headers.py

@@ -0,0 +1,8 @@
+class OpSummaryHeaders(object):
+    # op_summary
+    TASK_START_TIME = "Task Start Time(us)"
+    AIC_TOTAL_CYCLES = "aic_total_cycles"
+    AIV_TOTAL_CYCLES = "aiv_total_cycles"
+    TASK_DURATION = "Task Duration(us)"
+    OP_SUMMARY_SHOW_HEADERS = ["Op Name", "OP Type", "Task Type", TASK_START_TIME, TASK_DURATION,
+                               "Task Wait Time(us)", "Block Dim" ,AIC_TOTAL_CYCLES, AIV_TOTAL_CYCLES]

mindspore/_akg/akg/utils/ascend_profilier/op_summary_parser.py

@@ -0,0 +1,42 @@
+from .op_summary_headers import OpSummaryHeaders
+from .file_manager import FileManager
+from .op_summary_bean import OpSummaryBean
+from .cann_file_parser import CANNFileParser, CANNDataEnum
+
+
+class OpSummaryParser():
+
+    def __init__(self, profiler_path: str):
+        self._profiler_path = profiler_path
+
+    @classmethod
+    def _project_map_for_headers(cls, input_headers: list):
+        project_map_dict = {OpSummaryHeaders.OP_SUMMARY_SHOW_HEADERS[i]: OpSummaryHeaders.OP_SUMMARY_KERNEL_BASE_HEADERS[i] for i in
+                            range(len(OpSummaryHeaders.OP_SUMMARY_SHOW_HEADERS))}
+        output_headers = []
+        for header in input_headers:
+            if header in project_map_dict:
+                output_headers.append(project_map_dict.get(header))
+            else:
+                output_headers.append(header)
+        return output_headers
+
+    def generate_op_summary_data(self) -> dict:
+        op_summary_file_set = CANNFileParser(self._profiler_path).get_file_list_by_type(CANNDataEnum.OP_SUMMARY)
+        summary_data = []
+        for file_path in op_summary_file_set:
+            all_data = FileManager.read_csv_file(file_path, OpSummaryBean)
+            if all_data:
+                OpSummaryBean.headers = OpSummaryHeaders.OP_SUMMARY_SHOW_HEADERS
+                output_headers = OpSummaryBean.headers
+                summary_data.extend([data.row for data in all_data])
+            else:
+                raise RuntimeError("parse op summary csv failed.")
+
+        return self.create_dict(summary_data[0],output_headers)
+
+    def create_dict(self,summary_data,headers):
+        summary_dict={}
+        for i in range(len(summary_data)):
+            summary_dict[headers[i]] = summary_data[i]
+        return summary_dict

mindspore/_akg/akg/utils/ascend_profilier/path_manager.py

@@ -0,0 +1,65 @@
+import os
+import re
+
+class PathManager:
+
+    @classmethod
+    def get_cann_path(cls, profiler_path: str) -> str:
+        sub_dirs = os.listdir(os.path.realpath(profiler_path))
+        for sub_dir in sub_dirs:
+            sub_path = os.path.join(profiler_path, sub_dir)
+            if os.path.isdir(sub_path) and re.match(r"^PROF_\d+_\d+_[a-zA-Z]+", sub_dir):
+                return sub_path
+        return ""
+
+    @classmethod
+    def get_device_path(cls, cann_path: str) -> str:
+        sub_dirs = os.listdir(os.path.realpath(cann_path))
+        for sub_dir in sub_dirs:
+            sub_path = os.path.join(cann_path, sub_dir)
+            if os.path.isdir(sub_path) and re.match(r"^device_\d", sub_dir):
+                return sub_path
+        return ""
+
+    @classmethod
+    def get_start_info_path(cls, cann_path: str) -> str:
+        start_info_path = os.path.join(cann_path, "host", "start_info")
+        if os.path.exists(start_info_path):
+            return start_info_path
+        device_path = cls.get_device_path(cann_path)
+        if not device_path:
+            return ""
+        device_path_split = os.path.basename(device_path).split("_")
+        if len(device_path_split) != 2:
+            return ""
+        start_info_file = f"start_info.{device_path_split[1]}"
+        start_info_path = os.path.join(device_path, start_info_file)
+        if os.path.exists(start_info_path):
+            return start_info_path
+        return ""
+
+    @classmethod
+    def get_device_all_file_list_by_type(cls, profiler_path: str, summary_or_timeline: str) -> list:
+        file_list = []
+        _path = os.path.join(cls.get_device_path(profiler_path), summary_or_timeline)
+        if not os.path.isdir(_path):
+            return file_list
+        sub_files = os.listdir(os.path.realpath(_path))
+        if not sub_files:
+            return file_list
+        for sub_file in sub_files:
+            file_list.append(os.path.join(_path, sub_file))
+        return file_list
+
+    @classmethod
+    def get_analyze_all_file(cls, profiler_path: str, analyze: str) -> list:
+        file_list = []
+        _path = os.path.join(profiler_path, analyze)
+        if not os.path.isdir(_path):
+            return file_list
+        sub_files = os.listdir(os.path.realpath(_path))
+        if not sub_files:
+            return file_list
+        for sub_file in sub_files:
+            file_list.append(os.path.join(_path, sub_file))
+        return file_list

mindspore/_akg/akg/utils/composite_op_helper.py

@@ -95,7 +95,7 @@ def compare_tensor(acu_output, exp_output, rtol=1.e-5, atol=1.e-8, equal_nan=Fal
 
 def get_rtol_atol(op_name, dtype, rtol=5e-03, atol=5e-03):
     run_mode = os.environ.get('RUNTIME_MODE')
-    if run_mode in ("rpc_cloud", "air_cloud"):
+    if run_mode in ("air_cloud",):
         if dtype == "float16":
             rtol = atol = 1e-03
         else:
@@ -272,11 +272,7 @@ def _gen_uniq_file_name(op_name):
 
 def _collect_inplace_assign_infos(op, infos, sum_out):
     """Collect inplace assign infos."""
-    if op["name"] != "InplaceAssign":
-        return
-    fake_output = get_attr(op["attr"], "fake_output")
-    if fake_output:
-        infos["fake_output_tensors"].append(op["output_desc"][0]["tensor_name"])
+    infos["fake_output_tensors"].append(op["output_desc"][0]["tensor_name"])
     input0, input1 = op["input_desc"][0][0], op["input_desc"][1][0]
     if input1["tensor_name"] in sum_out:
         infos["clean_input"].append(input0["tensor_name"])
@@ -302,9 +298,8 @@ def _collect_infos(desc, infos):
                                                                         indices_shape=input1["shape"],
                                                                         indices_dtype=input1["data_type"],
                                                                         attrs=get_attr(op["attr"], "num_segments"))
-        elif op["name"] in ["InplaceAssign", "Assign"]:
-            if op["name"] == "InplaceAssign":
-                _collect_inplace_assign_infos(op, infos, sum_out)
+        elif op["name"] == "Assign":
+            _collect_inplace_assign_infos(op, infos, sum_out)
             infos["inplace_assign_write"].append(op["input_desc"][0][0]["tensor_name"])
         elif op["name"] in ["TensorScatterAdd", "Gather", "GatherNd"]:
             input0, input1 = op["input_desc"][0][0], op["input_desc"][1][0]
@@ -543,7 +538,11 @@ def _update_workspace_data(kernel_name, input_for_mod, output_indexes):
 def gen_json_data(op_desc, with_compute=True, input_for_mod=None):
     """Generating test data for composite json"""
     desc = json.loads(op_desc)
-
+    from akg.ms.info_version_adapt import InfoVersionAdapt
+    info_adapter = InfoVersionAdapt(desc)
+    ret = info_adapter.run()
+    if not ret:
+        raise RuntimeError(info_adapter.msg)
     output_indexes = []
     expect = []
     infos = {"gen_data": False,