mindstudio-probe 1.0.1__py3-none-any.whl → 1.0.4__py3-none-any.whl

msprobe/pytorch/parse_tool/lib/visualization.py

@@ -1,90 +1,85 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-import json
-import numpy as np
-
-from msprobe.pytorch.parse_tool.lib.config import Const
-from msprobe.pytorch.parse_tool.lib.utils import Util
-from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException
-from msprobe.core.common.file_check import FileOpen
-
-
-class Visualization:
-    def __init__(self):
-        self.util = Util()
-
-    def print_npy_summary(self, target_file):
-        try:
-            np_data = np.load(target_file, allow_pickle=True)
-        except UnicodeError as e:
-            self.util.log.error("%s %s" % ("UnicodeError", str(e)))
-            self.util.log.warning("Please check the npy file")
-            raise ParseException(ParseException.PARSE_UNICODE_ERROR) from e
-        table = self.util.create_table('', ['Index', 'Data'])
-        flatten_data = np_data.flatten()
-        tablesize = 8
-        for i in range(min(16, int(np.ceil(flatten_data.size / tablesize)))):
-            last_idx = min(flatten_data.size, i * tablesize + tablesize)
-            table.add_row(str(i * tablesize), ' '.join(flatten_data[i * tablesize: last_idx].astype('str').tolist()))
-        summary = ['[yellow]%s[/yellow]' % self.util.gen_npy_info_txt(np_data), 'Path: %s' % target_file,
-                   "TextFile: %s.txt" % target_file]
-        self.util.print_panel(self.util.create_columns([table, "\n".join(summary)]), target_file)
-        self.util.save_npy_to_txt(np_data, target_file + ".txt")
-
-    def print_npy_data(self, file_name):
-        file_name = self.util.path_strip(file_name)
-        self.util.check_path_valid(file_name)
-        self.util.check_path_format(file_name, Const.NPY_SUFFIX)
-        return self.print_npy_summary(file_name)
-
-    def parse_pkl(self, path, api_name):
-        path = self.util.path_strip(path)
-        self.util.check_path_valid(path)
-        self.util.check_path_format(path, Const.PKL_SUFFIX)
-        self.util.check_str_param(api_name)
-        with FileOpen(path, "r") as pkl_handle:
-            title_printed = False
-            while True:
-                pkl_line = pkl_handle.readline()
-                if pkl_line == '\n':
-                    continue
-                if len(pkl_line) == 0:
-                    break
-                try:
-                    msg = json.loads(pkl_line)
-                except json.JSONDecodeError as e:
-                    self.util.log.error("%s %s in line %s" % ("JSONDecodeError", str(e), pkl_line))
-                    self.util.log.warning("Please check the pkl file")
-                    raise ParseException(ParseException.PARSE_JSONDECODE_ERROR) from e
-                info_prefix = msg[0]
-                if not info_prefix.startswith(api_name):
-                    continue
-                if info_prefix.find("stack_info") != -1 and len(msg) == 2:
-                    self.util.log.info("\nTrace back({}):".format(msg[0]))
-                    if msg[1] and len(msg[1]) > 4:
-                        for item in reversed(msg[1]):
-                            self.util.log.info("  File \"{}\", line {}, in {}".format(item[0], item[1], item[2]))
-                            self.util.log.info("    {}".format(item[3]))
-                        continue
-                if len(msg) > 5 and len(msg[5]) >=  3:
-                    summery_info = "  [{}][dtype: {}][shape: {}][max: {}][min: {}][mean: {}]" \
-                        .format(msg[0], msg[3], msg[4], msg[5][0], msg[5][1], msg[5][2])
-                    if not title_printed:
-                        self.util.log.info("\nStatistic Info:")
-                        title_printed = True
-                    self.util.log.info(summery_info)
-            pkl_handle.close()
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+import json
+import numpy as np
+
+from msprobe.pytorch.parse_tool.lib.config import Const
+from msprobe.pytorch.parse_tool.lib.utils import Util
+from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException
+from msprobe.core.common.file_utils import FileOpen, load_npy, save_npy_to_txt
+
+
+class Visualization:
+    def __init__(self):
+        self.util = Util()
+
+    def print_npy_summary(self, target_file):
+        np_data = load_npy(target_file, enable_pickle=True)
+        table = self.util.create_table('', ['Index', 'Data'])
+        flatten_data = np_data.flatten()
+        tablesize = 8
+        for i in range(min(16, int(np.ceil(flatten_data.size / tablesize)))):
+            last_idx = min(flatten_data.size, i * tablesize + tablesize)
+            table.add_row(str(i * tablesize), ' '.join(flatten_data[i * tablesize: last_idx].astype('str').tolist()))
+        summary = ['[yellow]%s[/yellow]' % self.util.gen_npy_info_txt(np_data), 'Path: %s' % target_file,
+                   "TextFile: %s.txt" % target_file]
+        self.util.print_panel(self.util.create_columns([table, "\n".join(summary)]), target_file)
+        save_npy_to_txt(np_data, target_file + ".txt")
+
+    def print_npy_data(self, file_name):
+        file_name = self.util.path_strip(file_name)
+        self.util.check_path_valid(file_name)
+        self.util.check_file_path_format(file_name, Const.NPY_SUFFIX)
+        return self.print_npy_summary(file_name)
+
+    def parse_pkl(self, path, api_name):
+        path = self.util.path_strip(path)
+        self.util.check_path_valid(path)
+        self.util.check_file_path_format(path, Const.PKL_SUFFIX)
+        self.util.check_str_param(api_name)
+        with FileOpen(path, "r") as pkl_handle:
+            title_printed = False
+            while True:
+                pkl_line = pkl_handle.readline()
+                if pkl_line == '\n':
+                    continue
+                if len(pkl_line) == 0:
+                    break
+                try:
+                    msg = json.loads(pkl_line)
+                except json.JSONDecodeError as e:
+                    self.util.log.error("%s %s in line %s" % ("JSONDecodeError", str(e), pkl_line))
+                    self.util.log.warning("Please check the pkl file")
+                    raise ParseException(ParseException.PARSE_JSONDECODE_ERROR) from e
+                info_prefix = msg[0]
+                if not info_prefix.startswith(api_name):
+                    continue
+                if info_prefix.find("stack_info") != -1 and len(msg) == 2:
+                    self.util.log.info("\nTrace back({}):".format(msg[0]))
+                    if msg[1] and len(msg[1]) > 4:
+                        for item in reversed(msg[1]):
+                            self.util.log.info("  File \"{}\", line {}, in {}".format(item[0], item[1], item[2]))
+                            self.util.log.info("    {}".format(item[3]))
+                        continue
+                if len(msg) > 5 and len(msg[5]) >=  3:
+                    summery_info = "  [{}][dtype: {}][shape: {}][max: {}][min: {}][mean: {}]" \
+                        .format(msg[0], msg[3], msg[4], msg[5][0], msg[5][1], msg[5][2])
+                    if not title_printed:
+                        self.util.log.info("\nStatistic Info:")
+                        title_printed = True
+                    self.util.log.info(summery_info)
+            pkl_handle.close()

msprobe/pytorch/pt_config.py

@@ -1,93 +1,188 @@
-import json
-import os
-
-from msprobe.core.common_config import CommonConfig, BaseConfig
-from msprobe.core.common.file_check import FileOpen
-from msprobe.core.common.const import Const
-
-
-class TensorConfig(BaseConfig):
-    def __init__(self, json_config):
-        super().__init__(json_config)
-        self.check_config()
-        self._check_file_format()
-
-    def _check_file_format(self):
-        if self.file_format is not None and self.file_format not in ["npy", "bin"]:
-            raise Exception("file_format is invalid")
-
-
-class StatisticsConfig(BaseConfig):
-    def __init__(self, json_config):
-        super().__init__(json_config)
-        self.check_config()
-        self._check_summary_mode()
-
-    def _check_summary_mode(self):
-        if self.summary_mode and self.summary_mode not in ["statistics", "md5"]:
-            raise Exception("summary_mode is invalid")
-
-
-class OverflowCheckConfig(BaseConfig):
-    def __init__(self, json_config):
-        super().__init__(json_config)
-        self.overflow_num = json_config.get("overflow_nums")
-        self.check_mode = json_config.get("check_mode")
-        self.check_overflow_config()
-
-    def check_overflow_config(self):
-        if self.overflow_num is not None and not isinstance(self.overflow_num, int):
-            raise Exception("overflow_num is invalid")
-        if self.check_mode is not None and self.check_mode not in ["all", "aicore", "atomic"]:
-            raise Exception("check_mode is invalid")
-
-
-class FreeBenchmarkCheckConfig(BaseConfig):
-    def __init__(self, json_config):
-        super().__init__(json_config)
-        self.fuzz_device = json_config.get("fuzz_device")
-        self.pert_mode = json_config.get("pert_mode")
-        self.handler_type = json_config.get("handler_type")
-        self.fuzz_level = json_config.get("fuzz_level")
-        self.fuzz_stage = json_config.get("fuzz_stage")
-        self.if_preheat = json_config.get("if_preheat")
-        self.preheat_step = json_config.get("preheat_step")
-        self.max_sample = json_config.get("max_sample")
-        self.check_freebenchmark_config()
-
-    def check_freebenchmark_config(self):
-        if self.if_preheat and self.handler_type == "fix":
-            raise Exception("Preheating is not supported in fix handler type")
-        if self.preheat_step and self.preheat_step == 0:
-            raise Exception("preheat_step cannot be 0")
-
-def parse_task_config(task, json_config):
-    default_dic = {}
-    if task == Const.TENSOR:
-        config_dic = json_config.get(Const.TENSOR) if json_config.get(Const.TENSOR) else default_dic
-        return TensorConfig(config_dic)
-    elif task == Const.STATISTICS:
-        config_dic = json_config.get(Const.STATISTICS) if json_config.get(Const.STATISTICS) else default_dic
-        return StatisticsConfig(config_dic)
-    elif task == Const.OVERFLOW_CHECK:
-        config_dic = json_config.get(Const.OVERFLOW_CHECK) if json_config.get(Const.OVERFLOW_CHECK) else default_dic
-        return OverflowCheckConfig(config_dic)
-    elif task == Const.FREE_BENCHMARK:
-        config_dic = json_config.get(Const.FREE_BENCHMARK) if json_config.get(Const.FREE_BENCHMARK) else default_dic
-        return FreeBenchmarkCheckConfig(config_dic)
-    else:
-        return StatisticsConfig(default_dic)
-
-
-def parse_json_config(json_file_path, task):
-    if not json_file_path:
-        config_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-        json_file_path = os.path.join(os.path.join(config_dir, "config"), "config.json")
-    with FileOpen(json_file_path, 'r') as file:
-        json_config = json.load(file)
-    common_config = CommonConfig(json_config)
-    if task and task in Const.TASK_LIST:
-        task_config = parse_task_config(task, json_config)
-    else:
-        task_config = parse_task_config(common_config.task, json_config)
-    return common_config, task_config
+import json
+import os
+
+from msprobe.core.common_config import CommonConfig, BaseConfig
+from msprobe.core.common.file_utils import FileOpen
+from msprobe.core.common.const import Const
+from msprobe.pytorch.hook_module.utils import get_ops
+from msprobe.core.grad_probe.constant import level_adp
+from msprobe.core.grad_probe.utils import check_numeral_list_ascend
+
+
+class TensorConfig(BaseConfig):
+    def __init__(self, json_config):
+        super().__init__(json_config)
+        self.online_run_ut = json_config.get("online_run_ut", False)
+        self.nfs_path = json_config.get("nfs_path", "")
+        self.host = json_config.get("host", "")
+        self.port = json_config.get("port", -1)
+        self.tls_path = json_config.get("tls_path", "")
+        self.check_config()
+        self._check_file_format()
+        self._check_tls_path_config()
+
+    def _check_file_format(self):
+        if self.file_format is not None and self.file_format not in ["npy", "bin"]:
+            raise Exception("file_format is invalid")
+
+    def _check_tls_path_config(self):
+        if self.tls_path:
+            if not os.path.exists(self.tls_path):
+                raise Exception("tls_path: %s does not exist" % self.tls_path)
+            if not os.path.exists(os.path.join(self.tls_path, "client.key")):
+                raise Exception("tls_path does not contain client.key")
+            if not os.path.exists(os.path.join(self.tls_path, "client.crt")):
+                raise Exception("tls_path does not contain client.crt")
+
+
+class StatisticsConfig(BaseConfig):
+    def __init__(self, json_config):
+        super().__init__(json_config)
+        self.check_config()
+        self._check_summary_mode()
+
+    def _check_summary_mode(self):
+        if self.summary_mode and self.summary_mode not in ["statistics", "md5"]:
+            raise Exception("summary_mode is invalid")
+
+
+class OverflowCheckConfig(BaseConfig):
+    def __init__(self, json_config):
+        super().__init__(json_config)
+        self.overflow_nums = json_config.get("overflow_nums")
+        self.check_mode = json_config.get("check_mode")
+        self.check_overflow_config()
+
+    def check_overflow_config(self):
+        if self.overflow_nums is not None and not isinstance(self.overflow_nums, int):
+            raise Exception("overflow_num is invalid")
+        if self.check_mode is not None and self.check_mode not in ["all", "aicore", "atomic"]:
+            raise Exception("check_mode is invalid")
+
+
+class FreeBenchmarkCheckConfig(BaseConfig):
+    def __init__(self, json_config):
+        super().__init__(json_config)
+        self.fuzz_device = json_config.get("fuzz_device")
+        self.pert_mode = json_config.get("pert_mode")
+        self.handler_type = json_config.get("handler_type")
+        self.fuzz_level = json_config.get("fuzz_level")
+        self.fuzz_stage = json_config.get("fuzz_stage")
+        self.if_preheat = json_config.get("if_preheat")
+        self.preheat_step = json_config.get("preheat_step")
+        self.max_sample = json_config.get("max_sample")
+        self.check_freebenchmark_config()
+
+    def check_freebenchmark_config(self):
+        if self.if_preheat and self.handler_type == "fix":
+            raise Exception("Preheating is not supported in fix handler type")
+        if self.preheat_step and self.preheat_step == 0:
+            raise Exception("preheat_step cannot be 0")
+
+
+class RunUTConfig(BaseConfig):
+    WrapApi = get_ops()
+
+    def __init__(self, json_config):
+        super().__init__(json_config)
+        self.white_list = json_config.get("white_list", Const.DEFAULT_LIST)
+        self.black_list = json_config.get("black_list", Const.DEFAULT_LIST)
+        self.error_data_path = json_config.get("error_data_path", Const.DEFAULT_PATH)
+        self.is_online = json_config.get("is_online", False)
+        self.nfs_path = json_config.get("nfs_path", "")
+        self.host = json_config.get("host", "")
+        self.port = json_config.get("port", -1)
+        self.rank_list = json_config.get("rank_list", Const.DEFAULT_LIST)
+        self.tls_path = json_config.get("tls_path", "")
+        self.check_run_ut_config()
+
+    @classmethod
+    def check_filter_list_config(cls, key, filter_list):
+        if not isinstance(filter_list, list):
+            raise Exception("%s must be a list type" % key)
+        if not all(isinstance(item, str) for item in filter_list):
+            raise Exception("All elements in %s must be string type" % key)
+        invalid_api = [item for item in filter_list if item not in cls.WrapApi]
+        if invalid_api:
+            raise Exception("Invalid api in %s: %s" % (key, invalid_api))
+
+    @classmethod
+    def check_error_data_path_config(cls, error_data_path):
+        if not os.path.exists(error_data_path):
+            raise Exception("error_data_path: %s does not exist" % error_data_path)
+
+    @classmethod
+    def check_nfs_path_config(cls, nfs_path):
+        if nfs_path and not os.path.exists(nfs_path):
+            raise Exception("nfs_path: %s does not exist" % nfs_path)
+
+    @classmethod
+    def check_tls_path_config(cls, tls_path):
+        if tls_path:
+            if not os.path.exists(tls_path):
+                raise Exception("tls_path: %s does not exist" % tls_path)
+            if not os.path.exists(os.path.join(tls_path, "server.key")):
+                raise Exception("tls_path does not contain server.key")
+            if not os.path.exists(os.path.join(tls_path, "server.crt")):
+                raise Exception("tls_path does not contain server.crt")
+
+    def check_run_ut_config(self):
+        RunUTConfig.check_filter_list_config(Const.WHITE_LIST, self.white_list)
+        RunUTConfig.check_filter_list_config(Const.BLACK_LIST, self.black_list)
+        RunUTConfig.check_error_data_path_config(self.error_data_path)
+        RunUTConfig.check_nfs_path_config(self.nfs_path)
+        RunUTConfig.check_tls_path_config(self.tls_path)
+
+
+class GradToolConfig(BaseConfig):
+    def __init__(self, json_config):
+        super().__init__(json_config)
+        self.grad_level = json_config.get("grad_level", "L1")
+        self.param_list = json_config.get("param_list", [])
+        self.bounds = json_config.get("bounds", [-1, 0, 1])
+        self._check_config()
+    
+    def _check_config(self):
+        if self.grad_level not in level_adp.keys():
+            raise Exception(f"grad_level must be one of {level_adp.keys()}")
+        if not isinstance(self.param_list, list):
+            raise Exception(f"param_list must be a list")
+        check_numeral_list_ascend(self.bounds)
+
+
+def parse_task_config(task, json_config):
+    default_dic = {}
+    if task == Const.TENSOR:
+        config_dic = json_config.get(Const.TENSOR, default_dic)
+        return TensorConfig(config_dic)
+    elif task == Const.STATISTICS:
+        config_dic = json_config.get(Const.STATISTICS, default_dic)
+        return StatisticsConfig(config_dic)
+    elif task == Const.OVERFLOW_CHECK:
+        config_dic = json_config.get(Const.OVERFLOW_CHECK, default_dic)
+        return OverflowCheckConfig(config_dic)
+    elif task == Const.FREE_BENCHMARK:
+        config_dic = json_config.get(Const.FREE_BENCHMARK, default_dic)
+        return FreeBenchmarkCheckConfig(config_dic)
+    elif task == Const.RUN_UT:
+        config_dic = json_config.get(Const.RUN_UT, default_dic)
+        return RunUTConfig(config_dic)
+    elif task == Const.GRAD_PROBE:
+        config_dic = json_config.get(Const.GRAD_PROBE, default_dic)
+        return GradToolConfig(config_dic)
+    else:
+        return StatisticsConfig(default_dic)
+
+
+def parse_json_config(json_file_path, task):
+    if not json_file_path:
+        config_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+        json_file_path = os.path.join(config_dir, "config.json")
+    with FileOpen(json_file_path, 'r') as file:
+        json_config = json.load(file)
+    common_config = CommonConfig(json_config)
+    if task and task in Const.TASK_LIST:
+        task_config = parse_task_config(task, json_config)
+    else:
+        task_config = parse_task_config(common_config.task, json_config)
+    return common_config, task_config