mindstudio-probe 1.0.1__py3-none-any.whl → 1.0.4__py3-none-any.whl

msprobe/pytorch/parse_tool/lib/config.py

@@ -1,51 +1,52 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-
-import os
-import numpy as np
-
-
-class Const:
-
-    MS_ACCU_CMP_PATH = '/usr/local/Ascend/ascend-toolkit/latest/tools/operator_cmp/compare/msaccucmp.py'
-    MS_ACCU_CMP_FILE_NAME = 'msaccucmp.py'
-    ROOT_DIR = ""
-    LOG_LEVEL = "NOTSET"
-    DATA_ROOT_DIR = os.path.join(ROOT_DIR, 'parse_data')
-    DUMP_CONVERT_DIR = os.path.join(DATA_ROOT_DIR, 'dump_convert')
-    COMPARE_DIR = os.path.join(DATA_ROOT_DIR, 'compare_result')
-    BATCH_DUMP_CONVERT_DIR = os.path.join(DATA_ROOT_DIR, 'batch_dump_convert')
-    BATCH_COMPARE_DIR = os.path.join(DATA_ROOT_DIR, 'batch_compare_result')
-    OFFLINE_DUMP_CONVERT_PATTERN = \
-        r"^([A-Za-z0-9_-]+)\.([A-Za-z0-9_-]+)\.([0-9]+)(\.[0-9]+)?\.([0-9]{1,255})" \
-        r"\.([a-z]+)\.([0-9]{1,255})(\.[x0-9]+)?\.npy$"
-    NUMPY_PATTERN = r".*\.npy$"
-    NPY_SUFFIX = ".npy"
-    PKL_SUFFIX = ".pkl"
-    DIRECTORY_LENGTH = 4096
-    FILE_NAME_LENGTH = 255
-    FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$'
-    ONE_GB = 1 * 1024 * 1024 * 1024
-    TEN_GB = 10 * 1024 * 1024 * 1024
-    FLOAT_TYPE = [np.half, np.single, float, np.double, np.float64, np.longdouble, np.float32, np.float16]
-    HEADER = r"""    ____
-       / __ \____ ______________
-      / /_/ / __ `/ ___/ ___/ _ \
-     / ____/ /_/ / /  (__  )  __/
-    /_/    \__,_/_/  /____/\___/
-
-    """
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+import os
+import numpy as np
+
+
+class Const:
+
+    MS_ACCU_CMP_PATH = '/usr/local/Ascend/ascend-toolkit/latest/tools/operator_cmp/compare/msaccucmp.py'
+    MS_ACCU_CMP_FILE_NAME = 'msaccucmp.py'
+    ROOT_DIR = ""
+    LOG_LEVEL = "NOTSET"
+    DATA_ROOT_DIR = os.path.join(ROOT_DIR, 'parse_data')
+    DUMP_CONVERT_DIR = os.path.join(DATA_ROOT_DIR, 'dump_convert')
+    COMPARE_DIR = os.path.join(DATA_ROOT_DIR, 'compare_result')
+    BATCH_DUMP_CONVERT_DIR = os.path.join(DATA_ROOT_DIR, 'batch_dump_convert')
+    BATCH_COMPARE_DIR = os.path.join(DATA_ROOT_DIR, 'batch_compare_result')
+    OFFLINE_DUMP_CONVERT_PATTERN = \
+        r"^([A-Za-z0-9_-]+)\.([A-Za-z0-9_-]+)\.([0-9]+)(\.[0-9]+)?\.([0-9]{1,255})" \
+        r"\.([a-z]+)\.([0-9]{1,255})(\.[x0-9]+)?\.npy$"
+    NUMPY_PATTERN = r"^[\w\-_-]\.npy$"
+    NPY_SUFFIX = ".npy"
+    PKL_SUFFIX = ".pkl"
+    DIRECTORY_LENGTH = 4096
+    FILE_NAME_LENGTH = 255
+    MAX_TRAVERSAL_DEPTH = 5
+    FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$'
+    ONE_GB = 1 * 1024 * 1024 * 1024
+    TEN_GB = 10 * 1024 * 1024 * 1024
+    FLOAT_TYPE = [np.half, np.single, float, np.double, np.float64, np.longdouble, np.float32, np.float16]
+    HEADER = r"""    ____
+       / __ \____ ______________
+      / /_/ / __ `/ ___/ ___/ _ \
+     / ____/ /_/ / /  (__  )  __/
+    /_/    \__,_/_/  /____/\___/
+
+    """

msprobe/pytorch/parse_tool/lib/file_desc.py

@@ -1,31 +1,31 @@
-# coding=utf-8
-import os
-
-
-class FileDesc(object):
-    def __init__(self, file_name, dir_path, timestamp=-1):
-        self.file_name = file_name
-        self.dir_path = dir_path
-        self.path = os.path.join(dir_path, file_name)
-        self.timestamp = timestamp
-        self.idx = 0
-        if self.timestamp == -1:
-            self.timestamp = os.path.getmtime(self.path)
-
-
-class NpuDumpFileDesc(FileDesc):
-    def __init__(self, file_name, dir_path, timestamp, op_name, op_type, task_id, stream_id=0):
-        super(NpuDumpFileDesc, self).__init__(file_name, dir_path, timestamp)
-        self.op_name = op_name
-        self.op_type = op_type
-        self.task_id = task_id
-        stream_id = 0 if stream_id is None else int(stream_id)
-        self.stream_id = stream_id
-        self.idx = dir_path.split(os.sep)[-1]
-
-
-class DumpDecodeFileDesc(NpuDumpFileDesc):
-    def __init__(self, file_name, dir_path, timestamp, op_name, op_type, task_id, anchor_type, anchor_idx):
-        super(DumpDecodeFileDesc, self).__init__(file_name, dir_path, timestamp, op_name, op_type, task_id)
-        self.type = anchor_type
-        self.idx = anchor_idx
+# coding=utf-8
+import os
+
+
+class FileDesc(object):
+    def __init__(self, file_name, dir_path, timestamp=-1):
+        self.file_name = file_name
+        self.dir_path = dir_path
+        self.path = os.path.join(dir_path, file_name)
+        self.timestamp = timestamp
+        self.idx = 0
+        if self.timestamp == -1:
+            self.timestamp = os.path.getmtime(self.path)
+
+
+class NpuDumpFileDesc(FileDesc):
+    def __init__(self, file_name, dir_path, timestamp, op_name, op_type, task_id, stream_id=0):
+        super(NpuDumpFileDesc, self).__init__(file_name, dir_path, timestamp)
+        self.op_name = op_name
+        self.op_type = op_type
+        self.task_id = task_id
+        stream_id = 0 if stream_id is None else int(stream_id)
+        self.stream_id = stream_id
+        self.idx = dir_path.split(os.sep)[-1]
+
+
+class DumpDecodeFileDesc(NpuDumpFileDesc):
+    def __init__(self, file_name, dir_path, timestamp, op_name, op_type, task_id, anchor_type, anchor_idx):
+        super(DumpDecodeFileDesc, self).__init__(file_name, dir_path, timestamp, op_name, op_type, task_id)
+        self.type = anchor_type
+        self.idx = anchor_idx

msprobe/pytorch/parse_tool/lib/interactive_cli.py

@@ -1,102 +1,102 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-import cmd
-import argparse
-from msprobe.pytorch.parse_tool.lib.parse_tool import ParseTool
-from msprobe.pytorch.parse_tool.lib.utils import Util
-from msprobe.pytorch.parse_tool.lib.config import Const
-from msprobe.pytorch.parse_tool.lib.parse_exception import catch_exception
-
-
-class InteractiveCli(cmd.Cmd):
-    def __init__(self):
-        super().__init__()
-        self.prompt = "Parse >>> "
-        self.parse_tool = ParseTool()
-        self.util = Util()
-        self.util.print_panel(Const.HEADER)
-        self.prepare()
-
-    @staticmethod
-    def _parse_argv(line, insert=None):
-        argv = line.split() if line != "" else []
-        if "-h" in argv:
-            return argv
-        if insert is not None and len(argv) and argv[0] != insert:
-            argv.insert(0, insert)
-        return argv
-
-    def prepare(self):
-        self.parse_tool.prepare()
-
-    @catch_exception
-    def default(self, line=""):
-        self.util.execute_command(line)
-        return False
-
-    @catch_exception
-    def do_run(self, line=""):
-        self.util.execute_command(line)
-
-    @catch_exception
-    def do_vc(self, line=""):
-        parser = argparse.ArgumentParser()
-        parser.add_argument(
-            "-m", "--my_dump_path", dest="my_dump_path", default=None,
-            help="<Required> my dump path, the data compared with golden data",
-            required=True
-        )
-        parser.add_argument(
-            "-g", "--golden_dump_path", dest="golden_dump_path", default=None,
-            help="<Required> the golden dump data path",
-            required=True
-        )
-        parser.add_argument(
-            "-out", "--output_path", dest="output_path", default=None,
-            help="<Optional> the output path",
-            required=False
-        )
-        parser.add_argument(
-            "-cmp_path", "--msaccucmp_path", dest="msaccucmp_path", default=None,
-            help="<Optional> the msaccucmp.py file path",
-            required=False
-        )
-        args = parser.parse_args(self._parse_argv(line))
-        self.util.check_path_valid(args.my_dump_path)
-        self.util.check_path_valid(args.golden_dump_path)
-        self.util.check_files_in_path(args.my_dump_path)
-        self.util.check_files_in_path(args.golden_dump_path)
-        if self.util.dir_contains_only(args.my_dump_path, ".npy") and \
-            self.util.dir_contains_only(args.golden_dump_path, ".npy"):
-            self.parse_tool.do_compare_converted_dir(args)
-        else:
-            self.parse_tool.do_vector_compare(args)
-
-    def do_dc(self, line=""):
-        self.parse_tool.do_convert_dump(self._parse_argv(line))
-
-    def do_pt(self, line=""):
-        self.parse_tool.do_print_data(self._parse_argv(line))
-
-    def do_pk(self, line=""):
-        self.parse_tool.do_parse_pkl(self._parse_argv(line))
-
-    def do_cn(self, line=''):
-        self.parse_tool.do_compare_data(self._parse_argv(line))
-
-    def do_cad(self, line=''):
-        self.parse_tool.do_convert_api_dir(self._parse_argv(line))
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+import cmd
+import argparse
+from msprobe.pytorch.parse_tool.lib.parse_tool import ParseTool
+from msprobe.pytorch.parse_tool.lib.utils import Util
+from msprobe.pytorch.parse_tool.lib.config import Const
+from msprobe.pytorch.parse_tool.lib.parse_exception import catch_exception
+
+
+class InteractiveCli(cmd.Cmd):
+    def __init__(self):
+        super().__init__()
+        self.prompt = "Parse >>> "
+        self.parse_tool = ParseTool()
+        self.util = Util()
+        self.util.print_panel(Const.HEADER)
+        self.prepare()
+
+    @staticmethod
+    def _parse_argv(line, insert=None):
+        argv = line.split() if line != "" else []
+        if "-h" in argv:
+            return argv
+        if insert is not None and len(argv) and argv[0] != insert:
+            argv.insert(0, insert)
+        return argv
+
+    def prepare(self):
+        self.parse_tool.prepare()
+
+    @catch_exception
+    def default(self, line=""):
+        self.util.execute_command(line)
+        return False
+
+    @catch_exception
+    def do_run(self, line=""):
+        self.util.execute_command(line)
+
+    @catch_exception
+    def do_vc(self, line=""):
+        parser = argparse.ArgumentParser()
+        parser.add_argument(
+            "-m", "--my_dump_path", dest="my_dump_path", default=None,
+            help="<Required> my dump path, the data compared with golden data",
+            required=True
+        )
+        parser.add_argument(
+            "-g", "--golden_dump_path", dest="golden_dump_path", default=None,
+            help="<Required> the golden dump data path",
+            required=True
+        )
+        parser.add_argument(
+            "-out", "--output_path", dest="output_path", default=None,
+            help="<Optional> the output path",
+            required=False
+        )
+        parser.add_argument(
+            "-cmp_path", "--msaccucmp_path", dest="msaccucmp_path", default=None,
+            help="<Optional> the msaccucmp.py file path",
+            required=False
+        )
+        args = parser.parse_args(self._parse_argv(line))
+        self.util.check_path_valid(args.my_dump_path)
+        self.util.check_path_valid(args.golden_dump_path)
+        self.util.check_files_in_path(args.my_dump_path)
+        self.util.check_files_in_path(args.golden_dump_path)
+        if self.util.dir_contains_only(args.my_dump_path, ".npy") and \
+            self.util.dir_contains_only(args.golden_dump_path, ".npy"):
+            self.parse_tool.do_compare_converted_dir(args)
+        else:
+            self.parse_tool.do_vector_compare(args)
+
+    def do_dc(self, line=""):
+        self.parse_tool.do_convert_dump(self._parse_argv(line))
+
+    def do_pt(self, line=""):
+        self.parse_tool.do_print_data(self._parse_argv(line))
+
+    def do_pk(self, line=""):
+        self.parse_tool.do_parse_pkl(self._parse_argv(line))
+
+    def do_cn(self, line=''):
+        self.parse_tool.do_compare_data(self._parse_argv(line))
+
+    def do_cad(self, line=''):
+        self.parse_tool.do_convert_api_dir(self._parse_argv(line))

msprobe/pytorch/parse_tool/lib/parse_exception.py

@@ -1,54 +1,54 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-import logging
-from msprobe.core.common.exceptions import FileCheckException
-
-
-class ParseException(Exception):
-
-    PARSE_INVALID_PATH_ERROR = 0
-    PARSE_NO_FILE_ERROR = 1
-    PARSE_NO_MODULE_ERROR = 2
-    PARSE_INVALID_DATA_ERROR = 3
-    PARSE_INVALID_FILE_FORMAT_ERROR = 4
-    PARSE_UNICODE_ERROR = 5
-    PARSE_JSONDECODE_ERROR = 6
-    PARSE_MSACCUCMP_ERROR = 7
-    PARSE_LOAD_NPY_ERROR = 8
-    PARSE_INVALID_PARAM_ERROR = 9
-
-    def __init__(self, code, error_info=""):
-        super(ParseException, self).__init__()
-        self.error_info = error_info
-        self.code = code
-
-
-def catch_exception(func):
-    def inner(*args, **kwargs):
-        log = logging.getLogger()
-        line = args[-1] if len(args) == 2 else ""
-        result = None
-        try:
-            result = func(*args, **kwargs)
-        except OSError:
-            log.error("%s: command not found" % line)
-        except ParseException:
-            log.error("Command execution failed")
-        except FileCheckException:
-            log.error("Command execution failed")
-        return result
-    return inner
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+import logging
+from msprobe.core.common.exceptions import FileCheckException
+
+
+class ParseException(Exception):
+
+    PARSE_INVALID_PATH_ERROR = 0
+    PARSE_NO_FILE_ERROR = 1
+    PARSE_NO_MODULE_ERROR = 2
+    PARSE_INVALID_DATA_ERROR = 3
+    PARSE_INVALID_FILE_FORMAT_ERROR = 4
+    PARSE_UNICODE_ERROR = 5
+    PARSE_JSONDECODE_ERROR = 6
+    PARSE_MSACCUCMP_ERROR = 7
+    PARSE_LOAD_NPY_ERROR = 8
+    PARSE_INVALID_PARAM_ERROR = 9
+
+    def __init__(self, code, error_info=""):
+        super(ParseException, self).__init__()
+        self.error_info = error_info
+        self.code = code
+
+
+def catch_exception(func):
+    def inner(*args, **kwargs):
+        log = logging.getLogger()
+        line = args[-1] if len(args) == 2 else ""
+        result = None
+        try:
+            result = func(*args, **kwargs)
+        except OSError:
+            log.error("%s: command not found" % line)
+        except ParseException:
+            log.error("Command execution failed")
+        except FileCheckException:
+            log.error("Command execution failed")
+        return result
+    return inner