mindstudio-probe 1.0.1__py3-none-any.whl

msprobe/pytorch/online_dispatch/utils.py

@@ -0,0 +1,187 @@
+import os
+import inspect
+import logging
+import psutil
+import torch
+import numpy as np
+
+try:
+    import torch_npu
+except ImportError:
+    pta_cpu_device = None
+else:
+    pta_cpu_device = torch.device("cpu")
+
+from msprobe.core.common.const import CompareConst, FileCheckConst
+from msprobe.core.common.file_check import change_mode
+
+cpu_device = torch._C.device("cpu")
+COLOR_RED = '\033[31m'
+COLOR_GREEN = '\033[32m'
+COLOR_YELLOW = '\033[33m'
+COLOR_BLUE = '\033[34m'
+COLOR_PURPLE = '\033[35m'
+COLOR_CYAN = '\033[36m'
+COLOR_GRAY = '\033[37m'
+COLOR_RESET = '\033[0m'
+
+COMPARE_LOGO = '''
+            _ _                                                       
+  ___  _ __ | (_)_ __   ___    ___ ___  _ __ ___  _ __   __ _ _ __ ___ 
+ / _ \\| '_ \\| | | '_ \\ / _ \\  / __/ _ \\| '_ ` _ \\| '_ \\ / _` | '__/ _ \\ 
+| (_) | | | | | | | | |  __/ | (_| (_) | | | | | | |_) | (_| | | |  __/
+ \\___/|_| |_|_|_|_| |_|\\___|  \\___\\___/|_| |_| |_| .__/ \\__,_|_|  \\___|
+                                                 |_|    
+'''
+
+CSV_COLUMN_NAME = [CompareConst.NPU_NAME,
+                   CompareConst.BENCH_NAME,
+                   CompareConst.NPU_DTYPE,
+                   CompareConst.BENCH_DTYPE,
+                   CompareConst.NPU_SHAPE,
+                   CompareConst.BENCH_SHAPE,
+                   CompareConst.NPU_MAX,
+                   CompareConst.NPU_MIN,
+                   CompareConst.NPU_MEAN,
+                   CompareConst.BENCH_MAX,
+                   CompareConst.BENCH_MIN,
+                   CompareConst.BENCH_MEAN,
+                   CompareConst.COSINE,
+                   CompareConst.MAX_ABS_ERR,
+                   CompareConst.MAX_RELATIVE_ERR,
+                   CompareConst.ACCURACY,
+                   CompareConst.STACK,
+                   CompareConst.ERROR_MESSAGE]
+
+FLOAT_TYPE = [np.half, np.single, float, np.double, np.float64, np.longdouble, np.float32, np.float16]
+BOOL_TYPE = [bool, np.uint8]
+INT_TYPE = [np.int32, np.int64]
+
+
+def get_callstack():
+    callstack = []
+    for (_, path, line, func, code, _) in inspect.stack()[2:]:
+        if code:
+            stack_line = [path, str(line), func, code[0].strip() if code else code]
+        else:
+            stack_line = [path, str(line), func, code]
+        callstack.append(stack_line)
+    return callstack
+
+
+def np_save_data(data, file_name, data_path):
+    try:
+        if hasattr(data, "numpy"):
+            data = data.numpy()
+        dump_path = os.path.join(data_path, f'{file_name}.npy')
+        np.save(dump_path, data)
+        change_mode(dump_path, FileCheckConst.DATA_FILE_AUTHORITY)
+    except Exception as e:
+        logger_error("save numpy failed, error: {}".format(e))
+    finally:
+        pass
+
+
+def data_to_cpu(data, deep, data_cpu):
+    global cpu_device
+    list_cpu = []
+    if isinstance(data, torch.Tensor):
+        if data.device == cpu_device or data.device == pta_cpu_device:
+            tensor_copy = data.clone().detach()
+        else:
+            tensor_copy = data.cpu().detach()
+        if tensor_copy.dtype in [torch.float16, torch.half, torch.bfloat16]:
+            tensor_copy = tensor_copy.float()
+
+        if deep == 0:
+            data_cpu.append(tensor_copy)
+        return tensor_copy
+    elif isinstance(data, list):
+        for v in data:
+            list_cpu.append(data_to_cpu(v, deep + 1, data_cpu))
+        if deep == 0:
+            data_cpu.append(list_cpu)
+        return list_cpu
+    elif isinstance(data, tuple):
+        for v in data:
+            list_cpu.append(data_to_cpu(v, deep + 1, data_cpu))
+        tuple_cpu = tuple(list_cpu)
+        if deep == 0:
+            data_cpu.append(tuple_cpu)
+        return tuple_cpu
+    elif isinstance(data, dict):
+        dict_cpu = {}
+        for k, v in data.items():
+            dict_cpu[k] = data_to_cpu(v, deep + 1, data_cpu)
+        if deep == 0:
+            data_cpu.append(dict_cpu)
+        return dict_cpu
+    elif isinstance(data, torch._C.device):
+        return cpu_device
+    else:
+        if deep == 0:
+            data_cpu.append(data)
+        return data
+
+
+def get_mp_logger():
+    logger = logging.getLogger(__name__)
+    if not logger.handlers:
+        logger.setLevel(logging.INFO)
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter('%(asctime)s %(message)s')
+        logger.propagate = True
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+    return logger.info
+
+
+def logger_debug(mesg):
+    logger = get_mp_logger()
+    logger(f'DEBUG ' + mesg)
+
+
+def logger_info(mesg):
+    logger = get_mp_logger()
+    logger(f'INFO ' + mesg)
+
+
+def logger_warn(mesg):
+    logger = get_mp_logger()
+    logger(f'{COLOR_YELLOW}WARNING {mesg} {COLOR_RESET}')
+
+
+def logger_error(mesg):
+    logger = get_mp_logger()
+    logger(f'{COLOR_RED}ERROR {mesg} {COLOR_RESET}')
+
+
+def logger_user(mesg):
+    logger = get_mp_logger()
+    logger(mesg)
+
+
+def logger_logo():
+    logger_user(f'{COLOR_CYAN}{COMPARE_LOGO} {COLOR_RESET}')
+
+
+def get_sys_info():
+    mem = psutil.virtual_memory()
+    cpu_percent = psutil.cpu_percent(interval=1)
+    sys_info = f'Total: {mem.total / 1024 / 1024:.2f}MB ' \
+               f'Free: {mem.available / 1024 / 1024:.2f} MB ' \
+               f'Used: {mem.used / 1024 / 1024:.2f} MB ' \
+               f'CPU: {cpu_percent}% '
+    return sys_info
+
+
+class DispatchException(Exception):
+    INVALID_PARAMETER = 0
+
+    def __init__(self, err_code, err_msg=""):
+        super(DispatchException, self).__init__()
+        self.err_code = err_code
+        self.err_msg = err_msg
+
+    def __str__(self):
+        return self.err_msg

msprobe/pytorch/parse.py

@@ -0,0 +1,4 @@
+from msprobe.pytorch.parse_tool import cli
+
+if __name__ == '__main__':
+    cli.parse()

msprobe/pytorch/parse_tool/cli.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+from msprobe.pytorch.parse_tool.lib.interactive_cli import InteractiveCli
+from msprobe.pytorch.common.log import logger
+
+
+def _run_interactive_cli(cli=None):
+    logger.info("Interactive command mode")
+    if not cli:
+        cli = InteractiveCli()
+    try:
+        cli.cmdloop(intro="Start Parsing........")
+    except KeyboardInterrupt:
+        logger.info("Exit parsing.......")
+
+
+def parse():
+    _run_interactive_cli()

msprobe/pytorch/parse_tool/lib/compare.py

@@ -0,0 +1,259 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+import os
+import time
+import numpy as np
+from collections import namedtuple
+from msprobe.pytorch.parse_tool.lib.utils import Util
+from msprobe.pytorch.parse_tool.lib.config import Const
+from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException
+
+
+class Compare:
+    def __init__(self):
+        self.util = Util()
+        self.log = self.util.log
+        self.vector_compare_result = {}
+
+    def npu_vs_npu_compare(self, my_dump_path, golden_dump_path, result_dir, msaccucmp_path):
+        self.log.info("Start Compare ...............")
+        self.compare_vector(my_dump_path, golden_dump_path, result_dir, msaccucmp_path)
+        self.log.info("Compare finished!!")
+
+    def compare_vector(self, my_dump_path, golden_dump_path, result_dir, msaccucmp_path):
+        self.util.create_dir(result_dir)
+        self.util.check_path_valid(result_dir)
+        call_msaccucmp = self.util.check_msaccucmp(msaccucmp_path)
+        cmd = '%s %s compare -m %s -g %s -out %s' % (
+            self.util.python, call_msaccucmp, my_dump_path, golden_dump_path, result_dir
+        )
+        return self.util.execute_command(cmd)
+
+    def convert_dump_to_npy(self, dump_file, data_format, output, msaccucmp_path):
+        dump_file = self.util.path_strip(dump_file)
+        file_name = ""
+        if os.path.isfile(dump_file):
+            self.log.info("Covert file is: %s", dump_file)
+            file_name = os.path.basename(dump_file)
+        elif os.path.isdir(dump_file):
+            self.log.info("Convert all files in path: %s", dump_file)
+            file_name = ""
+        output = output if output else Const.DUMP_CONVERT_DIR
+        convert = self.convert(dump_file, data_format, output, msaccucmp_path)
+        if convert == 0:
+            convert_files = self.util.list_convert_files(output, file_name)
+
+            summary_txt = ["SrcFile: %s" % dump_file]
+            for convert_file in convert_files.values():
+                summary_txt.append(" - %s" % convert_file.file_name)
+            self.log.info("Transfer result is saved in : %s", os.path.realpath(output))
+            self.util.print_panel("\n".join(summary_txt))
+
+    def convert(self, dump_file, data_format, output, msaccucmp_path):
+        self.util.create_dir(output)
+        self.util.check_path_valid(output)
+        call_msaccucmp = self.util.check_msaccucmp(msaccucmp_path)
+        if data_format:
+            cmd = '%s %s convert -d %s -out %s -f %s' % (
+                self.util.python, call_msaccucmp, dump_file, output, data_format
+            )
+        else:
+            cmd = '%s %s convert -d %s -out %s' % (
+                self.util.python, call_msaccucmp, dump_file, output
+            )
+        return self.util.execute_command(cmd)
+
+    def compare_data(self, args):
+        """Compare data"""
+        (left, right, save_txt, rl, al, diff_count) = args
+        if left is None or right is None:
+            raise ParseException("invalid input or output")
+        try:
+            left_data = np.load(left)
+            right_data = np.load(right)
+        except UnicodeError as e:
+            self.log.error("%s %s" % ("UnicodeError", str(e)))
+            self.log.warning("Please check the npy file")
+            raise ParseException(ParseException.PARSE_UNICODE_ERROR) from e
+        except IOError:
+            self.log.error("Failed to load npy %s or %s." % (left, right))
+            raise ParseException(ParseException.PARSE_LOAD_NPY_ERROR) from e
+
+        # save to txt
+        if save_txt:
+            self.util.save_npy_to_txt(left_data, left + ".txt")
+            self.util.save_npy_to_txt(right_data, right + ".txt")
+        # compare data
+        (total_cnt, all_close, cos_sim, err_percent) = self.do_compare_data(left_data, right_data, rl, al, diff_count)
+        content = ['Left:', ' ├─ NpyFile: %s' % left]
+        if save_txt:
+            content.append(' ├─ TxtFile: [green]%s.txt[/green]' % left)
+        content.append(' └─ NpySpec: [yellow]%s[/yellow]' % self.util.gen_npy_info_txt(left_data))
+        content.append('Right:')
+        content.append(' ├─ NpyFile: %s' % right)
+        if save_txt:
+            content.append(' ├─ TxtFile: [green]%s.txt[/green]' % right)
+        content.append(' └─ NpySpec: [yellow]%s[/yellow]' % self.util.gen_npy_info_txt(right_data))
+        content.append('NumCnt:   %s' % total_cnt)
+        content.append('AllClose: %s' % all_close)
+        content.append('CosSim:   %s' % cos_sim)
+        content.append('ErrorPer: %s  (rl= %s, al= %s)' % (err_percent, rl, al))
+        self.util.print_panel("\n".join(content))
+
+    def do_compare_data(self, left, right, rl=0.001, al=0.001, diff_count=20):
+        data_left = left.astype(np.float32)
+        data_right = right.astype(np.float32)
+        shape_left = data_left.shape
+        shape_right = data_right.shape
+        if shape_left != shape_right:
+            self.log.warning("Data shape not equal: %s vs %s", data_left.shape, data_right.shape)
+        data_left = data_left.reshape(-1)
+        data_right = data_right.reshape(-1)
+        if data_left.shape[0] != data_right.shape[0]:
+            self.log.warning("Data size not equal: %s vs %s", data_left.shape, data_right.shape)
+            if data_left.shape[0] < data_right.shape[0]:
+                data_left = np.pad(data_left, (0, data_right.shape[0] - data_left.shape[0]), 'constant')
+            else:
+                data_right = np.pad(data_right, (0, data_left.shape[0] - data_right.shape[0]), 'constant')
+        all_close = np.allclose(data_left, data_right, atol=al, rtol=rl)
+        np.seterr(divide='raise')
+        cos_sim = np.dot(data_left, data_right) / (
+            np.sqrt(np.dot(data_left, data_left)) * np.sqrt(np.dot(data_right, data_right)))
+        err_cnt = 0
+        total_cnt = data_left.shape[0]
+        diff_table_columns = ['Index', 'Left', 'Right', 'Diff']
+        err_table = self.util.create_table("Error Item Table", diff_table_columns)
+        top_table = self.util.create_table("Top Item Table", diff_table_columns)
+        for i in range(total_cnt):
+            abs_diff = abs(data_left[i] - data_right[i])
+            if i < diff_count:
+                top_table.add_row(str(i), str(data_left[i]), str(data_right[i]), str(abs_diff))
+            if abs_diff > (al + rl * abs(data_right[i])):
+                if err_cnt < diff_count:
+                    err_table.add_row(str(i), str(data_left[i]), str(data_right[i]), str(abs_diff))
+                err_cnt += 1
+        if total_cnt == 0:
+            err_percent = float(0)
+        else:
+            err_percent = float(err_cnt / total_cnt)
+        self.util.print(self.util.create_columns([err_table, top_table]))
+        do_compare_data_result = namedtuple('do_compare_data_result', ['cnt', 'close', 'cos', 'err'])
+        res = do_compare_data_result(total_cnt, all_close, cos_sim, err_percent)
+        return res
+
+    def compare_npy(self, file, bench_file, output_path):
+        data = np.load(file)
+        bench_data = np.load(bench_file)
+        shape, dtype = data.shape, data.dtype
+        bench_shape, bench_dtype = bench_data.shape, bench_data.dtype
+        filename = os.path.basename(file)
+        bench_filename = os.path.basename(bench_file)
+        if shape != bench_shape or dtype != bench_dtype:
+            self.log.error(
+                "Shape or dtype between two npy files is inconsistent. Please check the two files."
+                "File 1: %s, file 2: %s", file, bench_file)
+            self.util.deal_with_dir_or_file_inconsistency(output_path)
+            return
+        md5_consistency = False
+        if self.util.get_md5_for_numpy(data) == self.util.get_md5_for_numpy(bench_data):
+            md5_consistency = True
+        data_mean = np.mean(data)
+        bench_data_mean = np.mean(bench_data)
+        abs_error = np.abs(data - bench_data)
+        bench_data = self.util.deal_with_value_if_has_zero(bench_data)
+        rel_error = np.abs(abs_error / bench_data)
+        abs_diff_max = abs_error.max()
+        rel_diff_max = np.max(rel_error)
+        compare_result = [[filename, bench_filename, data_mean, bench_data_mean, md5_consistency, abs_diff_max,
+                           rel_diff_max]]
+        self.util.write_csv(compare_result, output_path)
+
+    def compare_all_file_in_directory(self, my_dump_dir, golden_dump_dir, output_path):
+        if not (self.util.is_subdir_count_equal(my_dump_dir, golden_dump_dir)
+                and self.util.check_npy_files_valid_in_dir(my_dump_dir)
+                and self.util.check_npy_files_valid_in_dir(golden_dump_dir)):
+            self.log.error(
+                "Top level(Npy files level) directory structure is inconsistent. Please check the two directory.")
+            self.util.deal_with_dir_or_file_inconsistency(output_path)
+            return
+        my_npy_files = self.util.get_sorted_files_names(my_dump_dir)
+        golden_npy_files = self.util.get_sorted_files_names(golden_dump_dir)
+        for my_npy_file_name, golden_npy_file_name in zip(my_npy_files, golden_npy_files):
+            my_npy_path = os.path.join(my_dump_dir, my_npy_file_name)
+            golden_npy_path = os.path.join(golden_dump_dir, golden_npy_file_name)
+            self.compare_npy(my_npy_path, golden_npy_path, output_path)
+
+    def compare_timestamp_directory(self, my_dump_dir, golden_dump_dir, output_path):
+        if not self.util.is_subdir_count_equal(my_dump_dir, golden_dump_dir):
+            self.log.error(
+                "Second level(Timestamp level) directory structure is inconsistent. Please check the two directory.")
+            self.util.deal_with_dir_or_file_inconsistency(output_path)
+            return
+        my_ordered_subdirs = self.util.get_sorted_subdirectories_names(my_dump_dir)
+        golden_ordered_subdirs = self.util.get_sorted_subdirectories_names(golden_dump_dir)
+        for my_subdir_name, golden_subdir_name in zip(my_ordered_subdirs, golden_ordered_subdirs):
+            my_subdir_path = os.path.join(my_dump_dir, my_subdir_name)
+            golden_subdir_path = os.path.join(golden_dump_dir, golden_subdir_name)
+            self.compare_all_file_in_directory(my_subdir_path, golden_subdir_path, output_path)
+
+    def compare_converted_dir(self, my_dump_dir, golden_dump_dir, output_dir):
+        if not self.util.is_subdir_count_equal(my_dump_dir, golden_dump_dir):
+            self.log.error(
+                "Top level(Opname level) directory structure is inconsistent. Please check the two directory.")
+            return
+        timestamp = int(time.time())
+        output_file_name = f"batch_compare_{timestamp}.csv"
+        output_path = os.path.join(output_dir, output_file_name)
+        title_rows = [[
+            "NPU File Name",
+            "Bench File Name",
+            "Mean",
+            "Bench Mean",
+            "Md5 Consistency",
+            "Max Abs Error",
+            "Max Relative Error"
+        ]]
+        self.util.write_csv(title_rows, output_path)
+
+        my_ordered_subdirs = self.util.get_sorted_subdirectories_names(my_dump_dir)
+        golden_ordered_subdirs = self.util.get_sorted_subdirectories_names(golden_dump_dir)
+        for my_subdir_name, golden_subdir_name in zip(my_ordered_subdirs, golden_ordered_subdirs):
+            if not my_subdir_name == golden_subdir_name:
+                self.log.error(
+                    "Top level(Opname level) directory structure is inconsistent. Please check the two directory.")
+                self.util.deal_with_dir_or_file_inconsistency(output_path)
+                return
+            my_subdir_path = os.path.join(my_dump_dir, my_subdir_name)
+            golden_subdir_path = os.path.join(golden_dump_dir, golden_subdir_name)
+            self.compare_timestamp_directory(my_subdir_path, golden_subdir_path, output_path)
+        self.util.change_filemode_safe(output_path)
+        self.log.info("Compare result is saved in : %s", output_path)
+
+    def convert_api_dir_to_npy(self, dump_dir, param, output_dir, msaccucmp_path):
+        dump_dir = self.util.path_strip(dump_dir)
+        for root, _, files in os.walk(dump_dir):
+            for file in files:
+                file_path = os.path.join(root, file)
+                file_name = os.path.basename(file_path)
+                parts = file_name.split(".")
+                if len(parts) < 5:
+                    continue
+                op_name = parts[1]
+                timestamp = parts[-1]
+                output_path = os.path.join(output_dir, op_name, timestamp)
+                self.convert_dump_to_npy(file_path, param, output_path, msaccucmp_path)

msprobe/pytorch/parse_tool/lib/config.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+import os
+import numpy as np
+
+
+class Const:
+
+    MS_ACCU_CMP_PATH = '/usr/local/Ascend/ascend-toolkit/latest/tools/operator_cmp/compare/msaccucmp.py'
+    MS_ACCU_CMP_FILE_NAME = 'msaccucmp.py'
+    ROOT_DIR = ""
+    LOG_LEVEL = "NOTSET"
+    DATA_ROOT_DIR = os.path.join(ROOT_DIR, 'parse_data')
+    DUMP_CONVERT_DIR = os.path.join(DATA_ROOT_DIR, 'dump_convert')
+    COMPARE_DIR = os.path.join(DATA_ROOT_DIR, 'compare_result')
+    BATCH_DUMP_CONVERT_DIR = os.path.join(DATA_ROOT_DIR, 'batch_dump_convert')
+    BATCH_COMPARE_DIR = os.path.join(DATA_ROOT_DIR, 'batch_compare_result')
+    OFFLINE_DUMP_CONVERT_PATTERN = \
+        r"^([A-Za-z0-9_-]+)\.([A-Za-z0-9_-]+)\.([0-9]+)(\.[0-9]+)?\.([0-9]{1,255})" \
+        r"\.([a-z]+)\.([0-9]{1,255})(\.[x0-9]+)?\.npy$"
+    NUMPY_PATTERN = r".*\.npy$"
+    NPY_SUFFIX = ".npy"
+    PKL_SUFFIX = ".pkl"
+    DIRECTORY_LENGTH = 4096
+    FILE_NAME_LENGTH = 255
+    FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$'
+    ONE_GB = 1 * 1024 * 1024 * 1024
+    TEN_GB = 10 * 1024 * 1024 * 1024
+    FLOAT_TYPE = [np.half, np.single, float, np.double, np.float64, np.longdouble, np.float32, np.float16]
+    HEADER = r"""    ____
+       / __ \____ ______________
+      / /_/ / __ `/ ___/ ___/ _ \
+     / ____/ /_/ / /  (__  )  __/
+    /_/    \__,_/_/  /____/\___/
+
+    """

msprobe/pytorch/parse_tool/lib/file_desc.py

@@ -0,0 +1,31 @@
+# coding=utf-8
+import os
+
+
+class FileDesc(object):
+    def __init__(self, file_name, dir_path, timestamp=-1):
+        self.file_name = file_name
+        self.dir_path = dir_path
+        self.path = os.path.join(dir_path, file_name)
+        self.timestamp = timestamp
+        self.idx = 0
+        if self.timestamp == -1:
+            self.timestamp = os.path.getmtime(self.path)
+
+
+class NpuDumpFileDesc(FileDesc):
+    def __init__(self, file_name, dir_path, timestamp, op_name, op_type, task_id, stream_id=0):
+        super(NpuDumpFileDesc, self).__init__(file_name, dir_path, timestamp)
+        self.op_name = op_name
+        self.op_type = op_type
+        self.task_id = task_id
+        stream_id = 0 if stream_id is None else int(stream_id)
+        self.stream_id = stream_id
+        self.idx = dir_path.split(os.sep)[-1]
+
+
+class DumpDecodeFileDesc(NpuDumpFileDesc):
+    def __init__(self, file_name, dir_path, timestamp, op_name, op_type, task_id, anchor_type, anchor_idx):
+        super(DumpDecodeFileDesc, self).__init__(file_name, dir_path, timestamp, op_name, op_type, task_id)
+        self.type = anchor_type
+        self.idx = anchor_idx

msprobe/pytorch/parse_tool/lib/interactive_cli.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+import cmd
+import argparse
+from msprobe.pytorch.parse_tool.lib.parse_tool import ParseTool
+from msprobe.pytorch.parse_tool.lib.utils import Util
+from msprobe.pytorch.parse_tool.lib.config import Const
+from msprobe.pytorch.parse_tool.lib.parse_exception import catch_exception
+
+
+class InteractiveCli(cmd.Cmd):
+    def __init__(self):
+        super().__init__()
+        self.prompt = "Parse >>> "
+        self.parse_tool = ParseTool()
+        self.util = Util()
+        self.util.print_panel(Const.HEADER)
+        self.prepare()
+
+    @staticmethod
+    def _parse_argv(line, insert=None):
+        argv = line.split() if line != "" else []
+        if "-h" in argv:
+            return argv
+        if insert is not None and len(argv) and argv[0] != insert:
+            argv.insert(0, insert)
+        return argv
+
+    def prepare(self):
+        self.parse_tool.prepare()
+
+    @catch_exception
+    def default(self, line=""):
+        self.util.execute_command(line)
+        return False
+
+    @catch_exception
+    def do_run(self, line=""):
+        self.util.execute_command(line)
+
+    @catch_exception
+    def do_vc(self, line=""):
+        parser = argparse.ArgumentParser()
+        parser.add_argument(
+            "-m", "--my_dump_path", dest="my_dump_path", default=None,
+            help="<Required> my dump path, the data compared with golden data",
+            required=True
+        )
+        parser.add_argument(
+            "-g", "--golden_dump_path", dest="golden_dump_path", default=None,
+            help="<Required> the golden dump data path",
+            required=True
+        )
+        parser.add_argument(
+            "-out", "--output_path", dest="output_path", default=None,
+            help="<Optional> the output path",
+            required=False
+        )
+        parser.add_argument(
+            "-cmp_path", "--msaccucmp_path", dest="msaccucmp_path", default=None,
+            help="<Optional> the msaccucmp.py file path",
+            required=False
+        )
+        args = parser.parse_args(self._parse_argv(line))
+        self.util.check_path_valid(args.my_dump_path)
+        self.util.check_path_valid(args.golden_dump_path)
+        self.util.check_files_in_path(args.my_dump_path)
+        self.util.check_files_in_path(args.golden_dump_path)
+        if self.util.dir_contains_only(args.my_dump_path, ".npy") and \
+            self.util.dir_contains_only(args.golden_dump_path, ".npy"):
+            self.parse_tool.do_compare_converted_dir(args)
+        else:
+            self.parse_tool.do_vector_compare(args)
+
+    def do_dc(self, line=""):
+        self.parse_tool.do_convert_dump(self._parse_argv(line))
+
+    def do_pt(self, line=""):
+        self.parse_tool.do_print_data(self._parse_argv(line))
+
+    def do_pk(self, line=""):
+        self.parse_tool.do_parse_pkl(self._parse_argv(line))
+
+    def do_cn(self, line=''):
+        self.parse_tool.do_compare_data(self._parse_argv(line))
+
+    def do_cad(self, line=''):
+        self.parse_tool.do_convert_api_dir(self._parse_argv(line))