mindstudio-probe 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

msprobe/pytorch/parse_tool/lib/compare.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,16 +12,17 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
 
 import os
 import time
-import numpy as np
 from collections import namedtuple
-from msprobe.pytorch.parse_tool.lib.utils import Util
+
+import numpy as np
+
+from msprobe.core.common.file_utils import create_directory, load_npy, save_npy_to_txt, write_csv, os_walk_for_files
 from msprobe.pytorch.parse_tool.lib.config import Const
 from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException
-from msprobe.core.common.file_utils import create_directory, load_npy, save_npy_to_txt, write_csv
+from msprobe.pytorch.parse_tool.lib.utils import Util
 
 
 class Compare:
@@ -126,7 +126,7 @@ class Compare:
         all_close = np.allclose(data_left, data_right, atol=al, rtol=rl)
         np.seterr(divide='raise')
         cos_sim = np.dot(data_left, data_right) / (
-            np.sqrt(np.dot(data_left, data_left)) * np.sqrt(np.dot(data_right, data_right)))
+                np.sqrt(np.dot(data_left, data_left)) * np.sqrt(np.dot(data_right, data_right)))
         err_cnt = 0
         total_cnt = data_left.shape[0]
         diff_table_columns = ['Index', 'Left', 'Right', 'Diff']
@@ -240,20 +240,14 @@ class Compare:
 
     def convert_api_dir_to_npy(self, dump_dir, param, output_dir, msaccucmp_path):
         dump_dir = self.util.path_strip(dump_dir)
-        for root, _, files in os.walk(dump_dir, topdown=True):
-            self.util.check_path_valid(root)
-            for file in files:
-                file_path = os.path.join(root, file)
-                file_name = os.path.basename(file_path)
-                parts = file_name.split(".")
-                if len(parts) < 5:
-                    continue
-                op_name = parts[1]
-                timestamp = parts[-1]
-                output_path = os.path.join(output_dir, op_name, timestamp)
-                self.convert_dump_to_npy(file_path, param, output_path, msaccucmp_path)
-            path_depth = root.count(os.sep)
-            if path_depth <= Const.MAX_TRAVERSAL_DEPTH:
-                yield root, _, files
-            else:
-                _[:] = []
+        files = os_walk_for_files(dump_dir, Const.MAX_TRAVERSAL_DEPTH)
+        filepaths = [os.path.join(file['root'], file['file']) for file in files]
+        for path in filepaths:
+            filename = os.path.basename(path)
+            parts = filename.split(".")
+            if len(parts) < 5:
+                continue
+            op_name = parts[1]
+            timestamp = parts[-1]
+            output_path = os.path.join(output_dir, op_name, timestamp)
+            self.convert_dump_to_npy(path, param, output_path, msaccucmp_path)

msprobe/pytorch/parse_tool/lib/config.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,14 +12,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
 
 import os
+
 import numpy as np
 
 
 class Const:
-
     MS_ACCU_CMP_PATH = '/usr/local/Ascend/ascend-toolkit/latest/tools/operator_cmp/compare/msaccucmp.py'
     MS_ACCU_CMP_FILE_NAME = 'msaccucmp.py'
     ROOT_DIR = ""
@@ -33,7 +31,7 @@ class Const:
     OFFLINE_DUMP_CONVERT_PATTERN = \
         r"^([A-Za-z0-9_-]+)\.([A-Za-z0-9_-]+)\.([0-9]+)(\.[0-9]+)?\.([0-9]{1,255})" \
         r"\.([a-z]+)\.([0-9]{1,255})(\.[x0-9]+)?\.npy$"
-    NUMPY_PATTERN = r"^[\w\-_-]\.npy$"
+    NUMPY_PATTERN = r"^[\w\-_.]+\.npy$"
     NPY_SUFFIX = ".npy"
     PKL_SUFFIX = ".pkl"
     DIRECTORY_LENGTH = 4096

msprobe/pytorch/parse_tool/lib/file_desc.py

@@ -1,4 +1,18 @@
-# coding=utf-8
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
 
 

msprobe/pytorch/parse_tool/lib/interactive_cli.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,13 +12,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
-import cmd
+
 import argparse
-from msprobe.pytorch.parse_tool.lib.parse_tool import ParseTool
-from msprobe.pytorch.parse_tool.lib.utils import Util
+import cmd
+
 from msprobe.pytorch.parse_tool.lib.config import Const
 from msprobe.pytorch.parse_tool.lib.parse_exception import catch_exception
+from msprobe.pytorch.parse_tool.lib.parse_tool import ParseTool
+from msprobe.pytorch.parse_tool.lib.utils import Util
 
 
 class InteractiveCli(cmd.Cmd):
@@ -81,7 +81,7 @@ class InteractiveCli(cmd.Cmd):
         self.util.check_files_in_path(args.my_dump_path)
         self.util.check_files_in_path(args.golden_dump_path)
         if self.util.dir_contains_only(args.my_dump_path, ".npy") and \
-            self.util.dir_contains_only(args.golden_dump_path, ".npy"):
+                self.util.dir_contains_only(args.golden_dump_path, ".npy"):
             self.parse_tool.do_compare_converted_dir(args)
         else:
             self.parse_tool.do_vector_compare(args)

msprobe/pytorch/parse_tool/lib/parse_exception.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,13 +12,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
+
 import logging
+
 from msprobe.core.common.exceptions import FileCheckException
 
 
 class ParseException(Exception):
-
     PARSE_INVALID_PATH_ERROR = 0
     PARSE_NO_FILE_ERROR = 1
     PARSE_NO_MODULE_ERROR = 2
@@ -51,4 +50,5 @@ def catch_exception(func):
         except FileCheckException:
             log.error("Command execution failed")
         return result
+
     return inner

msprobe/pytorch/parse_tool/lib/parse_tool.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,17 +12,18 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
+
 import argparse
 import os
 from collections import namedtuple
 
+from msprobe.core.common.file_utils import create_directory
+from msprobe.pytorch.parse_tool.lib.compare import Compare
 from msprobe.pytorch.parse_tool.lib.config import Const
+from msprobe.pytorch.parse_tool.lib.parse_exception import catch_exception, ParseException
 from msprobe.pytorch.parse_tool.lib.utils import Util
-from msprobe.pytorch.parse_tool.lib.compare import Compare
 from msprobe.pytorch.parse_tool.lib.visualization import Visualization
-from msprobe.pytorch.parse_tool.lib.parse_exception import catch_exception, ParseException
-from msprobe.core.common.file_utils import create_directory
+
 
 class ParseTool:
     def __init__(self):
@@ -117,7 +117,8 @@ class ParseTool:
         self.util.check_path_valid(args.golden_dump_path)
         self.util.check_file_path_format(args.my_dump_path, Const.NPY_SUFFIX)
         self.util.check_file_path_format(args.golden_dump_path, Const.NPY_SUFFIX)
-        compare_data_args = namedtuple('compare_data_args', ['my_dump_path', 'golden_dump_path', 'save', 'rtol', 'atol', 'count'])
+        compare_data_args = namedtuple('compare_data_args',
+                                       ['my_dump_path', 'golden_dump_path', 'save', 'rtol', 'atol', 'count'])
         compare_data_args.__new__.__defaults__ = (False, 0.001, 0.001, 20)
         res = compare_data_args(args.my_dump_path, args.golden_dump_path, args.save, args.rtol, args.atol, args.count)
         self.compare.compare_data(res)
@@ -132,8 +133,7 @@ class ParseTool:
                                 " '-m' and '-g'.")
             raise ParseException("My directory path and golden directory path is same.")
         output_path = self.util.path_strip(args.output_path) if args.output_path else Const.BATCH_COMPARE_DIR
-        if not os.path.isdir(output_path):
-            os.makedirs(output_path, mode=0o750)
+        create_directory(output_path)
         self.compare.compare_converted_dir(my_dump_dir, golden_dump_dir, output_path)
 
     @catch_exception

msprobe/pytorch/parse_tool/lib/utils.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,24 +12,24 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
+
+import hashlib
 import os
 import re
-import sys
 import subprocess
-import hashlib
+import sys
 import time
-import numpy as np
 from collections import namedtuple
+
+import numpy as np
+from msprobe.core.common.const import FileCheckConst
+from msprobe.core.common.file_utils import change_mode, check_other_user_writable, \
+    check_path_executable, check_path_owner_consistent
+from msprobe.core.common.file_utils import check_file_or_directory_path, remove_path, check_file_type, os_walk_for_files
+from msprobe.pytorch.common.log import logger
 from msprobe.pytorch.parse_tool.lib.config import Const
 from msprobe.pytorch.parse_tool.lib.file_desc import DumpDecodeFileDesc, FileDesc
 from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException
-from msprobe.core.common.file_utils import change_mode, check_other_user_writable,\
-    check_path_executable, check_path_owner_consistent
-from msprobe.core.common.const import FileCheckConst
-from msprobe.core.common.file_utils import check_file_or_directory_path, remove_path, check_file_type
-from msprobe.pytorch.common.log import logger
-
 
 try:
     from rich.traceback import install
@@ -81,16 +80,8 @@ class Util:
 
     @staticmethod
     def get_subfiles_count(directory):
-        file_count = 0
-        for root, _, files in os.walk(directory, topdown=True):
-            check_file_or_directory_path(root, isdir=True)
-            file_count += len(files)
-            path_depth = root.count(os.sep)
-            if path_depth <= Const.MAX_TRAVERSAL_DEPTH:
-                yield root, _, files
-            else:
-                _[:] = []
-        return file_count
+        files = os_walk_for_files(directory, Const.MAX_TRAVERSAL_DEPTH)
+        return len(files)
 
     @staticmethod
     def get_sorted_subdirectories_names(directory):
@@ -143,25 +134,19 @@ class Util:
             zero_mask = (data == 0)
             data[zero_mask] += np.finfo(float).eps
         return data
-    
+
     @staticmethod
     def dir_contains_only(path, endfix):
-        for root, _, files in os.walk(path, topdown=True):
-            check_file_or_directory_path(root, isdir=True)
-            for file in files:
-                if not file.endswith(endfix):
-                    return False
-            path_depth = root.count(os.sep)
-            if path_depth <= Const.MAX_TRAVERSAL_DEPTH:
-                yield root, _, files
-            else:
-                _[:] = []
+        files = os_walk_for_files(path, Const.MAX_TRAVERSAL_DEPTH)
+        for file in files:
+            if not file['file'].endswith(endfix):
+                return False
         return True
-    
+
     @staticmethod
     def localtime_str():
         return time.strftime("%Y%m%d%H%M%S", time.localtime())
-    
+
     @staticmethod
     def change_filemode_safe(path):
         change_mode(path, FileCheckConst.DATA_FILE_AUTHORITY)
@@ -222,7 +207,7 @@ class Util:
 
     def list_numpy_files(self, path, extern_pattern=''):
         return self.list_file_with_pattern(path, Const.NUMPY_PATTERN, extern_pattern,
-                                            self._gen_numpy_file_info)
+                                           self._gen_numpy_file_info)
 
     def create_columns(self, content):
         if not Columns:
@@ -273,20 +258,15 @@ class Util:
         self.check_path_valid(path)
         file_list = {}
         re_pattern = re.compile(pattern)
-        for dir_path, _, file_names in os.walk(path, topdown=True):
-            check_file_or_directory_path(dir_path, isdir=True)
-            for name in file_names:
-                match = re_pattern.match(name)
-                if not match:
-                    continue
-                if extern_pattern != '' and re_pattern.match(extern_pattern) and not re.match(extern_pattern, name):
-                    continue
-                file_list[name] = gen_info_func(name, match, dir_path)
-            path_depth = dir_path.count(os.sep)
-            if path_depth <= Const.MAX_TRAVERSAL_DEPTH:
-                yield dir_path, _, file_names
-            else:
-                _[:] = []
+        files = os_walk_for_files(path, Const.MAX_TRAVERSAL_DEPTH)
+        for file in files:
+            name = file["file"]
+            match = re_pattern.match(name)
+            if not match:
+                continue
+            if extern_pattern != '' and re_pattern.match(extern_pattern) and not re.match(extern_pattern, name):
+                continue
+            file_list[name] = gen_info_func(name, match, file["root"])
         return file_list
 
     def check_file_path_format(self, path, suffix):

msprobe/pytorch/parse_tool/lib/visualization.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,14 +12,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
+
 import json
-import numpy as np
 
+import numpy as np
+from msprobe.core.common.file_utils import FileOpen, load_npy, save_npy_to_txt
 from msprobe.pytorch.parse_tool.lib.config import Const
-from msprobe.pytorch.parse_tool.lib.utils import Util
 from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException
-from msprobe.core.common.file_utils import FileOpen, load_npy, save_npy_to_txt
+from msprobe.pytorch.parse_tool.lib.utils import Util
 
 
 class Visualization:
@@ -65,6 +64,8 @@ class Visualization:
                     self.util.log.error("%s %s in line %s" % ("JSONDecodeError", str(e), pkl_line))
                     self.util.log.warning("Please check the pkl file")
                     raise ParseException(ParseException.PARSE_JSONDECODE_ERROR) from e
+                if not isinstance(msg, list) or len(msg) == 0:
+                    break
                 info_prefix = msg[0]
                 if not info_prefix.startswith(api_name):
                     continue
@@ -75,7 +76,7 @@ class Visualization:
                             self.util.log.info("  File \"{}\", line {}, in {}".format(item[0], item[1], item[2]))
                             self.util.log.info("    {}".format(item[3]))
                         continue
-                if len(msg) > 5 and len(msg[5]) >=  3:
+                if len(msg) > 5 and len(msg[5]) >= 3:
                     summery_info = "  [{}][dtype: {}][shape: {}][max: {}][min: {}][mean: {}]" \
                         .format(msg[0], msg[3], msg[4], msg[5][0], msg[5][1], msg[5][2])
                     if not title_printed:

msprobe/pytorch/pt_config.py

@@ -14,11 +14,13 @@
 # limitations under the License.
 
 import os
+import re
 
 from msprobe.core.common.const import Const
 from msprobe.core.common.exceptions import MsprobeException
-from msprobe.core.common.file_utils import FileOpen, load_json
+from msprobe.core.common.file_utils import FileOpen, load_json, check_file_or_directory_path, check_crt_valid
 from msprobe.core.common.log import logger
+from msprobe.core.common.utils import is_int
 from msprobe.core.common_config import BaseConfig, CommonConfig
 from msprobe.core.grad_probe.constant import level_adp
 from msprobe.core.grad_probe.utils import check_bounds
@@ -38,17 +40,38 @@ class TensorConfig(BaseConfig):
         self.host = json_config.get("host", "")
         self.port = json_config.get("port", -1)
         self.tls_path = json_config.get("tls_path", "./")
+        self.online_run_ut_recompute = json_config.get("online_run_ut_recompute", False)
         self.check_config()
         self._check_file_format()
-        self._check_tls_path_config()
+        if self.online_run_ut:
+            self._check_online_run_ut()
 
     def _check_file_format(self):
         if self.file_format is not None and self.file_format not in ["npy", "bin"]:
             raise Exception("file_format is invalid")
 
-    def _check_tls_path_config(self):
-        if self.tls_path and not os.path.exists(self.tls_path):
-            raise Exception("tls_path: %s does not exist" % self.tls_path)
+    def _check_online_run_ut(self):
+        if not isinstance(self.online_run_ut, bool):
+            raise Exception(f"online_run_ut: {self.online_run_ut} is invalid.")
+
+        if not isinstance(self.online_run_ut_recompute, bool):
+            raise Exception(f"online_run_ut_recompute: {self.online_run_ut_recompute} is invalid.")
+
+        if self.nfs_path:
+            check_file_or_directory_path(self.nfs_path, isdir=True)
+            return
+
+        if self.tls_path:
+            check_file_or_directory_path(self.tls_path, isdir=True)
+            check_file_or_directory_path(os.path.join(self.tls_path, "client.key"))
+            check_file_or_directory_path(os.path.join(self.tls_path, "client.crt"))
+            check_crt_valid(os.path.join(self.tls_path, "client.crt"))
+
+        if not isinstance(self.host, str) or not re.match(Const.ipv4_pattern, self.host):
+            raise Exception(f"host: {self.host} is invalid.")
+
+        if not isinstance(self.port, int) or not (0 < self.port <= 65535):
+            raise Exception(f"port: {self.port} is invalid, port range 0-65535.")
 
 
 class StatisticsConfig(BaseConfig):
@@ -70,7 +93,7 @@ class OverflowCheckConfig(BaseConfig):
         self.check_overflow_config()
 
     def check_overflow_config(self):
-        if self.overflow_nums is not None and not isinstance(self.overflow_nums, int):
+        if self.overflow_nums is not None and not is_int(self.overflow_nums):
             raise Exception("overflow_num is invalid")
         if self.check_mode is not None and self.check_mode not in ["all", "aicore", "atomic"]:
             raise Exception("check_mode is invalid")
@@ -170,7 +193,7 @@ class FreeBenchmarkCheckConfig(BaseConfig):
             )
 
     def _check_preheat_config(self):
-        if not isinstance(self.preheat_step, int):
+        if not is_int(self.preheat_step):
             msg = "preheat_step is invalid, it should be an integer"
             logger.error_log_with_exp(
                 msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg)
@@ -180,7 +203,7 @@ class FreeBenchmarkCheckConfig(BaseConfig):
             logger.error_log_with_exp(
                 msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg)
             )
-        if not isinstance(self.max_sample, int):
+        if not is_int(self.max_sample):
             msg = "max_sample is invalid, it should be an integer"
             logger.error_log_with_exp(
                 msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg)