mindstudio-probe 8.3.3__py3-none-any.whl → 26.0.0a1__py3-none-any.whl

msprobe/core/common/file_utils.py

@@ -1,17 +1,18 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
-# All rights reserved.
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
 #
-# Licensed under the Apache License, Version 2.0  (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#          http://license.coscl.org.cn/MulanPSL2
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
 
 import atexit
 import csv
@@ -85,7 +86,7 @@ class FileChecker:
         """
         check_path_exists(self.file_path)
         check_link(self.file_path)
-        self.file_path = os.path.realpath(self.file_path)
+        self.file_path = os.path.realpath(os.path.expanduser(self.file_path))
         check_path_length(self.file_path)
         check_path_type(self.file_path, self.path_type)
         self.check_path_ability()
@@ -252,13 +253,23 @@ def check_common_file_size(file_path):
             if file_path.endswith(suffix):
                 check_file_size(file_path, max_size)
                 return
-        check_file_size(file_path, FileCheckConst.COMMOM_FILE_SIZE)
+        check_file_size(file_path, FileCheckConst.MAX_COMMON_FILE_SIZE)
 
 
 def check_file_suffix(file_path, file_suffix):
+    # 如果 file_suffix 为 None，直接返回
+    if file_suffix is None:
+        return
+
+    # 如果 file_suffix 是字符串，则将其转化为列表
+    if isinstance(file_suffix, str):
+        file_suffix = [file_suffix]
+    elif not isinstance(file_suffix, (list, tuple)):
+        raise TypeError("file_suffix must be a string, list, or tuple.")
+
     if file_suffix:
-        if not file_path.endswith(file_suffix):
-            logger.error(f"The {file_path} should be a {file_suffix} file!")
+        if not any(file_path.endswith(suffix) for suffix in file_suffix):
+            logger.error(f"{file_path} should be one of the following types of files: {file_suffix}!")
             raise FileCheckException(FileCheckException.INVALID_FILE_ERROR)
 
 
@@ -287,17 +298,20 @@ def check_others_writable(file_path):
 
 def make_dir(dir_path):
     check_path_before_create(dir_path)
-    dir_path = os.path.realpath(dir_path)
+    dir_path = os.path.abspath(os.path.expanduser(dir_path))
+    dir_checker = FileChecker(dir_path, FileCheckConst.DIR)
     if os.path.isdir(dir_path):
+        dir_checker.common_check()
         return
     try:
+        parent_dir_checker = FileChecker(os.path.dirname(dir_path), FileCheckConst.DIR)
+        parent_dir_checker.common_check()
         os.makedirs(dir_path, mode=FileCheckConst.DATA_DIR_AUTHORITY, exist_ok=True)
     except OSError as ex:
         raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR,
                                  f"Failed to create {dir_path}. "
                                  f"Please check the path permission or disk space. {str(ex)}") from ex
-    file_check = FileChecker(dir_path, FileCheckConst.DIR)
-    file_check.common_check()
+    dir_checker.common_check()
 
 
 @recursion_depth_decorator('msprobe.core.common.file_utils.create_directory', max_depth=16)
@@ -310,21 +324,21 @@ def create_directory(dir_path):
     Exception Description:
         when invalid data throw exception
     """
-    check_link(dir_path)
     check_path_before_create(dir_path)
-    dir_path = os.path.realpath(dir_path)
-    parent_dir = os.path.dirname(dir_path)
+    abs_path = os.path.abspath(os.path.expanduser(dir_path))
+    parent_dir = os.path.dirname(abs_path)
     if not os.path.isdir(parent_dir):
         create_directory(parent_dir)
-    make_dir(dir_path)
+    make_dir(abs_path)
 
 
 def check_path_before_create(path):
     check_link(path)
+    path = os.path.realpath(os.path.expanduser(path))
     if path_len_exceeds_limit(path):
         raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR, 'The file path length exceeds limit.')
 
-    if not re.match(FileCheckConst.FILE_PATTERN, os.path.realpath(path)):
+    if not re.match(FileCheckConst.FILE_PATTERN, path):
         raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR,
                                  'The file path {} contains special characters.'.format(path))
 
@@ -341,7 +355,7 @@ def check_dirpath_permission(path):
             logger.warning(f"The directory {dirpath} is not yours.")
 
 
-def check_file_or_directory_path(path, isdir=False, is_strict=False):
+def check_file_or_directory_path(path, isdir=False, is_strict=False, file_suffix=None):
     """
     Function Description:
         check whether the path is valid
@@ -355,7 +369,7 @@ def check_file_or_directory_path(path, isdir=False, is_strict=False):
     if isdir:
         path_checker = FileChecker(path, FileCheckConst.DIR, FileCheckConst.WRITE_ABLE)
     else:
-        path_checker = FileChecker(path, FileCheckConst.FILE, FileCheckConst.READ_ABLE)
+        path_checker = FileChecker(path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, file_suffix)
     path_checker.common_check()
 
     if is_strict:
@@ -386,6 +400,49 @@ def check_path_no_group_others_write(file_path):
         )
 
 
+def check_if_valid_dir_pattern_path(path):
+    if os.path.isfile(path):
+        logger.error(f'The path {path} should be a directory path, but got a file')
+        raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR)
+
+
+def find_existing_path(path, depth=16):
+    path = os.path.abspath(os.path.expanduser(path))
+    if os.path.exists(path):
+        return path
+    if depth <= 0:
+        raise RecursionError("Output path was not valid")
+    parent_path = os.path.dirname(path)
+    # 递归查找父目录
+    if parent_path and parent_path != path:
+        return find_existing_path(parent_path, depth - 1)
+    else:
+        raise ValueError("Output path was not valid.")
+
+
+def check_and_get_real_path(path, ability, file_type=None, must_exist=True, is_strict=False):
+    ori_path = path
+    if ability == FileCheckConst.READ_ABLE or ability == FileCheckConst.EXECUTE_ABLE:
+        must_exist = True
+    if not must_exist:
+        path = find_existing_path(path)
+    if file_type is not None:
+        path_type = FileCheckConst.FILE
+    else:
+        path_type = FileCheckConst.DIR if os.path.isdir(path) else FileCheckConst.FILE
+
+    file_check = FileChecker(path, path_type, ability=ability, file_type=file_type)
+    file_check.common_check()
+
+    if is_strict and check_group_writable(file_check.file_path):
+        raise FileCheckException(
+            FileCheckException.FILE_PERMISSION_ERROR,
+            f"The directory must not allow write access to group. Directory path: {path}"
+        )
+
+    return os.path.realpath(os.path.expanduser(ori_path))
+
+
 def change_mode(path, mode):
     if not os.path.exists(path) or os.path.islink(path):
         return
@@ -695,6 +752,15 @@ def write_csv(data, filepath, mode="a+", malicious_check=False):
     change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY)
 
 
+def save_numpy_to_bin(numpy_data, saved_path):
+    try:
+        numpy_data.tofile(saved_path)
+    except Exception as e:
+        logger.error(f"Save numpy data to {saved_path} failed")
+        raise RuntimeError from e
+    change_mode(saved_path, FileCheckConst.DATA_FILE_AUTHORITY)
+
+
 def read_csv(filepath, as_pd=True, header='infer'):
     check_file_or_directory_path(filepath)
     try:
@@ -954,6 +1020,16 @@ def dedup_log(func_name, filter_name):
         return True
 
 
+def check_output_dir_path(path):
+    path = os.path.realpath(path)
+    check_link(path)
+    check_path_length(path)
+    check_path_pattern_valid(path)
+    if os.path.exists(path):
+        check_path_writability(path)
+        check_path_owner_consistent(path)
+
+
 class SharedDict:
     def __init__(self):
         self._changed = False

msprobe/core/common/framework_adapter.py

@@ -1,24 +1,23 @@
-# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd.
-# All rights reserved.
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
 #
-# Licensed under the Apache License, Version 2.0  (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#          http://license.coscl.org.cn/MulanPSL2
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.import functools
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
 
 import functools
-
 from msprobe.core.common.const import Const
-from msprobe.core.common.file_utils import check_file_or_directory_path, save_npy, DeserializationScanner
-from msprobe.core.common.log import logger
-from msprobe.core.common.utils import confirm
+from msprobe.core.common.file_utils import check_file_or_directory_path
+from msprobe.core.common.file_utils import save_npy
 
 
 class FrameworkDescriptor:
@@ -106,7 +105,7 @@ class FmkAdp:
     @classmethod
     def tensor_norm(cls, tensor):
         return cls.process_tensor(tensor, lambda x: x.norm())
-
+    
     @classmethod
     def save_tensor(cls, tensor, filepath):
         if cls.fmk == Const.PT_FRAMEWORK:
@@ -131,6 +130,25 @@ class FmkAdp:
             raise Exception(f"{module} is not a mindspore.nn.Cell")
         return module.parameters_and_names()
 
+    @classmethod
+    def iter_named_modules(cls, model):
+        """
+        Unified module iterator across frameworks.
+
+        Returns an iterable of (name, module) pairs using framework-specific
+        traversal APIs:
+        - PyTorch: model.named_modules()
+        - MindSpore: model.cells_and_names()
+        """
+        if cls.fmk == Const.PT_FRAMEWORK:
+            if not hasattr(model, "named_modules"):
+                raise Exception(f"{model} does not implement named_modules()")
+            return model.named_modules()
+        # MindSpore
+        if not hasattr(model, "cells_and_names"):
+            raise Exception(f"{model} does not implement cells_and_names()")
+        return model.cells_and_names()
+
     @classmethod
     def register_forward_pre_hook(cls, module, hook, with_kwargs=False):
         if cls.fmk == Const.PT_FRAMEWORK:
@@ -157,13 +175,6 @@ class FmkAdp:
         check_file_or_directory_path(path, is_strict=not weights_only)
         if cls.fmk == Const.PT_FRAMEWORK:
             try:
-                if not weights_only:
-                    if not DeserializationScanner.scan_pickle_content(path):
-                        if not confirm(
-                                f"Some insecure methods or modules are detected in {path}, "
-                                f"input yes to ignore and continue, otherwise exit", False):
-                            logger.error("Insecure risks found and exit!")
-                            raise Exception("Insecure risks found and exit!")
                 if to_cpu:
                     return cls.framework.load(path, map_location=cls.framework.device("cpu"), weights_only=weights_only)
                 else:
@@ -171,9 +182,9 @@ class FmkAdp:
             except Exception as e:
                 raise RuntimeError(f"load pt file {path} failed: {e}") from e
         return mindspore.load_checkpoint(path)
-
+    
     @classmethod
     def asnumpy(cls, tensor):
         if cls.fmk == Const.PT_FRAMEWORK:
             return tensor.float().numpy()
-        return tensor.float().asnumpy()
+        return tensor.float().asnumpy()

msprobe/core/common/global_lock.py

@@ -1,17 +1,18 @@
-# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd.
-# All rights reserved.
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
 #
-# Licensed under the Apache License, Version 2.0  (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#          http://license.coscl.org.cn/MulanPSL2
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
 
 import multiprocessing
 from multiprocessing.shared_memory import SharedMemory

msprobe/core/common/inplace_op_checker.py

@@ -1,17 +1,18 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
-# All rights reserved.
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
 #
-# Licensed under the Apache License, Version 2.0  (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#          http://license.coscl.org.cn/MulanPSL2
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
 
 import os
 from msprobe.core.common.file_utils import load_yaml

msprobe/core/common/log.py

@@ -1,17 +1,18 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
-# All rights reserved.
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
 #
-# Licensed under the Apache License, Version 2.0  (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#          http://license.coscl.org.cn/MulanPSL2
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
 
 import os
 import time
@@ -43,6 +44,16 @@ class BaseLogger:
         else:
             return int(input_level)
 
+    @filter_special_chars
+    def raw(self, msg, **kwargs):
+        """
+        直接输出原始内容，不添加日志前缀
+        用于输出子进程的原始日志，避免双份前缀
+        """
+        msg = msg.rstrip("_")
+        print(msg)
+        sys.stdout.flush()
+
     def get_rank(self):
         return self.rank