mindspore 2.0.0rc1__cp38-none-any.whl → 2.2.0__cp38-none-any.whl

mindspore/rewrite/symbol_tree_builder.py

@@ -13,54 +13,15 @@
 # limitations under the License.
 # ============================================================================
 """SymbolTree builder."""
-from copy import copy
 from typing import Optional
 import ast
 import inspect
 
 from mindspore.nn import Cell
 from .symbol_tree import SymbolTree
-from .node import TreeNode
-from .parser_register import ParserRegister
-from .parser import Parser
+from .parsers.parser_register import ParserRegister
+from .parsers.parser import Parser
 from .ast_transformers import FlattenRecursiveStmt
-from .ast_helpers import AstModifier
-from .ast_helpers import AstFinder
-
-
-class FunctionSymbolTreeBuilder:
-    """Create function SymbolTree"""
-    def __init__(self, network: Cell, ast_root):
-        self._origin_net = network
-        self._ast_root: ast.Module = ast_root
-        self._root_tree: Optional[SymbolTree] = None
-
-    @staticmethod
-    def _ast_transform(ast_root: ast.AST) -> ast.AST:
-        """
-        Optimize ast before parse.
-
-        Args:
-             ast_root (ast.AST): An instance of ast to be optimized.
-
-        Returns:
-             An instance of ast been optimized.
-        """
-        transform_list = [FlattenRecursiveStmt()]
-        for transformer in transform_list:
-            ast_root = transformer.transform(ast_root)
-        return ast_root
-
-    def build(self) -> SymbolTree:
-        """
-        Build SymbolTree.
-
-        Returns:
-             An instance of SymbolTree.
-        """
-        self._root_tree: SymbolTree = SymbolTree(self._origin_net, self._ast_root)
-        self._root_tree.finish_build()
-        return self._root_tree
 
 
 class SymbolTreeBuilder:
@@ -71,6 +32,9 @@ class SymbolTreeBuilder:
          network (Cell): An instance of Cell represents a network from which SymbolTree will be built.
     """
 
+    # Entry function of the forward computation process
+    entry_function = "construct"
+
     def __init__(self, network: Cell):
         if not isinstance(network, Cell):
             raise RuntimeError("Only support network with Cell type now, ", network)
@@ -78,29 +42,9 @@ class SymbolTreeBuilder:
         network_str = inspect.getsource(type(network))
         self._ast_root: ast.Module = ast.parse(network_str)
         self._root_tree: Optional[SymbolTree] = None
-        if isinstance(network, Cell) and network.jit_config_dict:
-            self._jit_config_dict = network.jit_config_dict
 
     @staticmethod
-    def merge_module_of_subtree(main_tree: SymbolTree, sub_stree: SymbolTree):
-        """
-        Merge ast.Module of sub-network into ast.Module of main-network.
-
-        1. Merge imports of ast.Module.
-        2. Merge classes of ast.Module.
-        3. Use merged ast.Module as module of main-network and sub-network.
-        """
-
-        if sub_stree.get_ori_cls_name() == "SequentialCell":
-            SymbolTreeBuilder._erase_unused_func_of_sequentialcell(sub_stree.get_class_ast())
-        father_mod = main_tree.get_module_ast()
-        sub_mod = sub_stree.get_module_ast()
-        SymbolTreeBuilder._merge_import_of_module(father_mod, sub_mod)
-        SymbolTreeBuilder._merge_class_of_module(father_mod, sub_mod)
-        sub_stree.set_module_ast(father_mod)
-
-    @staticmethod
-    def _ast_transform(ast_root: ast.AST) -> ast.AST:
+    def ast_transform(ast_root: ast.AST) -> ast.AST:
         """
         Optimize ast before parse.
 
@@ -110,117 +54,9 @@ class SymbolTreeBuilder:
         Returns:
              An instance of ast been optimized.
         """
-        transform_list = [FlattenRecursiveStmt()]
-        for transformer in transform_list:
-            ast_root = transformer.transform(ast_root)
+        ast_root = FlattenRecursiveStmt().transform(ast_root, [SymbolTreeBuilder.entry_function])
         return ast_root
 
-    @staticmethod
-    def _merge_import_of_module(main_mod: ast.Module, sub_mod: ast.Module):
-        """
-        Merge imports of ast.Module of sub-network to ast.Module of main-network.
-
-        Note:
-            Imports of sub_module would be added ahead of imports in main_module.
-
-            Error will occur if import name is over-load because of alise.
-
-        Args:
-             main_mod (ast.Module): An ast.Module corresponding to main-network.
-             sub_mod (ast.Module): An ast.Module corresponding to sub-network.
-
-        """
-
-        sub_mod_finder = AstFinder(sub_mod)
-        main_mod_finder = AstFinder(main_mod)
-        imports_in_sub = copy(sub_mod_finder.find_all((ast.Import, ast.ImportFrom)))
-        imports_in_main = copy(main_mod_finder.find_all((ast.Import, ast.ImportFrom)))
-        first_import = imports_in_main[0]
-        for clazz in imports_in_sub:
-            AstModifier.insert_sub_ast(main_mod, clazz, first_import, True)
-
-    @staticmethod
-    def _merge_class_of_module(main_mod: ast.Module, sub_mod: ast.Module):
-        """
-        Merge classes of ast.Module of sub-network to ast.Module of main-network.
-
-        Note:
-            Classes of sub_module would be added ahead of classes in main_module.
-
-        Args:
-             main_mod (ast.Module): An ast.Module corresponding to main-network.
-             sub_mod (ast.Module): An ast.Module corresponding to sub-network.
-
-        """
-
-        sub_mod_finder = AstFinder(sub_mod)
-        main_mod_finder = AstFinder(main_mod)
-        classes_in_sub = copy(sub_mod_finder.find_all(ast.ClassDef))
-        classes_in_main = copy(main_mod_finder.find_all(ast.ClassDef))
-        first_class = classes_in_main[0]
-        for clazz in classes_in_sub:
-            AstModifier.insert_class_into_module(main_mod, clazz, first_class, True)
-
-    @staticmethod
-    def _erase_unused_func_of_sequentialcell(ast_class: ast.ClassDef):
-        func_names = ("__getitem__", "__setitem__", "__delitem__", "__len__", "append")
-        for name in func_names:
-            AstModifier.erase_func_from_class_by_name(ast_class, name)
-
-    def _merge_module_of_subtrees(self):
-        """
-        Merge ast.Module of all sub-networks into ast.Module of main-network.
-
-        1. Merge imports of ast.Module.
-        2. Merge classes of ast.Module.
-        3. Use merged ast.Module as module of main-network and sub-network.
-        """
-
-        for node in self._root_tree.nodes():
-            if isinstance(node, TreeNode) and node.get_instance():
-                SymbolTreeBuilder.merge_module_of_subtree(self._root_tree, node.symbol_tree)
-
-    def _reduce_redundant_import(self):
-        """
-        Reduce redundant imports of ast.Module.
-
-        Redundant imports may be introduced into ast.Module while merging ast.Module of sub-network to main-network.
-        """
-
-        module: ast.Module = self._root_tree.get_module_ast()
-        import_list = []
-        exist_import = []
-        exist_import_from = []
-        for body in module.body:
-            if isinstance(body, ast.Import):
-                names = body.names
-                for name in names:
-                    import_hash = hash((name.name, name.asname))
-                    if import_hash in exist_import:
-                        continue
-                    exist_import.append(import_hash)
-                    import_list.append(ast.Import(names=[ast.alias(name=name.name, asname=name.asname)]))
-            if isinstance(body, ast.ImportFrom):
-                import_module = body.module
-                names = body.names
-                for name in names:
-                    import_hash = hash((import_module, name.name, name.asname))
-                    if import_hash in exist_import_from:
-                        continue
-                    exist_import_from.append(import_hash)
-                    import_list.append(ast.ImportFrom(module=import_module,
-                                                      names=[ast.alias(name=name.name, asname=name.asname)],
-                                                      level=0))
-        insert_pos = None
-        for i in range(len(module.body) - 1, -1, -1):
-            body = module.body[i]
-            if not isinstance(body, (ast.Import, ast.ImportFrom)):
-                insert_pos = body
-                continue
-            module.body.pop(i)
-        for import_ast in import_list:
-            AstModifier.insert_sub_ast(module, import_ast, insert_pos, True)
-
     def build(self) -> SymbolTree:
         """
         Build SymbolTree.
@@ -229,14 +65,12 @@ class SymbolTreeBuilder:
              An instance of SymbolTree.
         """
 
-        self._ast_root = SymbolTreeBuilder._ast_transform(self._ast_root)
+        self._ast_root = SymbolTreeBuilder.ast_transform(self._ast_root)
         if not isinstance(self._ast_root, ast.Module):
             raise RuntimeError("ast_root should be a ast.Module")
         self._root_tree: SymbolTree = SymbolTree(self._origin_net, self._ast_root)
         parser: Parser = ParserRegister.instance().get_parser(ast.Module)
-        parser.process(self._root_tree, self._ast_root)
-        self._merge_module_of_subtrees()
-        self._reduce_redundant_import()
+        parser.process(self._root_tree, self._ast_root, None)
         ast.fix_missing_locations(self._root_tree.get_module_ast())
         self._root_tree.finish_build()
         return self._root_tree

mindspore/rewrite/symbol_tree_dumper.py

@@ -15,7 +15,7 @@
 """SymbolTree dumper."""
 import inspect
 
-from .node import Node
+from .node.node import Node
 from .api.node_type import NodeType
 from .api.scoped_value import ScopedValue, ValueType
 
@@ -69,7 +69,7 @@ class SymbolTreeDumper:
         self._dump_buffer += f"Symbol Tree @construct {{ \n"
         node_no = -1
 
-        node: Node = self._symbol_tree.get_head_node().get_next()
+        node: Node = self._symbol_tree.get_head().get_next()
         while node is not None:
             if node.get_node_type() is NodeType.Output:
                 self._dump_buffer += f"  Return(%{node_no}) \n"

mindspore/run_check/_check_version.py

@@ -73,7 +73,7 @@ class GPUEnvChecker(EnvChecker):
 
     def __init__(self, library_path):
         self.version = ["10.1", "11.1", "11.6"]
-        self.lib_key_to_lib_name = {'libcu': 'libcuda.so', 'libcudnn': 'libcudnn.so'}
+        self.lib_key_to_lib_name = {'libcudart': 'libcuda.so', 'libcudnn': 'libcudnn.so'}
         self.library_path = library_path
         # env
         self.path = os.getenv("PATH")
@@ -81,7 +81,7 @@ class GPUEnvChecker(EnvChecker):
 
         # check
         self.v = "0"
-        self.cuda_lib_path = self._get_lib_path("libcu")
+        self.cuda_lib_path = self._get_lib_path("libcudart")
         self.cuda_bin_path = self._get_bin_path("cuda")
         self.cudnn_lib_path = self._get_lib_path("libcudnn")
 
@@ -124,6 +124,9 @@ class GPUEnvChecker(EnvChecker):
         """Get cuda runtime version by libcudart.so."""
         for path in self.cuda_lib_path:
             real_path = glob.glob(path + "/lib*/libcudart.so.*.*.*")
+            # /usr/lib/x86_64-linux-gnu is a default dir for cuda10.1 on ubuntu.
+            if real_path == []:
+                real_path = glob.glob(path + "/x86_64-linux-gnu/libcudart.so.*.*.*")
             if real_path == []:
                 continue
             ls_cudart = subprocess.run(["ls", real_path[0]], timeout=10, text=True,
@@ -144,6 +147,7 @@ class GPUEnvChecker(EnvChecker):
             os.environ['LD_LIBRARY_PATH'] = akg_dir + ":" + os.environ['LD_LIBRARY_PATH']
         else:
             os.environ['LD_LIBRARY_PATH'] = akg_dir
+        os.environ['CUDA_CACHE_MAXSIZE'] = "4000000000"
 
     def _get_bin_path(self, bin_name):
         """Get bin path by bin name."""
@@ -255,7 +259,7 @@ class AscendEnvChecker(EnvChecker):
 
     def __init__(self, library_path):
         self.library_path = library_path
-        self.version = ["6.11", "6.3"]
+        self.version = ["7.0"]
         atlas_nnae_version = "/usr/local/Ascend/nnae/latest/compiler/version.info"
         atlas_toolkit_version = "/usr/local/Ascend/ascend-toolkit/latest/compiler/version.info"
         hisi_fwk_version = "/usr/local/Ascend/latest/compiler/version.info"
@@ -311,6 +315,31 @@ class AscendEnvChecker(EnvChecker):
         self.ascend_opp_path_check = "/op"
         self.v = ""
 
+    @staticmethod
+    def _concat_variable(env_name, env_value):
+        if os.getenv(env_name) is None:
+            os.environ[env_name] = env_value
+        else:
+            os.environ[env_name] = env_value + ":" + os.environ[env_name]
+
+    @staticmethod
+    def _check_and_set_env(env_name, env_value, is_append=False):
+        """check and set environment variable by its name"""
+        if env_value is None:
+            return True
+
+        if not Path(env_value).is_dir():
+            logger.error(
+                f"No such directory: {env_value}. Please check if Ascend AI software package (Ascend Data Center"
+                " Solution) is installed correctly.")
+            return False
+
+        if is_append:
+            AscendEnvChecker._concat_variable(env_name, env_value)
+        else:
+            os.environ[env_name] = env_value
+        return True
+
     def check_env(self):
         self._check_env()
 
@@ -356,7 +385,7 @@ class AscendEnvChecker(EnvChecker):
         # DO NOT modify exception type to any other, you DO NOT know what kind of exceptions the te will throw.
         # pylint: disable=broad-except
         except Exception as e:
-            logger.error("CheckFailed:", e.args)
+            logger.error(f"CheckFailed: {e}")
             logger.error("MindSpore relies on whl packages of \"te\" and \"hccl\" in the \"latest\" "
                          "folder of the Ascend AI software package (Ascend Data Center Solution). Please check whether"
                          " they are installed correctly or not, refer to the match info on: "
@@ -368,12 +397,15 @@ class AscendEnvChecker(EnvChecker):
                 time.sleep(1)
 
     def set_env(self):
+        curr_path = os.path.abspath(os.path.dirname(__file__))
+        if os.getenv('ASCEND_CUSTOM_OPP_PATH'):
+            os.environ['ASCEND_CUSTOM_OPP_PATH'] = os.environ['ASCEND_CUSTOM_OPP_PATH'] + ":" + \
+                                                   curr_path + "/../lib/plugin/ascend/custom_aicpu_ops"
+        else:
+            os.environ['ASCEND_CUSTOM_OPP_PATH'] = curr_path + "/../lib/plugin/ascend/custom_aicpu_ops"
         plugin_dir = os.path.dirname(self.library_path)
         akg_dir = os.path.join(plugin_dir, "ascend")
-        if os.getenv('LD_LIBRARY_PATH'):
-            os.environ['LD_LIBRARY_PATH'] = akg_dir + ":" + os.environ['LD_LIBRARY_PATH']
-        else:
-            os.environ['LD_LIBRARY_PATH'] = akg_dir
+        AscendEnvChecker._concat_variable('LD_LIBRARY_PATH', akg_dir)
 
         if not self.tbe_path:
             self._check_env()
@@ -398,10 +430,7 @@ class AscendEnvChecker(EnvChecker):
 
         if Path(self.op_impl_path).is_dir():
             # python path for sub process
-            if os.getenv('PYTHONPATH'):
-                os.environ['PYTHONPATH'] = self.op_impl_path + ":" + os.environ['PYTHONPATH']
-            else:
-                os.environ['PYTHONPATH'] = self.op_impl_path
+            AscendEnvChecker._concat_variable('PYTHONPATH', self.op_impl_path)
             # sys path for this process
             sys.path.append(self.op_impl_path)
 
@@ -412,32 +441,11 @@ class AscendEnvChecker(EnvChecker):
                 "Center Solution) is installed correctly.")
             return
 
-        if Path(self.cce_path).is_dir():
-            os.environ['PATH'] = self.cce_path + ":" + os.environ['PATH']
-        else:
-            logger.error(
-                f"No such directory: {self.cce_path}. Please check if Ascend AI software package (Ascend Data Center "
-                "Solution) is installed correctly.")
+        if not AscendEnvChecker._check_and_set_env('PATH', self.cce_path, True):
             return
-
-        if self.op_path is None:
-            pass
-        elif Path(self.op_path).is_dir():
-            os.environ['ASCEND_OPP_PATH'] = self.op_path
-        else:
-            logger.error(
-                f"No such directory: {self.op_path}. Please check if Ascend AI software package (Ascend Data Center "
-                "Solution) is installed correctly.")
+        if not AscendEnvChecker._check_and_set_env('ASCEND_OPP_PATH', self.op_path):
             return
-
-        if self.aicpu_path is None:
-            pass
-        elif Path(self.aicpu_path).is_dir():
-            os.environ['ASCEND_AICPU_PATH'] = self.aicpu_path
-        else:
-            logger.error(
-                f"No such directory: {self.aicpu_path}. Please check if Ascend AI software package (Ascend Data Center"
-                " Solution) is installed correctly.")
+        if not AscendEnvChecker._check_and_set_env('ASCEND_AICPU_PATH', self.aicpu_path):
             return
 
     def _check_env(self):
@@ -523,13 +531,12 @@ def check_version_and_env_config():
         except OSError:
             logger.warning("Pre-Load Library libgomp.so.1 failed, which might cause TLS memory allocation failure. If "
                            "the failure occurs, please refer to the FAQ for a solution: "
-                           "https://www.mindspore.cn/docs/en/r2.0/faq/installation.html.")
+                           "https://www.mindspore.cn/docs/en/r2.2/faq/installation.html.")
         if not os.getenv("MS_DEV_CLOSE_VERSION_CHECK") is None:
             return
         MSContext.get_instance().register_check_env_callback(check_env)
         MSContext.get_instance().register_set_env_callback(set_env)
-        MSContext.get_instance().set_param(ms_ctx_param.device_target,
-                                           MSContext.get_instance().get_param(ms_ctx_param.device_target))
+        MSContext.get_instance().set_device_target_inner(MSContext.get_instance().get_param(ms_ctx_param.device_target))
 
 
 def _set_pb_env():

mindspore/run_check/run_check.py

@@ -47,13 +47,14 @@ def _check_mul():
 def run_check():
     """
     Provide a convenient API to check if the installation is successful or failed.
-    If there is no return value, the verification status will be displayed directly.
+    If the version in the check result is not what you expect, use :func:`mindspore.set_context` to set device_target
+    before run_check().
 
     Examples:
         >>> import mindspore
         >>> mindspore.run_check()
         MindSpore version: xxx
-        The result of multiplication calculation is correct, MindSpore has been installed successfully!
+        The result of multiplication calculation is correct, MindSpore has been installed on platform [XX] successfully!
     """
     try:
         _check_mul()

mindspore/{scipy/sparse → safeguard}/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,8 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============================================================================
-"""Sparse submodule"""
-from . import linalg
-from .linalg import cg
+"""MindSpore Safeguard package."""
+from .rewrite_obfuscation import obfuscate_ckpt, load_obf_params_into_net
 
-__all__ = ["cg"]
+__all__ = ['obfuscate_ckpt', 'load_obf_params_into_net']