mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/rewrite/parsers/function_def_parser.py

@@ -15,11 +15,10 @@
 """Parse bodies of ast.FunctionDef which is construct function to nodes of SymbolTree."""
 import ast
 from mindspore import log as logger
-from .parser_register import ParserRegister, reg_parser
-from .parser import Parser
+from . import Parser, ParserRegister, reg_parser
 from ..symbol_tree import SymbolTree
+from ..node import NodeManager
 from ..api.node_type import NodeType
-from ..node.node_manager import NodeManager
 
 
 class FunctionDefParser(Parser):
@@ -62,8 +61,17 @@ class FunctionDefParser(Parser):
         parser.process(stree, arguments, node_manager)
 
         # parse body as node of stree
-        for body in ast_node.body:
-            # avoid add dead code, so we need to break if return is added.
+        for body in ast_node.body[:]:
+            # delete the comment
+            if isinstance(body, ast.Expr) and \
+                (isinstance(body.value, ast.Str) or (isinstance(body.value, ast.Constant) and \
+                                                     isinstance(body.value.value, str))):
+                ast_node.body.remove(body)
+                continue
+            # closure syntax is not currently supported
+            if isinstance(body, (ast.FunctionDef, ast.ClassDef)):
+                stree.append_python_node(ast_node, body, node_manager)
+                continue
             parser: Parser = ParserRegister.instance().get_parser(type(body))
             if parser is None:
                 stree.append_python_node(ast_node, body, node_manager)

mindspore/rewrite/parsers/if_parser.py

@@ -15,12 +15,10 @@
 """Parse ast.If in construct function to node of SymbolTree."""
 
 import ast
-
-from ..symbol_tree import SymbolTree
-from .parser import Parser
-from .parser_register import ParserRegister, reg_parser
+from . import Parser, ParserRegister, reg_parser
 from ..node import NodeManager, ControlFlow
-from ..ast_transformers.flatten_recursive_stmt import FlattenRecursiveStmt
+from ..symbol_tree import SymbolTree
+from ..ast_helpers import AstFinder, AstConverter, AstFlattener
 
 
 class IfParser(Parser):
@@ -43,25 +41,45 @@ class IfParser(Parser):
             NotImplementedError: If test of ast.If can not be eval.
         """
         # expand codes in ast.if
-        ast_if = FlattenRecursiveStmt().transform_if(node, stree)
+        ast_if = AstFlattener().transform_control_flow(node, stree)
         # parse ast codes of if branch into ControlFlow Node
-        if_node = ControlFlow("if_node", ast_if.body, stree)
+        args = [AstConverter.create_scopedvalue(node.test)]
+        if_node = ControlFlow("if_node", ast_if, False, args, stree)
+        stree.append_origin_field(if_node, node_manager)
         for body in ast_if.body:
             parser: Parser = ParserRegister.instance().get_parser(type(body))
             if parser is None:
                 stree.append_python_node(ast_if, body, node_manager=if_node)
             else:
                 parser.process(stree, body, node_manager=if_node)
-        stree.append_origin_field(if_node, node_manager)
         # parse ast codes of else branch into ControlFlow Node
+        else_node = None
         if ast_if.orelse:
-            else_node = ControlFlow("else_node", ast_if.orelse, stree)
+            else_node = ControlFlow("else_node", ast_if, True, args, stree)
+            stree.append_origin_field(else_node, node_manager)
             for body in ast_if.orelse:
                 parser: Parser = ParserRegister.instance().get_parser(type(body))
                 if parser is None:
                     stree.append_python_node(ast_if, body, node_manager=else_node)
                 else:
                     parser.process(stree, body, node_manager=else_node)
-            stree.append_origin_field(else_node, node_manager)
+            else_node.set_body_node(if_node)
+            if_node.set_orelse_node(else_node)
+        # record eval result of ast.If's test
+        if ast_if in AstFlattener.ast_if_test_cache:
+            origin_test_ast = AstFlattener.ast_if_test_cache[ast_if]
+            # replace self.xxx to self._origin_network.xxx
+            ast_attributes = AstFinder(origin_test_ast).find_all(ast.Attribute)
+            for ast_attr in ast_attributes:
+                if isinstance(ast_attr.value, ast.Name) and ast_attr.value.id == 'self':
+                    new_ast_attr = ast.Attribute(value=ast.Name(id='self', ctx=ast.Load()),
+                                                 attr='_origin_network', ctx=ast.Load())
+                    ast_attr.value = new_ast_attr
+            # get result of ast.If's test
+            eval_success, eval_result = stree.eval_ast_result(origin_test_ast)
+            if eval_success:
+                if_node.test_result = eval_result
+                if else_node:
+                    else_node.test_result = eval_result
 
 g_if_parser = reg_parser(IfParser())

mindspore/rewrite/parsers/module_parser.py

@@ -17,16 +17,14 @@ import sys
 from typing import Any
 import os
 import ast
-import copy
 import inspect
 
 from mindspore import log as logger
+from . import Parser, ParserRegister, reg_parser
+from ..node import NodeManager
 from ..symbol_tree import SymbolTree
-from .parser import Parser
-from .parser_register import ParserRegister, reg_parser
 from ..ast_helpers import AstFinder
-from ..common import error_str
-from ..node.node_manager import NodeManager
+from ..common.error_log import error_str
 
 if sys.version_info >= (3, 9):
     import ast as astunparse # pylint: disable=reimported, ungrouped-imports
@@ -50,187 +48,16 @@ class ModuleParser(Parser):
             raise RuntimeError(error_str("multi-class in module is not supported now", father_node=ast_node))
         return classes[0]
 
-    @staticmethod
-    def _get_import_node(ast_root):
-        """Iterate over ast_root and return all ast.Import nodes or ast.ImportFrom nodes in ast_root."""
-        import_nodes = []
-        try_nodes = []
-        imports_str = []
-
-        class GetImportNode(ast.NodeVisitor):
-            """Find all import nodes from input ast node."""
-
-            def visit_Try(self, node: ast.Try) -> Any:
-                if isinstance(node.body[0], (ast.Import, ast.ImportFrom)):
-                    try_nodes.append(copy.deepcopy(node))
-                return node
-
-            def visit_Import(self, node: ast.Import) -> Any:
-                """Iterate over all nodes and save ast.Import nodes."""
-                import_nodes.append(copy.deepcopy(node))
-                imports_str.append(astunparse.unparse(node))
-                return node
-
-            def visit_ImportFrom(self, node: ast.ImportFrom) -> Any:
-                """Iterate over all nodes and save ast.ImportFrom nodes."""
-                import_nodes.append(copy.deepcopy(node))
-                imports_str.append(astunparse.unparse(node))
-                return node
-
-            def get_node(self, input_ast):
-                """Interface of GetImportNode."""
-                self.generic_visit(input_ast)
-                return True
-
-        def _remove_duplicated_import_in_try(node: [ast.Import, ast.ImportFrom]):
-            import_str = astunparse.unparse(node)
-            if import_str in imports_str:
-                import_nodes.remove(import_nodes[imports_str.index(import_str)])
-
-        get_node_handler = GetImportNode()
-        get_node_handler.get_node(ast_root)
-        for Try in try_nodes:
-            for body in Try.body:
-                _remove_duplicated_import_in_try(body)
-            for handler in Try.handlers:
-                for body in handler.body:
-                    _remove_duplicated_import_in_try(body)
-        import_nodes.extend(try_nodes)
-        return import_nodes
-
-    @staticmethod
-    def save_file_path_to_sys(stree, level_num, file_path):
-        """
-        Save file path into stree._import_asts. `level_num` is used when level exist in ast.ImportFrom.
-
-        When level_num = 0(e.g. from xxx import yyy), current path will be saved.
-        When level_num = 1(e.g. from .xxx import yyy), current path will be saved.
-        When level_num = 2(e.g. from ..xxx import yyy), the path one level above the current path will be saved.
-        """
-        file_path = os.path.dirname(os.path.abspath(file_path))
-        if level_num > 1:
-            for _ in range(level_num - 1):
-                file_path = os.path.dirname(file_path)
-        sys_path_append_ast = ast.parse(f"sys.path.insert(0, r'{file_path}')").body[0]
-        stree.get_import_asts().append(ast.Import([ast.alias(name='sys', asname=None)]))
-        stree.get_import_asts().append(sys_path_append_ast)
-
-    @staticmethod
-    def _save_imports(stree):
-        """Insert two groups of import nodes to ast.Module, common ones and those from class definition file."""
-        stree.get_import_asts().append(ast.Import([ast.alias(name='mindspore', asname=None)]))
-        stree.get_import_asts().append(ast.ImportFrom(module='mindspore', names=[ast.alias(name='nn', asname=None)],
-                                                      level=0))
-        stree.get_import_asts().append(ast.ImportFrom(module='mindspore.nn',
-                                                      names=[ast.alias(name='Cell', asname=None)], level=0))
-        stree.get_import_asts().append(ast.ImportFrom(module='mindspore.ops',
-                                                      names=[ast.alias(name='functional', asname='F')], level=0))
-        origin_net = stree.get_origin_network()
-        net_path = inspect.getfile(type(origin_net))
-        ModuleParser.save_file_path_to_sys(stree, 0, net_path)
-        ModuleParser.save_imports_from_file(stree, net_path)
-
-    @staticmethod
-    def get_valid_import_info(import_node, file_path):
-        """Get valid import info while import_node.module is at form of relative path"""
-        # copy to a new node to avoid origin import_node being modified.
-        import_node_test = copy.deepcopy(import_node)
-        file_path = os.path.dirname(os.path.abspath(file_path))
-        # get real path from import_node.level
-        # from .(A) import xxx: current path
-        # from ..(A) import xxx: last level path
-        import_node_module_name = import_node.module
-        level = import_node.level
-        # from A import xxx: it does not need to pad, directly return the module name
-        if level == 0:
-            return import_node_module_name, None
-        if level > 1:
-            for _ in range(level - 1):
-                file_path = os.path.dirname(file_path)
-        file_path_tmp = file_path[:]
-        max_level_count = file_path.count('/') + file_path.count('\\') - 1
-        level_count = 0
-        # suffix is the module_name, e.g. 'A' in 'from ..(A) import xxx'
-        suffix = ''
-        if import_node_module_name:
-            suffix = '.' + import_node_module_name
-        while level_count < max_level_count:
-            file_path_tmp = os.path.dirname(file_path_tmp)
-            import_node_test.module = file_path[len(file_path_tmp) + 1:].replace('/', '.') + suffix
-            import_node_test.level = 0
-            import_code = astunparse.unparse(import_node_test).strip()
-            test_code = f"import sys\nsys.path.insert(0, r'{file_path_tmp}')\n{import_code}"
-            try:
-                exec(test_code) # pylint: disable=W0122
-            except (ValueError, ImportError) as e:
-                # try upper level to avoid ValueError: attempted relative import beyond top-level package
-                # this exception is changed to ImportError after python3.9
-                logger.info(f"For MindSpore Rewrite, in module parser, test import code: "
-                            f"{import_code} failed: {e}. Try upper level.")
-                level_count += 1
-                continue
-            except Exception as e: # pylint: disable=W0703
-                logger.info(f"For MindSpore Rewrite, in module parser, process import code: "
-                            f"{import_code} failed: {e}. Ignore this import code.")
-                return None, None
-            else:
-                # try test code success
-                return import_node_test.module, file_path_tmp
-        # try codes with all level failed
-        logger.info(f"For MindSpore Rewrite, in module parser, test import code: "
-                    f"{astunparse.unparse(import_node).strip()} failed. Ignore this import code.")
-        return None, None
-
-    @staticmethod
-    def save_imports_from_file(stree, file_path):
-        """Save imports from file"""
-        if not os.path.exists(file_path):
-            raise RuntimeError(f"For MindSpore Rewrite, in module parser, file {file_path} not exist.")
-        try:
-            with open(file_path, "r", encoding="utf-8") as f:
-                source_code = f.read()
-                import_nodes = ModuleParser._get_import_node(ast.parse(source_code))
-        except RuntimeError as err:
-            raise RuntimeError(f"For MindSpore Rewrite, in module parser, get import nodes error: {err}")
-        if not import_nodes:
-            return
-        for import_node in import_nodes:
-            import_node = ModuleParser._process_relative_import(stree, import_node, file_path)
-            if import_node:
-                stree.get_import_asts().append(import_node)
-
-    @staticmethod
-    def _process_relative_import(stree, import_node, file_path):
-        """Process relative imports"""
-        if isinstance(import_node, ast.ImportFrom):
-            # pad the ImportFrom with parent path
-            # e.g. from ..C import xxx -> from A.B.C import xxx
-            import_module, import_path = ModuleParser.get_valid_import_info(import_node, file_path)
-            if import_path:
-                ModuleParser.save_file_path_to_sys(stree, 0, import_path)
-            module_name_list = [alias.name.strip() for alias in import_node.names]
-            # add the module into _imported_modules to direct the class
-            stree.save_imported_modules(file_path, import_module, module_name_list)
-            import_node = ast.ImportFrom(module=import_module, names=import_node.names, level=0)
-        elif isinstance(import_node, ast.Import):
-            for alias in import_node.names:
-                name = alias.name
-                stree.save_imported_modules(file_path, name.strip(), [])
-        return import_node
-
     @staticmethod
     def _add_decorator_to_class(class_ast: ast.ClassDef, origin_net):
         """Add decorators to class"""
         origin_net_source_code_file = inspect.getfile(type(origin_net))
         if not os.path.exists(origin_net_source_code_file):
-            raise RuntimeError("For MindSpore Rewrite, in module parser, File ", origin_net_source_code_file,
-                               " not exist")
-        try:
-            with open(origin_net_source_code_file, "r", encoding="utf-8") as f:
-                source_code = f.read()
-                decorators = ModuleParser._get_decorator(ast.parse(source_code), origin_net)
-        except RuntimeError:
-            raise RuntimeError("For MindSpore Rewrite, in module parser, get decorators error")
+            raise FileNotFoundError(f"For MindSpore Rewrite, in module parser, File {origin_net_source_code_file} "
+                                    f"not exist")
+        with open(origin_net_source_code_file, "r", encoding="utf-8") as f:
+            source_code = f.read()
+            decorators = ModuleParser._get_decorator(ast.parse(source_code), origin_net)
         if decorators:
             for decorator_index, decorator_node in enumerate(decorators):
                 class_ast.decorator_list.insert(decorator_index, decorator_node)
@@ -276,7 +103,6 @@ class ModuleParser(Parser):
 
     def process(self, stree: SymbolTree, node: ast.Module, node_manager: NodeManager):
         """Process ast.ClassDef nodes in ast.Module."""
-        ModuleParser._save_imports(stree)
         class_ast = ModuleParser._find_class(node)
         ModuleParser._add_decorator_to_class(class_ast, stree.get_origin_network())
         stree.set_class_ast(class_ast)

mindspore/rewrite/parsers/parser.py

@@ -14,10 +14,6 @@
 # ============================================================================
 """Base class of parser."""
 import abc
-import ast
-
-from ..symbol_tree import SymbolTree
-from ..node.node_manager import NodeManager
 
 
 class Parser(abc.ABC):
@@ -35,7 +31,7 @@ class Parser(abc.ABC):
         return type(None)
 
     @abc.abstractmethod
-    def process(self, stree: SymbolTree, node: ast.AST, node_manager: NodeManager):
+    def process(self, stree, node, node_manager):
         """
         Parse input ast node and add parse result into SymbolTree.
 

mindspore/rewrite/parsers/parser_register.py

@@ -15,7 +15,7 @@
 """Parser register."""
 from typing import Optional
 
-from .parser import Parser
+from . import Parser
 
 
 class ParserRegister:

mindspore/rewrite/parsers/return_parser.py

@@ -15,12 +15,10 @@
 """Parse ast.Return output-node of SymbolTree."""
 import ast
 
+from . import Parser, reg_parser
 from ..symbol_tree import SymbolTree
-from ..node.node import Node
-from ..node.node_manager import NodeManager
-from .parser import Parser
-from .parser_register import reg_parser
-from ..common import error_str
+from ..node import Node, NodeManager
+from ..ast_helpers import AstConverter
 
 
 class ReturnParser(Parser):
@@ -32,11 +30,8 @@ class ReturnParser(Parser):
 
     def process(self, stree: SymbolTree, node: ast.Return, node_manager: NodeManager):
         """Parse ast.Return to output-node of SymbolTree."""
-        return_value = node.value
-        if not isinstance(return_value, ast.Name):
-            raise RuntimeError(error_str(f"only support ast.Name as return value, but got ast type "
-                                         f"'{type(return_value).__name__}'", father_node=node, child_node=return_value))
-        node_return = Node.create_output_node(node, [return_value.id])
+        return_scoped_value = AstConverter.create_scopedvalue(node.value)
+        node_return = Node.create_output_node(node, [return_scoped_value])
         stree.append_origin_field(node_return, node_manager)
 
 g_return_parser = reg_parser(ReturnParser())

mindspore/rewrite/parsers/while_parser.py

@@ -0,0 +1,59 @@
+# Copyright 2022 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+""" Parse ast.While node """
+import ast
+
+from . import Parser, ParserRegister, reg_parser
+from ..node import NodeManager, ControlFlow
+from ..symbol_tree import SymbolTree
+from ..ast_helpers import AstConverter, AstFlattener
+
+
+class WhileParser(Parser):
+    """ Class that implements parsing ast.While nodes """
+
+    def target(self):
+        """Parse target type"""
+        return ast.While
+
+    def process(self, stree: SymbolTree, node: ast.While, node_manager: NodeManager):
+        """ Process ast.While node """
+        # expand codes in ast.While
+        ast_while = AstFlattener().transform_control_flow(node, stree)
+        # parse ast codes of for branch into ControlFlow Node
+        args = [AstConverter.create_scopedvalue(node.test)]
+        while_node = ControlFlow("while_node", ast_while, False, args, stree)
+        stree.append_origin_field(while_node, node_manager)
+        while_node.set_node_manager(node_manager)
+        for body in ast_while.body:
+            parser: Parser = ParserRegister.instance().get_parser(type(body))
+            if parser is None:
+                stree.append_python_node(ast_while, body, node_manager=while_node)
+            else:
+                parser.process(stree, body, node_manager=while_node)
+        # parse ast codes of else branch into ControlFlow Node
+        if ast_while.orelse:
+            while_else_node = ControlFlow("while_else_node", ast_while, True, args, stree)
+            stree.append_origin_field(while_else_node, node_manager)
+            for body in ast_while.orelse:
+                parser: Parser = ParserRegister.instance().get_parser(type(body))
+                if parser is None:
+                    stree.append_python_node(ast_while, body, node_manager=while_else_node)
+                else:
+                    parser.process(stree, body, node_manager=while_else_node)
+            while_else_node.set_body_node(while_node)
+            while_node.set_orelse_node(while_else_node)
+
+g_while_parser = reg_parser(WhileParser())

mindspore/rewrite/sparsify/utils.py

@@ -22,7 +22,7 @@ from typing import NamedTuple, Any, Optional, Callable, Union
 
 import mindspore
 from mindspore import ops, Tensor, CSRTensor, COOTensor
-from mindspore.rewrite.namespace import get_functional
+from mindspore.rewrite.common.namespace import get_functional
 
 
 class ArgType(Enum):

mindspore/rewrite/symbol_tree/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+Definitions and methods of SymbolTree.
+"""
+from .symbol_tree import SymbolTree, Position
+from .symbol_tree_builder import SymbolTreeBuilder
+from .symbol_tree_dumper import SymbolTreeDumper