mindspore 2.1.0__cp38-none-any.whl → 2.2.11__cp38-none-any.whl

mindspore/_extends/parse/__init__.py

@@ -23,13 +23,13 @@ from .parser import (Parser, create_instance, is_supported_create_instance_type,
                      get_ast_type, get_node_type, get_args, get_args_default_values, get_ast_namespace_symbol,
                      get_operation_symbol, get_operation_namespace_symbol, get_parse_method_of_class, get_scope_name,
                      eval_script, get_script_id_attrs, expand_expr_statement, is_class_member_of_self, parse_cb,
-                     convert_to_ms_tensor, get_object_description, get_class_attr_namespace_symbol, get_ms_class_name,
-                     is_class_type, check_obj_bool, python_isinstance, ms_isinstance, convert_to_ms_csrtensor,
-                     convert_to_ms_cootensor, convert_class_to_function, convert_cell_list_to_sequence, is_cell_list,
-                     get_obj_from_sequence, get_type, is_class_member_recursive, merge_global_params, get_global_params,
-                     get_adapter_tensor_attr, get_obj_defined_from_obj_type,
-                     is_from_third_party_library, get_const_abs, get_const_round,
-                     get_const_len, is_adapter_tensor_class, is_adapter_parameter_class)
+                     convert_to_ms_tensor, get_object_description, get_ms_class_name, is_class_type, check_obj_bool,
+                     python_isinstance, ms_isinstance, convert_to_ms_csrtensor, convert_to_ms_cootensor,
+                     convert_class_to_function, convert_cell_list_to_sequence, is_cell_list, get_obj_from_sequence,
+                     get_type, is_class_member_recursive, get_global_params, get_adapter_tensor_attr,
+                     get_obj_defined_from_obj_type, is_from_third_party_library, get_const_abs, get_const_round,
+                     get_const_len, is_adapter_tensor_class, is_adapter_parameter_class, convert_to_namedtuple,
+                     is_module_list)
 
 __all__ = ['Parser', 'create_instance', 'is_supported_create_instance_type', 'generate_scope', 'get_attr_from_object',
            'get_bprop_method_of_class', 'get_class_instance_type', 'get_class_member_namespace_symbol',
@@ -37,11 +37,9 @@ __all__ = ['Parser', 'create_instance', 'is_supported_create_instance_type', 'ge
            'get_ast_type', 'get_node_type', 'get_args', 'get_args_default_values', 'get_ast_namespace_symbol',
            'get_operation_symbol', 'get_operation_namespace_symbol', 'get_parse_method_of_class', 'get_scope_name',
            'eval_script', 'get_script_id_attrs', 'expand_expr_statement', 'is_class_member_of_self', 'parse_cb',
-           'resolve_symbol', 'convert_to_ms_tensor', 'get_object_description', 'get_class_attr_namespace_symbol',
-           'get_ms_class_name', 'is_class_type', 'check_obj_bool', 'python_isinstance', 'ms_isinstance',
-           'convert_to_ms_csrtensor', 'convert_to_ms_cootensor', 'convert_class_to_function',
-           'convert_cell_list_to_sequence', 'is_cell_list', 'get_obj_from_sequence', 'get_type',
-           'is_class_member_recursive', 'get_adapter_tensor_attr',
-           'get_obj_defined_from_obj_type', 'is_from_third_party_library',
-           'get_const_abs', 'get_const_round', 'get_const_len', 'is_adapter_tensor_class',
-           'is_adapter_parameter_class',]
+           'resolve_symbol', 'convert_to_ms_tensor', 'get_object_description', 'get_ms_class_name', 'is_class_type',
+           'check_obj_bool', 'python_isinstance', 'ms_isinstance', 'convert_to_ms_csrtensor', 'convert_to_ms_cootensor',
+           'convert_class_to_function', 'convert_cell_list_to_sequence', 'is_cell_list', 'get_obj_from_sequence',
+           'get_type', 'is_class_member_recursive', 'get_adapter_tensor_attr', 'get_obj_defined_from_obj_type',
+           'is_from_third_party_library', 'get_const_abs', 'get_const_round', 'get_const_len',
+           'is_adapter_tensor_class', 'is_adapter_parameter_class', 'convert_to_namedtuple', 'is_module_list']

mindspore/_extends/parse/namespace.py

@@ -1,6 +1,6 @@
 # This is the Python adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/).
 #
-# Copyright 2020-2022 Huawei Technologies Co., Ltd
+# Copyright 2020-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -30,6 +30,7 @@ class Namespace:
         name (str): The namespace's name.
         dicts (dict): A list of dict containing the namespace's variable.
     """
+
     def __init__(self, name, *dicts):
         self.name = name
         self.dicts = dicts
@@ -57,6 +58,7 @@ class CellNamespace(Namespace):
     Args:
         name (str): Valid module name, it can be imported.
     """
+
     def __init__(self, name):
         mod_dict = vars(__import__(name, fromlist=['_']))
         builtins_dict = vars(builtins)
@@ -79,6 +81,7 @@ class ClosureNamespace(Namespace):
     Args:
         fn (Function): A python function.
     """
+
     def __init__(self, fn):
         name = f'{fn.__module__}..<{fn.__name__}>'
         names = fn.__code__.co_freevars
@@ -101,6 +104,7 @@ class ClassMemberNamespace(Namespace):
     Args:
         obj (Object): A python class object.
     """
+
     def __init__(self, obj):
         self.__class_member_namespace__ = True
         label = f'{obj.__module__}..<{obj.__class__.__name__}::{id(obj)}>'
@@ -108,8 +112,6 @@ class ClassMemberNamespace(Namespace):
 
     def __getitem__(self, name):
         d, = self.dicts
-        if name == "self":
-            return d
         if name == "namespace":
             return self
         try:
@@ -127,36 +129,8 @@ class ClassMemberNamespace(Namespace):
                     logger.warning(f"The private attribute or method '{name}' is used in '{cls.__name__}'. " + \
                                    f"In graph mode, '{name}' will be adjusted to '{private_member}' for parsing.")
                     return getattr(d, private_member)
-            logger.info(f"'{cls.__name__ }' object has no attribute or method: '{name}', so will return None.")
+            logger.info(f"'{cls.__name__}' object has no attribute or method: '{name}', so will return None.")
             raise AttributeError(name)
 
-
-class ClassAttrNamespace(Namespace):
-    """
-    Namespace of a class.
-
-    Args:
-        obj (Object): A python class object.
-    """
-    def __init__(self, obj):
-        name = f'{obj.__module__}..<{obj.__class__.__name__}::{id(obj)}>'
-        super().__init__(name, obj)
-
     def __getattr__(self, name):
-        d, = self.dicts
-        try:
-            if hasattr(d, name):
-                return getattr(d, name)
-            return d.__dict__[name]
-        except ValueError:
-            raise UnboundLocalError(name)
-        except KeyError:
-            # Class private attribute.
-            cls = d.__class__
-            if name.startswith("__"):
-                private_attr = "_" + cls.__name__ + name
-                if hasattr(d, private_attr):
-                    logger.warning(f"The private attribute or method '{name}' is used in '{cls.__name__}'. " + \
-                                   f"In graph mode, '{name}' will be adjusted to '{private_attr}' for parsing.")
-                    return getattr(d, private_attr)
-            raise AttributeError(f"'{cls.__name__ }' object has no attribute or method: '{name}'.")
+        return self.__getitem__(name)

mindspore/_extends/parse/parser.py

@@ -24,6 +24,8 @@ import re
 import hashlib
 import inspect
 import types
+from collections import namedtuple
+from typing import NamedTuple
 from textwrap import dedent
 import numpy
 
@@ -42,7 +44,7 @@ from mindspore.common import mutable
 from mindspore.common._register_for_adapter import ms_adapter_registry
 from mindspore._checkparam import is_stub_tensor
 from mindspore.ops._tracefunc import _PackSourceBuilder
-from .namespace import Namespace, CellNamespace, ClosureNamespace, ClassMemberNamespace, ClassAttrNamespace
+from .namespace import Namespace, CellNamespace, ClosureNamespace, ClassMemberNamespace
 from .resources import parse_object_map, ops_symbol_map, convert_object_map, convert_class_to_function_map, trope_ns
 from .resources import SYMBOL_UNDEFINE
 from ...common.api import _convert_python_data
@@ -121,43 +123,15 @@ _hybrid_type = (
 
 # Unsupported python builtin type in JIT Fallback.
 _fallback_unsupported_python_builtin_type = (
-    compile, eval, exec, input, open, delattr, setattr, super, staticmethod, classmethod, __import__,
-    memoryview, property,
+    compile, eval, exec
 )
 
 _modules_from_mindspore = (
-    "mindspore",           # https://gitee.com/mindspore/mindspore
-    "msadapter",           # https://openi.pcl.ac.cn/OpenI/MSAdapter
-    "mindocr",             # https://github.com/mindspore-lab/mindocr
-    "mindyolo",            # https://github.com/mindspore-lab/mindyolo
-    "mindnlp",             # https://github.com/mindspore-lab/mindnlp
-    "mindcv",              # https://github.com/mindspore-lab/mindcv
-    "mindspore_rec",       # https://github.com/mindspore-lab/mindrec
-    "mindaudio",           # https://github.com/mindspore-lab/mindaudio
-    "mindone",             # https://github.com/mindspore-lab/mindone
-    "mindspore_rl",        # https://github.com/mindspore-lab/mindrl
-    "mindformers",         # https://github.com/mindspore-lab/mindformers
-    "mindpet",             # https://github.com/mindspore-lab/mindpet
-    "mindpose",            # https://github.com/mindspore-lab/mindpose
-    "mindface",            # https://github.com/mindspore-lab/mindface
-    "mindsearch",          # https://github.com/mindspore-lab/mindsearch
-    "mindinsight",         # https://gitee.com/mindspore/mindinsight
-    "mindelec",            # https://gitee.com/mindspore/mindscience
-    "mindflow",            # https://gitee.com/mindspore/mindscience
-    "mindsponge",          # https://gitee.com/mindspore/mindscience
-    "mindearth",           # https://gitee.com/mindspore/mindscience
-    "sciai",               # https://gitee.com/mindspore/mindscience
-    "mindquantum",         # https://gitee.com/mindspore/mindquantum
-    "mindarmour",          # https://gitee.com/mindspore/mindarmour
-    "mindpandas",          # https://gitee.com/mindspore/mindpandas
-    "mindvision",          # https://gitee.com/mindspore/vision
-    "mindspore_gl",        # https://gitee.com/mindspore/graphlearning
-    "mindspore_federated", # https://gitee.com/mindspore/federated
-    "mindspore_gs",        # https://gitee.com/mindspore/golden-stick
-    "mindspore_serving",   # https://gitee.com/mindspore/serving
-    "mindspore_xai",       # https://gitee.com/mindspore/xai
-    "mindspore_hub",       # https://gitee.com/mindspore/hub
-    "ringmo_framework",    # https://gitee.com/mindspore/ringmo-framework
+    "mindspore", "msadapter", "mindocr", "mindyolo", "mindnlp", "mindcv", "mindspore_rec", "mindaudio", "mindone",
+    "mindspore_rl", "mindformers", "mindpet", "mindpose", "mindface", "mindsearch", "mindinsight", "mindelec",
+    "mindflow", "mindsponge", "mindearth", "sciai", "mindquantum", "mindarmour", "mindpandas", "mindvision",
+    "mindspore_gl", "mindspore_federated", "mindspore_gs", "mindspore_serving", "mindspore_xai", "mindspore_hub",
+    "ringmo_framework", "troubleshooter", "mindtorch",
 )
 
 _global_params = {}
@@ -190,6 +164,7 @@ def get_attr_from_object(obj, attr_name=None):
     Returns:
         Object, obj's attr.
     """
+
     if obj is not None and attr_name is not None and hasattr(obj, attr_name):
         return getattr(obj, attr_name)
     return None
@@ -206,6 +181,7 @@ def get_parse_method_of_class(obj, parse_method=None):
     Returns:
         Function, obj's method.
     """
+
     method_name = None
     if parse_method is not None:
         method_name = parse_method
@@ -229,6 +205,7 @@ def get_bprop_method_of_class(obj, parse_method=None):
     Returns:
         Function, obj's method.
     """
+
     if isinstance(obj, nn.Cell):
         method_name = "bprop"
         return get_attr_from_object(obj, method_name)
@@ -475,11 +452,11 @@ def create_instance(cls_type, params=None):
 def convert_class_to_function(cls_str, cls_obj):
     """Convert class to function."""
     if issubclass(cls_obj, (Parameter, ops.MultitypeFuncGraph)):
-        raise ValueError(f"Failed to compile in GRAPH_MODE because creating {cls_str} instances is not " \
-                         f"supported in 'construct' or @jit decorated function. Try to create {cls_str} " \
-                         f"instances external such as initialized in the method '__init__' before assigning. " \
-                         f"For more details, please refer to " \
-                         f"https://www.mindspore.cn/docs/zh-CN/r2.1/design/dynamic_graph_and_static_graph.html \n")
+        raise ValueError(f"Failed to compile in GRAPH_MODE because creating {cls_str} instances is not "
+                         f"supported in 'construct' or @jit decorated function. Try to create {cls_str} "
+                         f"instances external such as initialized in the method '__init__' before assigning. "
+                         f"For more details, please refer to "
+                         f"https://www.mindspore.cn/docs/zh-CN/r2.2/design/dynamic_graph_and_static_graph.html \n")
     return convert_class_to_function_map.get(cls_str)
 
 
@@ -516,9 +493,14 @@ def is_cell_list(obj):
     return isinstance(obj, nn.CellList)
 
 
+def is_module_list(obj):
+    """Check if obj is nn.ModuleList"""
+    return hasattr(obj, "__cell_as_list__") and not isinstance(obj, nn.CellList)
+
+
 def convert_cell_list_to_sequence(obj):
     """Convert nn.CellList to sequence."""
-    if not isinstance(obj, nn.CellList):
+    if not hasattr(obj, "__cell_as_list__"):
         raise TypeError(f"Obj should be nn.CellList, but got {obj}")
     if not hasattr(obj, "_cells"):
         raise AttributeError(f"nn.CellList is missing _cells property.")
@@ -545,14 +527,6 @@ def get_module_namespace(obj):
     return mod_namespace
 
 
-def get_class_attr_namespace_symbol(obj):
-    """Get class namespace."""
-    logger.debug("get class namespace, object: %r", obj)
-    class_namespace = ClassAttrNamespace(obj)
-    logger.debug("class namespace: %r", class_namespace)
-    return class_namespace
-
-
 def get_class_member_namespace_symbol(obj):
     """Get obj class member type."""
     logger.debug("get class instance namespace, object: %r", obj)
@@ -583,6 +557,7 @@ def get_obj_defined_from_obj_type(obj_type):
         "List": list,
         "Tuple": tuple,
         "Dictionary": dict,
+        "NamedTuple": NamedTuple,
     }
 
     return obj_type_defined_map.get(obj_type)
@@ -637,6 +612,12 @@ def convert_to_ms_cootensor(data):
     return COOTensor(coo_tensor=data)
 
 
+def convert_to_namedtuple(type_name, key_sequeue, value_sequeue):
+    """Convert C++ namedtuple to python object namedtuple."""
+    logger.debug(f"type_name: {type_name}, key_sequeue: {key_sequeue}, value_sequeue: {value_sequeue}")
+    return namedtuple(type_name, [*key_sequeue])(*value_sequeue)
+
+
 def get_object_description(obj, fname, fline):
     """Return method or funcition description for error report, include location, class name, etc."""
     if isinstance(obj, types.MethodType):
@@ -793,14 +774,24 @@ def _convert_stub_tensor(data):
     if is_stub_tensor(data):
         return data.stub_sync()
     if isinstance(data, tuple):
+        # Handle namedtuple since its type is tuple.
+        if hasattr(data, "_fields"):
+            type_name = data.__class__.__name__
+            data_dict = data._asdict()
+            fields = data_dict.keys()
+            return namedtuple(type_name, fields)(**_convert_stub_tensor(data_dict))
         return tuple(_convert_stub_tensor(x) for x in data)
     if isinstance(data, list):
         # Keep the list object not change.
-        for _, input_data in enumerate(data):
-            input_data = _convert_stub_tensor(input_data)
+        for i in range(len(data)):
+            data[i] = _convert_stub_tensor(data[i])
         return data
     if isinstance(data, dict):
-        return dict((_convert_stub_tensor(key), _convert_stub_tensor(value)) for key, value in data.items())
+        # Keep the dict object not change.
+        keys = tuple(data.keys())
+        for key in keys:
+            data[_convert_stub_tensor(key)] = _convert_stub_tensor(data.pop(key))
+        return data
     return data
 
 
@@ -844,15 +835,9 @@ def get_script_id_attrs(script):
     return res
 
 
-def merge_global_params(global_dict):
-    """Merge the global parameter."""
-    logger.debug(f'merge global_dict: {global_dict}')
-    _global_params.update(global_dict)
-
-
 def get_global_params():
     """Get the global parameter."""
-    logger.debug(f'get global_dict: {_global_params}')
+    logger.debug(f"get global_dict: {_global_params}")
     return _global_params
 
 
@@ -877,8 +862,6 @@ class ThirdPartyLibraryChecker:
     4. Third-party libraries have 'site-packages' in their installation path.
     """
     def __init__(self):
-        self.jit_modules = self.get_jit_modules()
-        self.jit_ignore_modules = self.get_jit_ignore_modules()
         self.user_workspace_dir = self.get_top_level_module_path(os.getcwd())
         self.python_builtin_dir = os.path.abspath(os.path.dirname(os.__file__))
 
@@ -927,11 +910,13 @@ class ThirdPartyLibraryChecker:
         """Check if module is a third-party library."""
         module_leftmost_name = module.__name__.split('.')[0]
         # Modules in jit_ignore_modules are treated as third-party libraries, such as sys.builtin_module_names.
-        if module_leftmost_name in self.jit_ignore_modules:
+        jit_ignore_modules = self.get_jit_ignore_modules()
+        if module_leftmost_name in jit_ignore_modules:
             logger.debug(f"Found third-party module '{module_leftmost_name}' in jit_ignore_modules.")
             return True
         # Modules in jit_modules require jit and they are considered to be in user workspace.
-        if module_leftmost_name in self.jit_modules:
+        jit_modules = self.get_jit_modules()
+        if module_leftmost_name in jit_modules:
             logger.debug(f"Found user-defined module '{module_leftmost_name}' in jit_modules.")
             return False
         # A modules without __file__ attribute is considered to be in user workspace.
@@ -965,10 +950,13 @@ class ThirdPartyLibraryChecker:
         """Check if value is from a third-party library."""
         if inspect.ismodule(value):
             module = value
-        elif inspect.isfunction(value) or inspect.ismethod(value):
+        elif (isinstance(value, types.FunctionType) and not hasattr(value, "__jit_function__")) or \
+            (isinstance(value, types.MethodType) and not hasattr(value.__func__, "__jit_function__")):
             if value in _convert_map():
                 return False
             module = inspect.getmodule(value)
+            if module is None:
+                return False
         else:
             return False
         return self.get_module_source_location(module) == MODULE_FROM_THIRDPARTY
@@ -983,12 +971,12 @@ def is_from_third_party_library(value):
 
 
 def get_const_abs(obj):
-    """get absolute value of const object."""
+    """Get absolute value of const object."""
     return abs(obj)
 
 
 def get_const_round(obj):
-    """get round value of const object."""
+    """Get round value of const object."""
     if isinstance(obj, tuple):
         val = obj[0]
         point_num = obj[1]
@@ -997,7 +985,7 @@ def get_const_round(obj):
 
 
 def get_const_len(obj):
-    """get the length of const object."""
+    """Get the length of const object."""
     return len(obj)
 
 
@@ -1051,6 +1039,8 @@ class Parser:
             if value == item:
                 logger.debug(f"Found unsupported internal type: '{value}'.")
                 return True
+        if ms_adapter_registry.is_registered and value == ms_adapter_registry.tensor:
+            return True
         return False
 
     @staticmethod
@@ -1161,7 +1151,11 @@ class Parser:
         """Get mindspore builtin namespace and symbol."""
         if var in self.closure_namespace:
             logger.debug(f"Found '{var}' in closure_namespace {self.closure_namespace.__str__()}.")
-            return self.closure_namespace, var
+            try:
+                value = self.closure_namespace[var]
+                return self.closure_namespace, var, value
+            except UnboundLocalError:
+                return self.closure_namespace, var, None
         if var in self.global_namespace:
             logger.debug(f"Found '{var}' in global_namespace {self.global_namespace.__str__()}.")
             value = self.global_namespace[var]
@@ -1174,7 +1168,7 @@ class Parser:
             return support_info
 
         logger.debug(f"The name '{var}' is an undefined symbol.")
-        return None, None
+        return None, None, None
 
     def check_third_party_library_side_effect(self, var, attr):
         """Check if value is from a third-party library."""
@@ -1192,11 +1186,12 @@ class Parser:
             "json": {"load", "loads", "dump", "dumps"},
             "pickle": {"load", "loads", "dump", "dumps"},
             "h5py": {"File", "Group", "Dataset"},
-            "os": {"listdir", "isfile", "exists", "isdir", "mkdir", "remove", "rmdir", "symlink"},
+            "os": {"listdir", "isfile", "exists", "isdir", "mkdir", "remove", "rmdir", "symlink", "rename"},
             "shutil": {"copy", "copy2", "copytree", "move", "rmtree"},
             "pathlib": {"Path", "mkdir", "rmdir", "unlink", "rename", "symlink_to"},
             "glob": {"glob", "iglob"},
-            "zipfile": {"zipfile", "ZipFile", "write", "extractall"}}
+            "zipfile": {"zipfile", "ZipFile", "write", "extractall"},
+            "troubleshooter": {"save", "load"}}
         if var in self.global_namespace:
             logger.debug(f"Found '{var}' in global_namespace {self.global_namespace.__str__()}.")
             value = self.global_namespace[var]
@@ -1268,7 +1263,6 @@ class Parser:
                 start_lineno += 1
         return comments
 
-
     def get_source_code(self, start_lineno, start_colno, end_lineno, end_colno):
         """
         Get the script source at the location.
@@ -1282,6 +1276,7 @@ class Parser:
         Returns:
             str, the source string.
         """
+
         if start_lineno == 0:
             logger.critical('start_lineno should not be 0')
 

mindspore/_extends/parse/resources.py

@@ -1,6 +1,6 @@
 # This is the Python adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/).
 #
-# Copyright 2020-2022 Huawei Technologies Co., Ltd
+# Copyright 2020-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.