mindspore 1.10.0__cp37-cp37m-win_amd64.whl → 2.0.0rc1__cp37-cp37m-win_amd64.whl

mindspore/common/api.py

@@ -1,6 +1,6 @@
 # This is the Python adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/).
 #
-# Copyright 2020-2021 Huawei Technologies Co., Ltd
+# Copyright 2020-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,27 +24,29 @@ import time
 import ast
 import inspect
 import importlib
+import hashlib
 from collections import OrderedDict
 from functools import wraps
+import numpy as np
 import mindspore as ms
 from mindspore import context
 from mindspore import log as logger
 from mindspore._extends.remote import kernel_build_server
 from mindspore.common.tensor import Tensor as PythonTensor
-from mindspore.common.tensor import CSRTensor as PythonCSRTensor
-from mindspore.common.tensor import COOTensor as PythonCOOTensor
-from mindspore.common.tensor import RowTensor as PythonRowTensor
-from mindspore.common.initializer import initializer
-from mindspore._c_expression import GraphExecutor_, Tensor, MetaTensor, CSRTensor, RowTensor, COOTensor, \
-    PynativeExecutor_, verify_inputs_signature, init_exec_dataset, _set_dataset_mode_config, init_pipeline, \
-    _ms_memory_recycle
-from mindspore.parallel._tensor import _load_tensor_by_layout
-from mindspore.parallel._ps_context import _is_role_pserver, _is_role_sched, _enable_distributed_mindrt
-from mindspore.parallel._utils import _check_full_batch, _get_parameter_broadcast, _get_pipeline_stages
-from mindspore._checkparam import Validator
+from mindspore.common.sparse_tensor import CSRTensor as PythonCSRTensor
+from mindspore.common.sparse_tensor import COOTensor as PythonCOOTensor
+from mindspore.common.sparse_tensor import RowTensor as PythonRowTensor
+from mindspore._c_expression import GraphExecutor_, Tensor, CSRTensor, RowTensor, COOTensor, \
+    PyNativeExecutor_, verify_inputs_signature, init_exec_dataset, _set_dataset_mode_config, init_pipeline, \
+    _ms_memory_recycle, _bind_device_ctx
+from mindspore.parallel._ps_context import _is_role_sched
+from mindspore.parallel._utils import _check_full_batch, _get_parameter_broadcast, _is_pynative_parallel, \
+    _get_pipeline_stages, _is_in_auto_parallel_mode
+from mindspore import _checkparam as Validator
+from mindspore._checkparam import is_stub_tensor
 from mindspore.common._utils import is_shape_unknown
 from mindspore.common.mutable import mutable
-
+from mindspore.common._register_for_adapter import ms_adapter_registry
 
 # store ms_function class compiled pipeline cache
 ms_compile_cache = set()
@@ -52,7 +54,7 @@ ms_compile_cache = set()
 cells_compile_cache = {}
 
 BROADCAST_PHASE = "_broadcast_"
-_PYNATIVE_PARRALLEL_FUNC_NAME = "after_shard"
+_PYNATIVE_PARALLEL_FUNC_NAME = "after_shard"
 
 
 def _convert_python_data(data):
@@ -65,6 +67,8 @@ def _convert_python_data(data):
     Returns:
         data, a data convert C++ to python
     """
+    if isinstance(data, Tensor) and data.adapter_flag:
+        return ms_adapter_registry.tensor(data)
     if isinstance(data, Tensor) and not isinstance(data, PythonTensor):
         return PythonTensor(data, internal=True)
     if isinstance(data, CSRTensor) and not isinstance(data, PythonCSRTensor):
@@ -103,7 +107,8 @@ def _wrap_func(fn):
 
 def _check_all_tensor(sequence):
     for element in sequence:
-        if not isinstance(element, Tensor) and not (isinstance(element, tuple) and _check_all_tensor(element)):
+        if not isinstance(element, Tensor) and not is_stub_tensor(element) and not (isinstance(element, tuple)
+                                                                                    and _check_all_tensor(element)):
             return False
     return True
 
@@ -117,28 +122,28 @@ def _handle_func_args(func, *args, **kwargs):
         bound_arguments.apply_defaults()
         args = bound_arguments.args
         kwargs = bound_arguments.kwargs
-        # After apply_defaults, kwargs should be empty here.
-        if kwargs:
-            raise ValueError(f"Failed to handle kwargs of {func.__name__}. Maybe you pass wrong arguments, "
-                             f"or there is a key in kwargs that is not used as a function argument, "
-                             f"args: {args}, kwargs: {kwargs}")
 
     positional_args = 0
     default_args = 0
+    has_var = False
     for value in inspect.signature(func).parameters.values():
         if value.kind is inspect.Parameter.VAR_POSITIONAL or value.kind is inspect.Parameter.VAR_KEYWORD:
-            return args
+            has_var = True
         if value.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD:
             if value.default is inspect.Parameter.empty:
                 positional_args += 1
             else:
                 default_args += 1
+
+    if has_var:
+        return args, kwargs
+
     if len(args) < positional_args:
         raise TypeError(f"Function {func.__name__} needs {positional_args} positional argument, but got {len(args)}.")
     if len(args) > positional_args + default_args:
         raise TypeError(f"Function {func.__name__} needs {positional_args} positional argument and {default_args} "
                         f"default argument, total {positional_args + default_args}, but got {len(args)}.")
-    return args
+    return args, kwargs
 
 
 sys_path = list(sys.path)
@@ -164,7 +169,8 @@ def __get_compile_cache_dep_files(file_path, compile_cache_dep_files, pkg):
     for node in ast.iter_child_nodes(root):
         module_name = ""
         if isinstance(node, ast.ImportFrom):
-            module_name = node.module
+            if node.module is not None:
+                module_name = node.module
             if node.level == 1:
                 module_name = "." + module_name
         elif not isinstance(node, ast.Import):
@@ -219,31 +225,59 @@ def _restore_mutable_attr(args_list, compile_args):
     for idx, arg in enumerate(args_list):
         if hasattr(arg, "__ms_mutable__") and getattr(arg, "__ms_mutable__") and \
                 not (hasattr(arg, "const_arg") and getattr(arg, "const_arg")):
-            new_compile_args += (mutable(compile_args[idx]),)
+            if hasattr(arg, "__ms_dynamic_len__"):
+                new_compile_args += (mutable(compile_args[idx], getattr(arg, "__ms_dynamic_len__")),)
+            else:
+                new_compile_args += (mutable(compile_args[idx], False),)
         else:
             new_compile_args += (compile_args[idx],)
     return new_compile_args
 
 
-def _get_args_for_run(obj, args_list):
-    """Get the actual input args for runtime."""
-    inputs = []
-    for i in args_list:
-        if isinstance(i, PythonTensor):
-            if i.has_init:
-                i.init_data()
-            if not i.const_arg:
-                inputs.append(i)
-        elif isinstance(i, (Tensor, CSRTensor, COOTensor)):
-            inputs.append(i)
-        elif hasattr(i, "__ms_mutable__") and getattr(i, "__ms_mutable__"):
-            inputs.append(i)
-        elif context.get_context("grad_for_scalar") and isinstance(i, (int, float)):
-            inputs.append(i)
-        elif hasattr(obj, "enable_tuple_broaden") and obj.enable_tuple_broaden and isinstance(i, tuple) and \
-                _check_all_tensor(i):
-            inputs.append(i)
-    return inputs
+def _get_parameter_layout():
+    graph_executor = GraphExecutor_.get_instance()
+    layout = dict()
+    for phase in ms_compile_cache:
+        layout.update(graph_executor.get_parameter_layout(phase))
+    return layout
+
+
+def _handle_arg(obj, arg):
+    """Handle arg for runtime .If need handle the arg, return True"""
+    if isinstance(arg, PythonTensor):
+        if arg.has_init:
+            arg.init_data()
+        if not arg.const_arg:
+            return arg
+    elif isinstance(arg, (Tensor, CSRTensor, COOTensor)):
+        return arg
+    elif hasattr(arg, "__ms_mutable__") and getattr(arg, "__ms_mutable__"):
+        # mutable([]) will be eliminated by FuncGraphSpecializer, and empty list is not supported by backend.
+        if isinstance(arg, list) and not arg:
+            return None
+        return arg
+    elif context.get_context("grad_for_scalar") and isinstance(arg, (int, float)):
+        return arg
+    elif hasattr(obj, "enable_tuple_broaden") and obj.enable_tuple_broaden and isinstance(arg, tuple) and \
+            _check_all_tensor(arg):
+        return arg
+    return None
+
+
+def _get_args_for_run(obj, args, kwargs):
+    """Get the actual input args and kwargs for runtime."""
+    new_args = []
+    for arg in args:
+        new_arg = _handle_arg(obj, arg)
+        if new_arg is not None:
+            new_args.append(new_arg)
+
+    for _, value in kwargs.items():
+        new_value = _handle_arg(obj, value)
+        if new_value is not None:
+            new_args.append(new_value)
+
+    return new_args
 
 
 class _MindsporeFunctionExecutor:
@@ -256,7 +290,7 @@ class _MindsporeFunctionExecutor:
     Args:
         fn (Function): The root function to compile.
         input_signature (Function): User defines signature to verify input.
-        ms_create_time(TimeStamp): The time ms_function created
+        ms_create_time(TimeStamp): Time the function was created
         obj (Object): If function is a method, obj is the owner of function,
              else, obj is none.
 
@@ -280,69 +314,60 @@ class _MindsporeFunctionExecutor:
         self._create_time = ms_create_time
         self.jit_config_dict = jit_config.jit_config_dict if jit_config else None
 
-    def _set_compile_cache_dep_files(self):
-        # If enable compile cache, get the dependency files list
-        enable_compile_cache = context.get_context("enable_compile_cache")
-        if enable_compile_cache is None:
-            enable_compile_cache = os.getenv('MS_COMPILER_CACHE_ENABLE')
-        if enable_compile_cache is True or enable_compile_cache == "1":
-            self._graph_executor.set_compile_cache_dep_files(_get_compile_cache_dep_files())
-
-    def _parallel_process_for_ms_function(self, phase):
-        """Set parameter and optimizer states data according to sliced shape for shard"""
-        obj = self.shard_parent_obj if self.obj is None else self.obj
-        obj.parameter_layout_dict = self._graph_executor.get_parameter_layout(phase)
-        obj.parallel_parameter_name_list = self._graph_executor.get_parallel_parameter_name_list(phase)
-        replace = obj.init_parameters_data(auto_parallel_mode=True)
-        new_param = {x.name: replace[x] for x in replace if id(x) != id(replace[x])}
-        self._graph_executor.updata_param_node_default_input(phase, new_param)
-        obj.load_parameter_slice(None)
-
-        if _pynative_executor.get_optimizer():
-            params = obj.trainable_params()
-            opt_params = _pynative_executor.get_optimizer().trainable_params()
-            opt_states = []
-            for opt_param in opt_params:
-                for param in params:
-                    if opt_param.name.find(param.name) > 0:
-                        opt_states.append(opt_param)
-                        obj.parameter_layout_dict[opt_param.name] = obj.parameter_layout_dict[param.name]
-                        continue
-
-            if len(opt_states) != len(params):
-                states_tuple = (opt_states[:len(params)], opt_states[len(params):])
+    @_wrap_func
+    def __call__(self, *args, **kwargs):
+        args_list = args
+        if self.obj is not None:
+            args_list = args_list[1:]
+        phase = ""
+        try:
+            if context.get_context("mode") == context.PYNATIVE_MODE:
+                _pynative_executor.set_ms_function_compile_status(True, phase)
+                phase = self.compile(self.fn.__name__, *args_list, **kwargs)
+                _pynative_executor.set_ms_function_compile_status(False, phase)
             else:
-                states_tuple = (opt_states[:len(params)],)
-            for states in states_tuple:
-                for param, state in zip(params, states):
-                    if param.shape != state.shape:
-                        if state.has_init:
-                            state.set_data(initializer("zeros", param.shape), True)
-                        else:
-                            layout = obj.parameter_layout_dict[param.name]
-                            new_tensor = _load_tensor_by_layout(state.data, layout)
-                            state.set_data(new_tensor, True)
-
-        _pynative_executor.get_top_cell().parameter_layout_dict = obj.parameter_layout_dict
-
-    def compile(self, args_list, method_name):
+                phase = self.compile(self.fn.__name__, *args_list, **kwargs)
+        except Exception as err:
+            _pynative_executor.clear_res()
+            raise err
+
+        if context.get_context("precompile_only"):
+            return None
+
+        new_inputs = self._generate_run_args(args_list, kwargs)
+        output = self._graph_executor(tuple(new_inputs), phase)
+        if context.get_context("mode") == context.PYNATIVE_MODE:
+            output = _pynative_executor.grad_ms_function(output, *new_inputs)
+
+        enable_ge = os.getenv("MS_ENABLE_GE") == "1"
+        if enable_ge and self.jit_config_dict is None:
+            raise RuntimeError("GE and jit_level=O3 should be used together, but jit_config is None.")
+        if self.jit_config_dict:
+            enable_jit_level_o3 = self.jit_config_dict.get('jit_level') == "O3"
+            if (enable_ge and not enable_jit_level_o3) or (not enable_ge and enable_jit_level_o3):
+                raise RuntimeError("GE and jit_level=O3 should be used together, but got MS_ENABLE_GE={}, jit_level={}".
+                                   format(os.getenv("MS_ENABLE_GE"), self.jit_config_dict.get('jit_level')))
+
+        return output
+
+    def compile(self, method_name, *args, **kwargs):
         """Returns pipeline for the given args."""
         # Check whether hook function registered on Cell object.
         if self.obj and hasattr(self.obj, "_hook_fn_registered"):
             if self.obj._hook_fn_registered():
-                logger.warning(f"For 'Cell', it's not support hook function when using ms_function. If you want to "
-                               f"use hook function, please use context.set_context to set pynative mode and remove "
-                               f"`ms_function`.")
+                logger.warning(f"For 'Cell', it's not support hook function when using 'jit' decorator. "
+                               f"If you want to use hook function, please use context.set_context to set "
+                               f"pynative mode and remove 'jit' decorator.")
         # Chose dynamic shape tensors or actual input tensors as compile args.
-        compile_args = self._generate_compile_args(args_list)
+        compile_args = self._generate_compile_args(args)
         # Restore the mutable attr for every arg.
-        compile_args = _restore_mutable_attr(args_list, compile_args)
+        compile_args = _restore_mutable_attr(args, compile_args)
 
         generate_name = self.fn.__module__ + "." + self.fn.__name__ + "." + self.fn.__code__.co_filename + "." + \
-                        str(self.fn.__code__.co_firstlineno) + '.' + str(id(self.fn))
+            str(self.fn.__code__.co_firstlineno)
         if _pynative_executor.grad_flag():
             generate_name = generate_name + ".grad"
-        if is_pynative_parallel():
+        if _is_pynative_parallel():
             generate_name = generate_name[:generate_name.rfind(str(id(self.fn)))] + str(id(self.shard_parent_obj))
 
         # Add key with obj
@@ -365,7 +390,7 @@ class _MindsporeFunctionExecutor:
             self.enable_tuple_broaden = self.obj.enable_tuple_broaden
 
         self._graph_executor.set_enable_tuple_broaden(self.enable_tuple_broaden)
-        key = self._graph_executor.generate_arguments_key(compile_args, self.enable_tuple_broaden)
+        key = self._graph_executor.generate_arguments_key(self.fn, compile_args, kwargs, self.enable_tuple_broaden)
         phase = generate_name + '.' + str(key)
         if phase in ms_compile_cache:
             return phase
@@ -376,43 +401,17 @@ class _MindsporeFunctionExecutor:
             self._graph_executor.set_jit_config(self.jit_config_dict)
 
         if self.obj is None:
-            is_compile = self._graph_executor.compile(self.fn, compile_args, phase, True)
+            is_compile = self._graph_executor.compile(self.fn, compile_args, kwargs, phase, True)
         else:
             if isinstance(self.obj, ms.nn.Cell):
                 self._graph_executor.set_weights_values(self.obj.parameters_dict())
-            is_compile = self._graph_executor.compile(self.obj, compile_args, phase, True)
-
-        # init sliced parameter and optimizer state
-        if is_pynative_parallel() and self.fn.__name__ == _PYNATIVE_PARRALLEL_FUNC_NAME:
-            self._parallel_process_for_ms_function(phase)
-
-        # init the rest optimizer states
-        if is_pynative_parallel() and _pynative_executor.get_optimizer():
-            opt_states = _pynative_executor.get_optimizer().trainable_params()
-            self._optimizer_state_init(opt_states)
+            is_compile = self._graph_executor.compile(self.obj, compile_args, kwargs, phase, True)
 
         if not is_compile:
             raise RuntimeError("Executor compile failed.")
         ms_compile_cache.add(phase)
         return phase
 
-    @_wrap_func
-    def __call__(self, *args):
-        args_list = args
-        if self.obj is not None:
-            args_list = args_list[1:]
-
-        phase = self.compile(args_list, self.fn.__name__)
-        if context.get_context("precompile_only"):
-            return None
-        new_inputs = self._generate_run_args(args_list)
-        output = self._graph_executor(tuple(new_inputs), phase)
-        if context.get_context("mode") == context.PYNATIVE_MODE:
-            _pynative_executor.set_graph_phase(phase)
-            output = _pynative_executor.grad_ms_function(output, *new_inputs)
-
-        return output
-
     @staticmethod
     def _optimizer_state_init(opt_states):
         """set data for all optimizer states in case it is executed in graph mode"""
@@ -423,52 +422,64 @@ class _MindsporeFunctionExecutor:
             if opt_param.has_init and (prefix in prefix_list or opt_param.name == "global_step"):
                 opt_param.init_data()
 
+    def _set_compile_cache_dep_files(self):
+        # If enable compile cache, get the dependency files list
+        enable_compile_cache = context.get_context("enable_compile_cache")
+        if enable_compile_cache is None:
+            enable_compile_cache = os.getenv('MS_COMPILER_CACHE_ENABLE')
+        if enable_compile_cache is True or enable_compile_cache == "1":
+            self._graph_executor.set_compile_cache_dep_files(_get_compile_cache_dep_files())
+
     def _generate_compile_args(self, args_list):
         """Chose dynamic shape tensors or actual input tensors as compile args."""
         # Case: If the shape of input args is dynamic, get dynamic shape tensor from context and use it to compile.
-        compile_args = _pynative_executor.get_dynamic_input(args_list)
+        compile_args = args_list
         # Case: The `set_inputs()` of Cell object has been set, using these dynamic shape args as compile args.
-        if isinstance(self.obj, ms.nn.Cell) and self.obj.get_inputs():
+        if self.fn.__name__ == 'construct' and isinstance(self.obj, ms.nn.Cell) and self.obj.get_inputs():
             compile_args = self.obj.get_inputs()
-            for args in compile_args:
-                Validator.check_isinstance("args set in `set_inputs()` of Cell", args, PythonTensor)
-            Validator.check_dynamic_shape(compile_args, args_list)
+            if len(compile_args) != len(args_list):
+                raise ValueError(f"The number of actual input tensors: {len(args_list)} is not equal to the number of "
+                                 f"dynamic shape tensors: {len(compile_args)}.")
+            for i, elem in enumerate(compile_args):
+                if isinstance(elem, PythonTensor):
+                    Validator.check_dynamic_shape(compile_args[i], args_list[i], i)
+
         # Case: If dynamic shape tensors have been assigned to `input_signature`, they are preferred as compile args.
         if self.input_signature is not None:
             if not isinstance(self.input_signature, (tuple, list)):
                 self.input_signature = (self.input_signature,)
             self.input_signature = list(self.input_signature)
             dyn_shape = False
-            for sig_args in self.input_signature:
-                Validator.check_isinstance("args in `input_signature` of `ms_function`", sig_args, MetaTensor)
-                if is_shape_unknown(sig_args.shape):
+            for i, elem in enumerate(self.input_signature):
+                if isinstance(elem, PythonTensor) and is_shape_unknown(elem.shape):
+                    Validator.check_dynamic_shape(self.input_signature[i], args_list[i], i)
                     dyn_shape = True
-            if not dyn_shape:
-                if not verify_inputs_signature(self.input_signature, args_list):
-                    raise ValueError("The input args is incompatible with the args in `input_signature`!")
-            else:
+            if dyn_shape:
                 # Checkout whether the `sens` has been added to args_list.
                 if len(self.input_signature) == len(args_list) - 1:
-                    logger.warning(f"The number of actual input args `{len(args_list)}` is one more than the number "
-                                   f"of input_signature args `{len(self.input_signature)}`. The last actual args may "
-                                   f"be `sens` and added it to compile args.")
+                    logger.warning(f"The number of actual input args '{len(args_list)}' is one more than the number "
+                                   f"of input_signature args '{len(self.input_signature)}'. The last actual args may "
+                                   f"be 'sens' and added it to compile args.")
                     self.input_signature.append(args_list[-1])
-                Validator.check_dynamic_shape(self.input_signature, args_list)
                 compile_args = tuple(self.input_signature)
-                _pynative_executor.set_dynamic_input(self.obj, *compile_args)
+                _pynative_executor.set_dynamic_input(self.obj)
+            else:
+                if not verify_inputs_signature(self.input_signature, args_list):
+                    raise ValueError("The input args is incompatible with the args in `input_signature`!")
         return compile_args
 
-    def _generate_run_args(self, args_list):
+    def _generate_run_args(self, args_list, kwargs):
         """
         Generate input args, which are required for running.
 
         Args:
             args_list (Tuple): Actual input args.
+            kwargs (Dict): Actual input kwargs.
 
         Returns:
             new_inputs, new input args, which are required for running.
         """
-        return _get_args_for_run(self, args_list)
+        return _get_args_for_run(self, args_list, kwargs)
 
 
 # The attributes used to identify a given object.
@@ -496,7 +507,7 @@ def _get_ms_function_hash(hash_input):
     return _get_obj_id(hash_input)
 
 
-def ms_function(fn=None, input_signature=None, hash_args=None, jit_config=None):
+def jit(fn=None, input_signature=None, hash_args=None, jit_config=None):
     """
     Create a callable MindSpore graph from a Python function.
 
@@ -529,30 +540,30 @@ def ms_function(fn=None, input_signature=None, hash_args=None, jit_config=None):
         >>> import numpy as np
         >>> from mindspore import Tensor
         >>> from mindspore import ops
-        >>> from mindspore import ms_function
+        >>> from mindspore import jit
         ...
         >>> x = Tensor(np.ones([1, 1, 3, 3]).astype(np.float32))
         >>> y = Tensor(np.ones([1, 1, 3, 3]).astype(np.float32))
         ...
-        >>> # create a callable MindSpore graph by calling ms_function
+        >>> # create a callable MindSpore graph by calling decorator @jit
         >>> def tensor_add(x, y):
         ...     z = x + y
         ...     return z
         ...
-        >>> tensor_add_graph = ms_function(fn=tensor_add)
+        >>> tensor_add_graph = jit(fn=tensor_add)
         >>> out = tensor_add_graph(x, y)
         ...
-        >>> # create a callable MindSpore graph through decorator @ms_function
-        >>> @ms_function
+        >>> # create a callable MindSpore graph through decorator @jit
+        >>> @jit
         ... def tensor_add_with_dec(x, y):
         ...     z = x + y
         ...     return z
         ...
         >>> out = tensor_add_with_dec(x, y)
         ...
-        >>> # create a callable MindSpore graph through decorator @ms_function with input_signature parameter
-        >>> @ms_function(input_signature=(Tensor(np.ones([1, 1, 3, 3]).astype(np.float32)),
-        ...                               Tensor(np.ones([1, 1, 3, 3]).astype(np.float32))))
+        >>> # create a callable MindSpore graph through decorator @jit with input_signature parameter
+        >>> @jit(input_signature=(Tensor(np.ones([1, 1, 3, 3]).astype(np.float32)),
+        ...                       Tensor(np.ones([1, 1, 3, 3]).astype(np.float32))))
         ... def tensor_add_with_sig(x, y):
         ...     z = x + y
         ...     return z
@@ -565,7 +576,7 @@ def ms_function(fn=None, input_signature=None, hash_args=None, jit_config=None):
         ...     return ops.exp(x)
         ...
         >>> def closure_fn(x, fn):
-        ...     @ms_function(hash_args=fn)
+        ...     @jit(hash_args=fn)
         ...     def inner_fn(a):
         ...         return fn(a)
         ...     return inner_fn(x)
@@ -583,15 +594,18 @@ def ms_function(fn=None, input_signature=None, hash_args=None, jit_config=None):
 
         @wraps(func)
         def staging_specialize(*args, **kwargs):
-            args = _handle_func_args(func, *args, **kwargs)
+            if os.getenv("MS_JIT") == '0':
+                return func(*args, **kwargs)
+
+            args, kwargs = _handle_func_args(func, *args, **kwargs)
+
             process_obj = None
             if args and not isinstance(args[0], PythonTensor) and hasattr(args[0], func.__name__):
                 process_obj = args[0]
             # only the function or cell instance wrapped by shard will fall into this branch
-            if is_pynative_parallel() and func.__name__ == _PYNATIVE_PARRALLEL_FUNC_NAME:
-                process_obj = args[0]
-                args = args[1:]
-            out = _MindsporeFunctionExecutor(func, hash_obj, input_signature, process_obj, jit_config)(*args)
+            if _is_pynative_parallel() and func.__name__ == _PYNATIVE_PARALLEL_FUNC_NAME:
+                process_obj = hash_args
+            out = _MindsporeFunctionExecutor(func, hash_obj, input_signature, process_obj, jit_config)(*args, **kwargs)
             return out
 
         return staging_specialize
@@ -601,21 +615,201 @@ def ms_function(fn=None, input_signature=None, hash_args=None, jit_config=None):
     return wrap_mindspore
 
 
+def ms_function(fn=None, input_signature=None, hash_args=None, jit_config=None):
+    """
+    Create a callable MindSpore graph from a Python function.
+
+    This allows the MindSpore runtime to apply optimizations based on graph.
+
+    Note:
+        `ms_function` will be deprecated and removed in a future version. Please use `jit` instead.
+        If `input_signature` is specified, each input of `fn` must be a Tensor. And the input arguments for `fn`
+        will not accept `**kwargs`.
+
+    Args:
+        fn (Function): The Python function that will be run as a graph. Default: None.
+        input_signature (Tensor): The Tensor which describes the input arguments. The shape and dtype of the Tensor
+            will be supplied to this function. If input_signature is specified, each input to `fn` must be a `Tensor`.
+            And the input parameters of `fn` cannot accept `**kwargs`. The shape and dtype of actual inputs should
+            keep the same as input_signature. Otherwise, TypeError will be raised. Default: None.
+        hash_args (Union[Object, List or Tuple of Objects]): The local free variables used inside `fn`,
+            like functions or objects of class defined outside `fn`. Calling `fn` again with change of `hash_args`
+            will trigger recompilation.
+        jit_config (JitConfig): Jit config for compile. Default: None.
+
+    Returns:
+        Function, if `fn` is not None, returns a callable function that will execute the compiled function; If `fn` is
+        None, returns a decorator and when this decorator invokes with a single `fn` argument, the callable function is
+        equal to the case when `fn` is not None.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import numpy as np
+        >>> from mindspore import Tensor
+        >>> from mindspore import ops
+        >>> from mindspore import ms_function
+        ...
+        >>> x = Tensor(np.ones([1, 1, 3, 3]).astype(np.float32))
+        >>> y = Tensor(np.ones([1, 1, 3, 3]).astype(np.float32))
+        ...
+        >>> # create a callable MindSpore graph by calling ms_function
+        >>> def tensor_add(x, y):
+        ...     z = x + y
+        ...     return z
+        ...
+        >>> tensor_add_graph = ms_function(fn=tensor_add)
+        >>> out = tensor_add_graph(x, y)
+        ...
+        >>> # create a callable MindSpore graph through decorator @ms_function
+        >>> @ms_function
+        ... def tensor_add_with_dec(x, y):
+        ...     z = x + y
+        ...     return z
+        ...
+        >>> out = tensor_add_with_dec(x, y)
+        ...
+        >>> # create a callable MindSpore graph through decorator @ms_function with input_signature parameter
+        >>> @ms_function(input_signature=(Tensor(np.ones([1, 1, 3, 3]).astype(np.float32)),
+        ...                               Tensor(np.ones([1, 1, 3, 3]).astype(np.float32))))
+        ... def tensor_add_with_sig(x, y):
+        ...     z = x + y
+        ...     return z
+        ...
+        >>> out = tensor_add_with_sig(x, y)
+        ...
+        ... # Set hash_args as fn, otherwise cache of compiled `closure_fn` will not be reused.
+        ... # While fn differs during calling again, recompilation will be triggered.
+        >>> def func(x):
+        ...     return ops.exp(x)
+        ...
+        >>> def closure_fn(x, fn):
+        ...     @ms_function(hash_args=fn)
+        ...     def inner_fn(a):
+        ...         return fn(a)
+        ...     return inner_fn(x)
+        ...
+        >>> inputs = Tensor(np.ones([10, 10, 10]).astype(np.float32))
+        >>> for i in range(10):
+        ...     closure_fn(inputs, func)
+    """
+
+    logger.warning("'mindspore.ms_function' will be deprecated and removed in a future version. "
+                   "Please use 'mindspore.jit' instead.")
+    return jit(fn=fn, input_signature=input_signature, hash_args=hash_args, jit_config=jit_config)
+
+
+def _core(fn=None, **flags):
+    """
+    A decorator that adds a flag to the function.
+
+    By default, the function is marked as True, enabling to use this decorator to
+    set flag to a graph.
+
+    Args:
+        fn (Function): Function to add flag. Default: None.
+        flags (dict): The following flags can be set core, which indicates that this is a core function or
+                      other flag. Default: None.
+
+    Returns:
+        Function, the function with core flag.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+    """
+
+    # need set the attr and access on c++
+    def deco(fn):
+        fn._func_graph_flags = {
+            'core': True,
+            **flags,
+        }
+        return fn
+
+    if fn is not None:
+        ret = deco(fn)
+    else:
+        ret = deco
+    return ret
+
+
+def _add_flags(fn=None, **flags):
+    """
+    A decorator that adds a flag to the function.
+
+    Note:
+        Only supports bool value.
+
+    Args:
+        fn (Function): Function or cell to add flag. Default: None.
+        flags (dict): Flags use kwargs. Default: None.
+
+    Returns:
+        Function, the function with added flags.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+    """
+
+    def deco(fn):
+        # need set the attr and access on c++
+        if not hasattr(fn, "_func_graph_flags"):
+            fn._func_graph_flags = {}
+
+        fn._func_graph_flags.update({**flags})
+        return fn
+
+    ret = deco
+    if fn is not None:
+        ret = deco(fn)
+    return ret
+
+
+def _no_recursive(callable_obj):
+    """
+    Method or function decorator for ignoring recursive check.
+
+    This allows MindSpore to skip the procedure of checking function or method recursive.
+
+    Args:
+        callable_obj (Union(method, function)): The function or method to call.
+
+    Returns:
+        Function or method with no_recursive flag.
+
+    Raises:
+        TypeError: If ms_class is used for non-class types or nn.Cell.
+        AttributeError: If the private attributes or magic methods of the class decorated by ms_class is called.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+    """
+    isCellSubClass = inspect.isclass(callable_obj) and issubclass(callable_obj, ms.nn.Cell)
+    if not isCellSubClass and not inspect.ismethod(callable_obj) and not inspect.isfunction(callable_obj):
+        raise TypeError(f"Decorator no_recursive is used for callable object, but got {callable_obj}.")
+    _add_flags(callable_obj, no_recursive=True)
+    return callable_obj
+
+
 def ms_class(cls):
     """
     Class decorator for user-defined classes.
 
     This allows MindSpore to identify user-defined classes and thus obtain their attributes and methods.
 
+    Note:
+        `ms_class` will be deprecated and removed in a future version. Please use `jit_class` instead.
+
     Args:
         cls (Class): User-defined class.
 
     Returns:
-        Class with __ms_class__ attribute.
+        Class.
 
     Raises:
         TypeError: If ms_class is used for non-class types or nn.Cell.
-        AttributeError: If the private attributes or magic methods of the class decorated by ms_class is called.
+        AttributeError: If the private attributes or magic methods of the class decorated with ms_class is called.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -647,6 +841,9 @@ def ms_class(cls):
         20
     """
 
+    logger.warning("'mindspore.ms_class' will be deprecated and removed in a future version. "
+                   "Please use 'mindspore.jit_class' instead.")
+
     # Check if cls is of type class.
     if not inspect.isclass(cls):
         raise TypeError(f'Decorator ms_class can only be used for class type, but got {cls}.')
@@ -658,6 +855,81 @@ def ms_class(cls):
     return cls
 
 
+def jit_class(cls):
+    """
+    Class decorator for user-defined classes.
+
+    This allows MindSpore to identify user-defined classes and thus obtain their attributes and methods.
+
+    Args:
+        cls (Class): User-defined class.
+
+    Returns:
+        Class.
+
+    Raises:
+        TypeError: If `jit_class` is used for non-class types or nn.Cell.
+        AttributeError: If the private attributes or magic methods of the class decorated with `jit_class` is called.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import mindspore.nn as nn
+        >>> from mindspore import jit_class
+        ...
+        >>> @jit_class
+        ... class UserDefinedNet:
+        ...     def __init__(self):
+        ...         self.value = 10
+        ...
+        ...     def func(self, x):
+        ...         return 2 * x
+        ...
+        >>> class Net(nn.Cell):
+        ...     def __init__(self):
+        ...         super(Net, self).__init__()
+        ...         self.net = UserDefinedNet()
+        ...
+        ...     def construct(self, x):
+        ...         out = self.net.value + self.net.func(x)
+        ...         return out
+        ...
+        >>> net = Net()
+        >>> out = net(5)
+        >>> print(out)
+        20
+    """
+
+    # Check if cls is of type class.
+    if not inspect.isclass(cls):
+        raise TypeError(f'Decorator jit_class can only be used for class type, but got {cls}.')
+    # Check if cls is nn.Cell.
+    if issubclass(cls, ms.nn.Cell):
+        raise TypeError(f"Decorator jit_class is used for user-defined classes and cannot be used for nn.Cell: {cls}.")
+    setattr(cls, '__ms_class__', True)
+    return cls
+
+
+def set_adapter_config(config):
+    """
+    Register configuration information for MSAdapter.
+
+    Args:
+        config (dict): Configuration information.
+    """
+    if not isinstance(config, dict):
+        raise TypeError(f"The input argument of 'set_adapter_config' should be a dict, but got {config}.")
+    for key, value in config.items():
+        if key == "Tensor":
+            setattr(value, "__adapter_tensor__", True)
+            ms_adapter_registry.register_tensor(value)
+        elif key == "convert_object_map":
+            ms_adapter_registry.register_convert_map(value)
+        else:
+            raise ValueError(f"Unsupported key in adapter config: {key}")
+
+
 def _function_forbid_reuse(func):
     if not inspect.isfunction(func):
         raise TypeError(f'Decorator _function_forbid_reuse can only be used for function type, but got {func}.')
@@ -665,23 +937,6 @@ def _function_forbid_reuse(func):
     return func
 
 
-def is_pynative_parallel():
-    run_mode = context.get_context('mode')
-    parallel_mode = context.get_auto_parallel_context('parallel_mode')
-    return run_mode == context.PYNATIVE_MODE and parallel_mode in (
-        context.ParallelMode.SEMI_AUTO_PARALLEL, context.ParallelMode.AUTO_PARALLEL)
-
-
-def _get_auto_split_param_names(parameter_layout_dict):
-    auto_split_param_names = []
-    for key, value in parameter_layout_dict.items():
-        for dim in value[1]:
-            if dim != -1:
-                auto_split_param_names.append(key)
-                break
-    return auto_split_param_names
-
-
 def _build_broadcast_graph(broadcast_params_dict, broadcast_phase):
     """Build broadcast graph."""
     from mindspore.nn.wrap.cell_wrapper import _BroadCastCell
@@ -697,10 +952,24 @@ def _build_broadcast_graph(broadcast_params_dict, broadcast_phase):
         broadcast_params_dict[param_name].set_data(param)
 
 
-def _parameter_broadcast(obj, auto_parallel_mode):
-    """Parameter broadcast."""
+def _get_auto_split_param_names(parameter_layout_dict):
     auto_split_param_names = []
-    if auto_parallel_mode:
+    for key, value in parameter_layout_dict.items():
+        for dim in value[1]:
+            if dim != -1:
+                auto_split_param_names.append(key)
+                break
+    return auto_split_param_names
+
+
+def _parameter_broadcast(obj):
+    """
+    Parameter broadcast.
+    When the parallel mode is 'semi_auto_parallel' or 'auto_parallel', it will broadcast the parameters that have not
+    split.
+    """
+    auto_split_param_names = []
+    if _is_in_auto_parallel_mode():
         auto_split_param_names = _get_auto_split_param_names(obj.parameter_layout_dict)
 
     broadcast_params_dict = obj.parameters_broadcast_dict()
@@ -713,7 +982,7 @@ def _parameter_broadcast(obj, auto_parallel_mode):
     _build_broadcast_graph(broadcast_params_dict, broadcast_phase)
 
 
-class _PynativeExecutor:
+class _PyNativeExecutor:
     """
     A pynative executor used to compile/manage/run single op.
 
@@ -731,27 +1000,59 @@ class _PynativeExecutor:
     """
 
     def __init__(self):
-        self._executor = PynativeExecutor_.get_instance()
+        self._executor = PyNativeExecutor_.get_instance()
         self._executor.set_py_exe_path(sys.executable)
         self._executor.set_kernel_build_server_dir(os.path.split(kernel_build_server.__file__)[0] + os.sep)
-        self._optimizer = None
         self._top_cell = None
 
+    def __call__(self):
+        """
+        PyNative executor run grad graph.
+
+        Return:
+            The return object after running grad graph.
+        """
+        return self._executor()
+
     @staticmethod
-    def parameter_broadcast(obj, phase, auto_parallel_mode):
+    def parameter_broadcast(obj, phase):
         """
         Run broadcast for parameter.
 
         Args:
             obj (Cell): The cell instance.
             phase (str): The phase of cell instance.
-            auto_parallel_mode (bool): The flag of running auto parallel.
 
         Return:
             None.
         """
         if BROADCAST_PHASE not in phase and _get_parameter_broadcast():
-            _parameter_broadcast(obj, auto_parallel_mode)
+            _parameter_broadcast(obj)
+
+    def real_run_op(self, *args):
+        """
+        Run single op.
+
+        Args:
+            args (tuple): Op prim and input arguments.
+
+        Return:
+            Tensor, result of run op.
+        """
+        return self._executor.real_run_op(*args)
+
+    def run_op_async(self, prim, args):
+        """
+        Run single op async.
+
+        Args:
+            prim (Primitive): Op primitive
+            args (tuple): input arguments.
+
+        Return:
+            StubNode, result of run op.
+        """
+        return self._executor.run_op_async(prim, args)
 
     def new_graph(self, obj, *args, **kwargs):
         """
@@ -782,7 +1083,7 @@ class _PynativeExecutor:
         """
         self._executor.end_graph(obj, output, *args, *(kwargs.values()))
 
-    def check_run(self, grad, obj, grad_hash_id, *args, **kwargs):
+    def check_run(self, grad, obj, weights, grad_hash_id, *args, **kwargs):
         """
         Whether the forward graph need to construct.
 
@@ -796,7 +1097,7 @@ class _PynativeExecutor:
         Return:
             bool, specifies whether the forward graph need to construct.
         """
-        return self._executor.check_run(grad, obj, grad_hash_id, *args, *(kwargs.values()))
+        return self._executor.check_run(grad, obj, weights, grad_hash_id, *args, *(kwargs.values()))
 
     def grad(self, obj, grad, weights, grad_position, *args, **kwargs):
         """
@@ -816,41 +1117,15 @@ class _PynativeExecutor:
         """
         self._executor.grad_net(grad, obj, weights, grad_position, *args, *(kwargs.values()))
 
-    def del_cell(self, obj):
-        """
-        Clean resource for cell.
-
-        Args:
-            obj (Function/Cell): The function or cell instance.
-
-        Return:
-            None.
-        """
-        self._executor.clear_cell(obj)
-
     def clear_res(self):
         """
-        Clean resource for _PynativeExecutor.
+        Clean resource for _PyNativeExecutor.
 
         Return:
             None.
         """
         return self._executor.clear_res()
 
-    def clear_grad(self, obj, *args, **kwargs):
-        """
-        Clean resource after building grad graph.
-
-        Args:
-            obj (Function/Cell): The function or cell instance.
-            args (tuple): Function or cell input arguments.
-            kwargs (dict): keyword arguments.
-
-        Return:
-            None.
-        """
-        self._executor.clear_grad(obj, *args, *(kwargs.values()))
-
     def sync(self):
         """
         SyncStream.
@@ -885,18 +1160,6 @@ class _PynativeExecutor:
         """
         return self._executor.grad_ms_function(output, *args)
 
-    def set_graph_phase(self, phase):
-        """
-        Set the phase of cell/function instance.
-
-        Args:
-            phase (str): The phase of cell/function instance.
-
-        Return:
-            None.
-        """
-        self._executor.set_graph_phase(phase)
-
     def grad_flag(self):
         """
         The flag of building grad graph.
@@ -918,30 +1181,29 @@ class _PynativeExecutor:
         """
         self._executor.set_grad_flag(flag)
 
-    def set_dynamic_input(self, obj, *args):
+    def set_ms_function_compile_status(self, status, phase):
         """
-        Set dynamic shape tensor of input arguments.
+        Set ms_function is compiling
 
         Args:
-            obj (Function/Cell): The function or cell instance.
-            args (tuple): Function or cell dynamic input arguments.
-
+            status(bool): ms_function compile status
+            phase (str): The phase of cell/function instance.
         Return:
             None.
         """
-        self._executor.set_dynamic_input(obj, *args)
+        self._executor.set_ms_function_compile_status(status, phase)
 
-    def get_dynamic_input(self, *actual_args):
+    def set_dynamic_input(self, obj):
         """
-        Get dynamic shape arguments according to actual input arguments.
+        Set dynamic shape tensor of input arguments.
 
         Args:
-            actual_args(tuple): Actual input arguments of Function or Cell.
+            obj (Function/Cell): The function or cell instance.
 
         Return:
-            dynamic_shape_args(tuple): Dynamic shape arguments of Function or Cell.
+            None.
         """
-        return self._executor.get_dynamic_input(*actual_args)
+        self._executor.set_dynamic_input(obj)
 
     def is_first_cell(self):
         """
@@ -965,14 +1227,6 @@ class _PynativeExecutor:
         """
         self._executor.set_hook_changed(cell)
 
-    def get_optimizer(self):
-        """
-        Get the optimizer.
-
-        Return:
-            The optimizer.
-        """
-        return self._optimizer
 
     def get_top_cell(self):
         """
@@ -983,20 +1237,18 @@ class _PynativeExecutor:
         """
         return self._top_cell
 
-    def __call__(self, sens_param, obj, *args, **kwargs):
+
+    def constant_folding(self, *args):
         """
-        PyNative executor run grad graph.
+        Get value by infer value.
 
         Args:
-            obj (Function/Cell): The function or cell instance.
-            args (tuple): Function or cell input arguments.
-            kwargs (dict): keyword arguments.
+            args (tuple): Op prim and input arguments.
 
         Return:
-            The return object after running grad graph.
+            Tensor, the value get by op infer.
         """
-        args = args + tuple(kwargs.values())
-        return self._executor(sens_param, obj, args)
+        return self._executor.constant_folding(*args)
 
 
 class _CellGraphExecutor:
@@ -1012,10 +1264,12 @@ class _CellGraphExecutor:
     Returns:
         Graph, return the result of pipeline running.
     """
+
     def __init__(self):
         # create needed graph by lazy mode
         self.is_init = False
         self.enable_tuple_broaden = False
+        self.obfuscate_config = None  # used for model's dynamic obfuscation
         self._graph_executor = GraphExecutor_.get_instance()
         self._graph_executor.set_py_exe_path(sys.executable)
         self._graph_executor.set_kernel_build_server_dir(os.path.split(kernel_build_server.__file__)[0] + os.sep)
@@ -1084,17 +1338,17 @@ class _CellGraphExecutor:
         if "train" in phase and (enable_compile_cache is True or enable_compile_cache == "1"):
             self._graph_executor.set_compile_cache_dep_files(_get_compile_cache_dep_files())
 
-    def compile(self, obj, *args, phase='predict', do_convert=True, auto_parallel_mode=False, jit_config_dict=None):
+    def compile(self, obj, *args, phase='predict', do_convert=True, jit_config_dict=None, **kwargs):
         """
         Compiles graph.
 
         Args:
             obj (Function/Cell): The function or cell instance need compile.
-            args (tuple): Function or cell input arguments.
             phase (str): The name of compile phase. Default: 'predict'.
             do_convert (bool): When set to True, convert ME graph to GE graph after compiling graph.
-            auto_parallel_mode: When set to True, use auto parallel mode to compile graph.
             jit_config_dict (dict): Jit config for compile. Default: None.
+            args (tuple): Args of the Cell object.
+            kwargs (dict): Kwargs of the Cell object.
 
         Return:
             Str, the full phase of the cell.
@@ -1104,14 +1358,13 @@ class _CellGraphExecutor:
         if not hasattr(obj, obj.__parse_method__):
             raise AttributeError(
                 'The class {} dose not have method {}'.format(obj.__class__.__name__, obj.__parse_method__))
-        args_list = args
 
         self.enable_tuple_broaden = False
         if hasattr(obj, "enable_tuple_broaden"):
             self.enable_tuple_broaden = obj.enable_tuple_broaden
 
         self._graph_executor.set_enable_tuple_broaden(self.enable_tuple_broaden)
-        key = self._graph_executor.generate_arguments_key(args_list, self.enable_tuple_broaden)
+        key = self._graph_executor.generate_arguments_key(obj, args, kwargs, self.enable_tuple_broaden)
         obj.arguments_key = str(key)
         phase = phase + '.' + str(obj.create_time) + '.' + str(id(obj)) + '.' + obj.arguments_key
 
@@ -1121,14 +1374,14 @@ class _CellGraphExecutor:
 
         obj.check_names()
         _check_full_batch()
-        self._set_dataset_mode(args_list)
+        self._set_dataset_mode(args)
         self._set_compile_cache_dep_files(phase)
 
         enable_ge = context.get_context("enable_ge")
         self._graph_executor.set_weights_values(obj.parameters_dict())
         if jit_config_dict:
             self._graph_executor.set_jit_config(jit_config_dict)
-        result = self._graph_executor.compile(obj, args_list, phase, self._use_vm_mode())
+        result = self._graph_executor.compile(obj, args, kwargs, phase, self._use_vm_mode())
         obj.compile_cache.add(phase)
         if not result:
             raise RuntimeError("Executor compile failed.")
@@ -1137,7 +1390,15 @@ class _CellGraphExecutor:
         if graph is None:
             raise RuntimeError("Compile graph failed for phase {}.".format(phase))
 
-        self._auto_parallel_process(obj, phase, auto_parallel_mode, *args)
+        auto_parallel_mode = _is_in_auto_parallel_mode()
+        if not auto_parallel_mode:
+            replace = obj.init_parameters_data(auto_parallel_mode=auto_parallel_mode)
+            self._update_param_node_default_input(phase, replace)
+        elif 'skip_auto_parallel_compile' not in obj.get_flags().keys():
+            obj.parameter_layout_dict = self._graph_executor.get_parameter_layout(phase)
+            obj.parallel_parameter_name_list = self._graph_executor.get_parallel_parameter_name_list(phase)
+            if _get_pipeline_stages() > 1 and (not hasattr(obj, "is_first_iteration") or not obj.is_first_iteration):
+                obj.remove_redundant_parameters()
 
         if not do_convert:
             return phase, True
@@ -1148,27 +1409,10 @@ class _CellGraphExecutor:
         elif "export" in phase:
             self._build_data_graph(obj, phase)
         elif BROADCAST_PHASE not in phase and _get_parameter_broadcast():
-            _parameter_broadcast(obj, auto_parallel_mode)
+            _parameter_broadcast(obj)
 
         return phase, True
 
-    def _auto_parallel_process(self, obj, phase, auto_parallel_mode, *args):
-        """compile graph in auto parallel mode."""
-        if not auto_parallel_mode:
-            replace = obj.init_parameters_data(auto_parallel_mode=auto_parallel_mode)
-            self._update_param_node_default_input(phase, replace)
-            return
-
-        obj.parameter_layout_dict = self._graph_executor.get_parameter_layout(phase)
-        obj.parallel_parameter_name_list = self._graph_executor.get_parallel_parameter_name_list(phase)
-        replace = obj.init_parameters_data(auto_parallel_mode=True)
-        if _get_pipeline_stages() > 1 and (not hasattr(obj, "is_first_iteration") or not obj.is_first_iteration):
-            obj.remove_redundant_parameters()
-        if not context.get_context("enable_debug_runtime") or context.get_context("enable_ge"):
-            obj.load_parameter_slice(None)
-
-        self._update_param_node_default_input(phase, replace)
-
     def _update_param_node_default_input(self, phase, replace):
         new_param = {x.name: replace[x] for x in replace if id(x) != id(replace[x])}
         return self._graph_executor.updata_param_node_default_input(phase, new_param)
@@ -1186,8 +1430,7 @@ class _CellGraphExecutor:
         return self._graph_executor.get_allreduce_fusion(real_phase)
 
     def __call__(self, obj, *args, phase='predict'):
-        if context.get_context("precompile_only") or\
-           (_is_role_pserver() and not _enable_distributed_mindrt()) or _is_role_sched():
+        if context.get_context("precompile_only") or _is_role_sched():
             return None
         return self.run(obj, *args, phase=phase)
 
@@ -1215,6 +1458,8 @@ class _CellGraphExecutor:
         Run the specific graph.
 
         Args:
+            obj (Cell): The cell object.
+            args (tuple): Args of the Cell object.
             phase (str): The phase name. Default: 'predict'.
 
         Returns:
@@ -1229,16 +1474,33 @@ class _CellGraphExecutor:
             return self._exec_pip(obj, *args, phase=phase_real)
         raise KeyError('{} graph is not exist.'.format(phase_real))
 
-    def del_net_res(self, net_id):
-        self._graph_executor.del_net_res(net_id)
+    def del_net_res(self, obj, net_id):
+        """Clear the memory resource of a network."""
+        self._graph_executor.del_net_res(obj, net_id)
+
+    def _get_branch_control_input(self):
+        if ('obf_ratio' not in self.obfuscate_config.keys()) or (
+                'obf_random_seed' not in self.obfuscate_config.keys()):
+            raise ValueError("'obf_ratio' and 'obf_random_seed' must be in obfuscate_config.")
+        obf_random_seed = self.obfuscate_config.get('obf_random_seed')
+        if obf_random_seed == 0:
+            branch_control_input = 0
+        else:
+            branch_control_input = _generate_branch_control_input(obf_random_seed)
+        return branch_control_input
 
-    def _get_func_graph_proto(self, obj, exec_id, ir_type="onnx_ir", use_prefix=False):
+    def _get_func_graph_proto(self, obj, exec_id, ir_type="onnx_ir", use_prefix=False, incremental=False):
         """Get graph proto from pipeline."""
         if use_prefix:
             exec_id = exec_id + '.' + obj.arguments_key
         if self._graph_executor.has_compiled(exec_id) is False:
             return None
-        return self._graph_executor.get_func_graph_proto(exec_id, ir_type)
+        if self.obfuscate_config is not None:
+            branch_control_input = self._get_branch_control_input()
+            return self._graph_executor.get_obfuscate_func_graph_proto(exec_id, incremental,
+                                                                       self.obfuscate_config['obf_ratio'],
+                                                                       branch_control_input)
+        return self._graph_executor.get_func_graph_proto(exec_id, ir_type, incremental)
 
     def get_optimize_graph_proto(self, obj):
         """Return optimize graph binary proto."""
@@ -1261,12 +1523,6 @@ class _CellGraphExecutor:
         """
         self._graph_executor.export_graph(file_name, graph_id, encrypt_func, enc_key)
 
-    def fetch_info_for_quant_export(self, exec_id):
-        """Get graph proto from pipeline."""
-        if self._graph_executor.has_compiled(exec_id) is False:
-            return None
-        return self._graph_executor.fetch_info_for_quant_export(exec_id)
-
 
 def ms_memory_recycle():
     """
@@ -1276,16 +1532,45 @@ def ms_memory_recycle():
     To recycle these cached memory, users can call this function after training of one model.
     """
     if ms_compile_cache:
-        _cell_graph_executor.del_net_res(ms_compile_cache)
+        _cell_graph_executor.del_net_res(None, ms_compile_cache)
         ms_compile_cache.clear()
     for cell_cache in cells_compile_cache.values():
         if cell_cache:
-            _cell_graph_executor.del_net_res(cell_cache)
+            _cell_graph_executor.del_net_res(None, cell_cache)
             cell_cache.clear()
     _ms_memory_recycle()
 
 
+def _generate_branch_control_input(obf_random_seed):
+    """Generate append network input for dynamic obfuscation in random seed mode."""
+    seed_max = 2 ** 32 - 1
+    int_max = 2 ** 31 - 1
+    np.random.seed(obf_random_seed % seed_max)
+    # generate a string as hash function inputs
+    word_repo = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghigklmnopqrstuvwxyz" + "0123456789"
+    repo_len = len(word_repo)
+    sha_string = ''
+    string_len = 1024 * 1024
+    for _ in range(string_len):
+        rand_index = np.random.randint(0, repo_len)
+        sha_string += word_repo[rand_index]
+    # get hash result
+    sha_result = hashlib.sha256(sha_string.encode('utf-8')).hexdigest()  # len is 64
+    branch_control_input = 1
+    hex_base = 16
+    for item in sha_result:
+        if int(item, hex_base) > 0:
+            branch_control_input *= int(item, hex_base)
+    branch_control_input %= int_max
+    return branch_control_input
+
+
+def _bind_device_context():
+    """Bind device context to current thread"""
+    _bind_device_ctx()
+
+
 _cell_graph_executor = _CellGraphExecutor()
-_pynative_executor = _PynativeExecutor()
+_pynative_executor = _PyNativeExecutor()
 
-__all__ = ['ms_function', 'ms_memory_recycle', 'ms_class']
+__all__ = ['ms_function', 'ms_memory_recycle', 'ms_class', 'jit', 'jit_class']