mindspore 1.10.0__cp39-cp39-win_amd64.whl → 2.0.0rc1__cp39-cp39-win_amd64.whl

mindspore/common/dtype.py

@@ -17,9 +17,9 @@
 """Data type for MindSpore."""
 from __future__ import absolute_import
 
+import enum
 from inspect import isfunction
 import numpy as np
-from mindspore import log as logger
 from mindspore._c_expression import typing
 from mindspore._c_expression.typing import Type
 
@@ -46,11 +46,11 @@ __dtype__ = [
 ]
 
 __method__ = [
-    "dtype_to_nptype", "issubclass_", "dtype_to_pytype",
+    "dtype_to_nptype", "dtype_to_pytype",
     "pytype_to_dtype", "get_py_obj_dtype"
 ]
 
-__all__ = ["Type"]
+__all__ = ["Type", "QuantDtype"]
 __all__.extend(__dtype__)
 __all__.extend(__method__)
 
@@ -142,7 +142,10 @@ number_type = (int8,
 int_type = (int8, int16, int32, int64,)
 uint_type = (uint8, uint16, uint32, uint64,)
 float_type = (float16, float32, float64,)
-
+signed_type = (int8, byte, int16, short, int32, intc, int64,
+               intp, float16, half, float32, single, float64,
+               double, complex64, complex128)
+complex_type = (complex64, complex128,)
 all_types = (bool_, int8, uint8, int16, int32, int64, float16, float32, float64, complex64, complex128)
 implicit_conversion_seq = {t: idx for idx, t in enumerate(all_types)}
 
@@ -296,20 +299,6 @@ def _issubclass_(type_, dtype):
     return typing.is_subclass(type_, dtype)
 
 
-def issubclass_(type_, dtype):
-    """
-    Determine whether `type_` is a subclass of `dtype`.
-
-    Args:
-        type_ (:class:`mindspore.dtype`): Target MindSpore dtype.
-        dtype (:class:`mindspore.dtype`): Compare MindSpore dtype.
-
-    Returns:
-        bool, True or False.
-    """
-    logger.warning("'issubclass_' will be deprecated and removed in a future version.")
-    return _issubclass_(type_, dtype)
-
 
 def type_size_in_bytes(dtype):
     """
@@ -325,3 +314,62 @@ def type_size_in_bytes(dtype):
     if not isinstance(dtype, typing.Type):
         raise TypeError("The argument `dtype` should be instance of ", typing.Type)
     return typing.type_size_in_bytes(dtype)
+
+
+@enum.unique
+class QuantDtype(enum.Enum):
+    """
+    An enum for quant datatype, contains `INT1` ~ `INT16`, `UINT1` ~ `UINT16`.
+
+    `QuantDtype` is defined in `mindspore/common/dtype.py`, use command below to import:
+
+    .. code-block::
+
+        from mindspore import QuantDtype
+
+    """
+    INT1 = 0
+    INT2 = 1
+    INT3 = 2
+    INT4 = 3
+    INT5 = 4
+    INT6 = 5
+    INT7 = 6
+    INT8 = 7
+    INT9 = 8
+    INT10 = 9
+    INT11 = 10
+    INT12 = 11
+    INT13 = 12
+    INT14 = 13
+    INT15 = 14
+    INT16 = 15
+
+    UINT1 = 100
+    UINT2 = 101
+    UINT3 = 102
+    UINT4 = 103
+    UINT5 = 104
+    UINT6 = 105
+    UINT7 = 106
+    UINT8 = 107
+    UINT9 = 108
+    UINT10 = 109
+    UINT11 = 110
+    UINT12 = 111
+    UINT13 = 112
+    UINT14 = 113
+    UINT15 = 114
+    UINT16 = 115
+
+    def __str__(self):
+        return f"{self.name}"
+
+    def value(self) -> int:
+        """
+        Return value of `QuantDtype`.
+
+        Returns:
+            An int as value of `QuantDtype`.
+        """
+        return self._value_

mindspore/common/dump.py

@@ -27,11 +27,11 @@ def set_dump(target, enabled=True):
     `target` should be an instance of :class:`mindspore.nn.Cell` or :class:`mindspore.ops.Primitive` .
     Please note that this API takes effect only when Asynchronous Dump is enabled and the `dump_mode`
     field in dump config file is "2". See the `dump document <https://www.mindspore.cn/tutorials/
-    experts/en/r1.10/debug/dump.html>`_ for details. The default enabled status for
+    experts/en/r2.0/debug/dump.html>`_ for details. The default enabled status for
     a :class:`mindspore.nn.Cell` or :class:`mindspore.ops.Primitive` is False.
 
     .. warning::
-        This is an experimental prototype that is subject to change or deletion.
+        This is an experimental API that is subject to change or deletion.
 
     Note:
         1. This API is only effective for GRAPH_MODE with Ascend backend.

mindspore/common/initializer.py

@@ -258,6 +258,40 @@ def _calculate_in_and_out(arr):
     return n_in, n_out
 
 
+@_register('xavier_normal')
+class XavierNormal(Initializer):
+    r"""
+    Generates an array with values sampled from Xavier normal distribution
+    :math:`{N}(0, \text{sigma}^2)` in order to initialize a tensor, where
+
+    .. math::
+        sigma = gain * \sqrt{\frac{2}{n_{in} + n_{out}}}
+
+    where :math:`gain` is an optional scaling factor, :math:`n_{in}` is the number of input units in the weight tensor,
+    :math:`n_{out}` is the number of output units in the weight tensor.
+
+    Args:
+        gain (float): An optional scaling factor. Default: 1.
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore.common.initializer import initializer, XavierNormal
+        >>> tensor1 = initializer(XavierNormal(), [1, 2, 3], mindspore.float32)
+        >>> tensor2 = initializer('xavier_normal', [1, 2, 3], mindspore.float32)
+    """
+    def __init__(self, gain=1):
+        super().__init__(gain=gain)
+        self.gain = gain
+
+    def _initialize(self, arr):
+        fan_in, fan_out = _calculate_fan_in_and_fan_out(arr.shape)
+
+        std = self.gain * math.sqrt(2.0 / float(fan_in + fan_out))
+        data = _init_random_normal(0, std, arr.shape)
+
+        _assignment(arr, data)
+
+
 @_register('xavier_uniform')
 class XavierUniform(Initializer):
     r"""
@@ -576,7 +610,9 @@ class VarianceScaling(Initializer):
     When `distribution` is 'truncated_normal' or 'untruncated_normal', the value will be sampled from truncated or
     untruncated normal distribution with a mean of 0 and a scaled standard deviation
     :math:`stddev = \sqrt{\frac{scale}{n}}`. :math:`n` will be the number of input units if `mode` is 'fan_in',
-    the number of output units if `mode` is 'fan_out', the average of 'fan_in' and 'fan_out' if `mode` is 'fan_avg'.
+    while :math:`n` will be
+    the number of output units if `mode` is 'fan_out'. :math:`n` will be the average of 'fan_in' and 'fan_out'
+    if `mode` is 'fan_avg'.
     When `distribution` is 'uniform', the value will be sampled from a uniform distribution within the limit of
     :math:`[-\sqrt{\frac{3*scale}{n}}, \sqrt{\frac{3*scale}{n}}]`.
 
@@ -803,6 +839,7 @@ __all__ = [
     'HeUniform',
     'HeNormal',
     'XavierUniform',
+    'XavierNormal',
     'One',
     'Zero',
     'Constant',

mindspore/common/jit_config.py

@@ -19,27 +19,39 @@ class JitConfig:
     """
     Jit config for compile.
 
-    Note:
-        This is an experimental function that is subject to change or deletion.
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
 
     Args:
         jit_level (str): Option for argument `level` for Optimization of lift graph.
-            Supports ["O0", "O1", "O2"]. Default: "O1".
+            Supports ["O0", "O1", "O2", "O3"]. Default: "O1".
 
             - "O0": Basic optimization.
             - "O1": Manual optimization.
             - "O2": Manual optimization and graph computation fusion.
+            - "O3": Performance optimization, no generalization guaranteed.
+
+        exc_mode (str): Mode for execute the network. Supports ["auto", "sink", "no_sink"]. Default: "auto".
+
+            - "auto": Automatic Policies.
+            - "sink": Build computational graphs with the sink mode.
+            - "no_sink": Build computational graphs with no sink mode.
 
-        task_sink (bool): Determines whether to pass the data through dataset channel. Default: True.
         **kwargs (dict): A dictionary of keyword arguments that the class needs.
+
+    Examples:
+        >>> from mindspore import JitConfig
+        >>>
+        >>> jitconfig = JitConfig(jit_level="O1")
+        >>> net = LeNet5()
+        >>>
+        >>> net.set_jit_config(jitconfig)
     """
-    def __init__(self, jit_level="O1", task_sink=True, **kwargs):
-        if jit_level not in ["O0", "O1", "O2"]:
-            raise ValueError("For 'jit_level' must be one of ['O0', 'O1', 'O2'].")
-        if not isinstance(task_sink, bool):
-            raise TypeError("For 'task_sink' must be bool.")
-        self.jit_config_dict = dict()
+    def __init__(self, jit_level="O1", exc_mode="auto", **kwargs):
+        if jit_level not in ["O0", "O1", "O2", "O3"]:
+            raise ValueError("For 'jit_level' must be one of ['O0', 'O1', 'O2', 'O3'].")
+        if exc_mode not in ['auto', 'sink', 'no_sink']:
+            raise ValueError("For 'exc_mode' must be one of '['auto', 'sink', 'no_sink']'.")
+        self.jit_config_dict = kwargs
         self.jit_config_dict["jit_level"] = jit_level
-        self.jit_config_dict["task_sink"] = str(int(task_sink))
-        for key, value in kwargs.items():
-            self.jit_config_dict[key] = value
+        self.jit_config_dict["exc_mode"] = exc_mode

mindspore/common/mutable.py

@@ -19,6 +19,14 @@ from mindspore.common.tensor import Tensor
 from mindspore._c_expression import Tensor as Tensor_
 
 
+class _Int(int):
+    pass
+
+
+class _Float(float):
+    pass
+
+
 class _Tuple(tuple):
     pass
 
@@ -31,22 +39,22 @@ class _Dict(dict):
     pass
 
 
-def _check_all_tensor(value):
+def _check_element_type(value):
     """Check if all the elements are Tensor."""
     if isinstance(value, (tuple, list)):
         for element in value:
-            if not _check_all_tensor(element):
+            if not _check_element_type(element):
                 return False
         return True
     if isinstance(value, dict):
         for element in value.values():
-            if not _check_all_tensor(element):
+            if not _check_element_type(element):
                 return False
         return True
-    return isinstance(value, Tensor_)
+    return isinstance(value, (Tensor, Tensor_, int, float)) and not isinstance(value, bool)
 
 
-def mutable(input_data):
+def mutable(input_data, dynamic_len=False):
     """
     Make a constant value mutable.
 
@@ -61,22 +69,33 @@ def mutable(input_data):
     input means that it is changed to be a variable input just like Tensor and the most important thing is that it
     will be differentiable.
 
+    When the `input_data` is tuple or list and `dynamic_len` is False, `mutable` will return a constant length tuple
+    or list with all mutable elements. If `dynamic_len` is True, the length of the return tuple or list will be dynamic.
+
+    If a dynamic length tuple or list is used as the input of the network and the network is repeatedly called, and
+    the length of the tuple or list is different for each run, it does not need to be re-compiled.
+
     Args:
-        input_data (Union[Tensor, tuple[Tensor], list[Tensor], dict[Tensor]]): The input data to be made mutable.
+        input_data (Union[int, float, Tensor, tuple, list, dict]): The input data to be made mutable. If
+            'input_data' is list/tuple/dict, the type of each element should also in the valid types.
+        dynamic_len (bool): Whether to set the whole sequence to be dynamic length. In graph compilation, if
+            `dynamic_len` is True, the `input_data` must be list or tuple and the elements of `input_data` must have
+            the same type and shape. Default: False.
 
     .. warning::
-        - This is an experimental prototype that is subject to change or deletion.
-        - The runtime has not yet supported to handle the scalar data flow. So we only support tuple[Tensor],
-          list[Tensor] or dict[Tensor] for network input to avoid the re-compiled problem now.
-        - Currently we only support to use this api outside the network temporarily.
-        - Currently this api only works in GRAPH mode.
+        This is an experimental API that is subject to change or deletion.
+
+    Note:
+        Currently this api only works in GRAPH mode.
 
     Returns:
         The origin input data which has been set mutable.
 
     Raises:
-        TypeError: If `input_data` is not one of Tensor, tuple[Tensor], list[Tensor], dict[Tensor] or their nested
-            structure.
+        TypeError: If `input_data` is not one of int, float, Tensor, tuple, list, dict or their nested structure.
+        TypeError: If `dynamic_len` is True and `input_data` is not tuple or list.
+        ValueError: If `dynamic_len` is True, `input_data` is tuple or list but the elements within `input_data` do not
+            have the same shape and type.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -84,10 +103,11 @@ def mutable(input_data):
     Examples:
         >>> import mindspore.nn as nn
         >>> import mindspore.ops as ops
-        >>> from mindspore.ops.composite import GradOperation
         >>> from mindspore.common import mutable
         >>> from mindspore.common import dtype as mstype
         >>> from mindspore import Tensor
+        >>> from mindspore import context
+        >>> context.set_context(mode=context.GRAPH_MODE)
         >>> class Net(nn.Cell):
         ...     def __init__(self):
         ...         super(Net, self).__init__()
@@ -103,7 +123,7 @@ def mutable(input_data):
         ...     def __init__(self, net):
         ...         super(GradNetWrtX, self).__init__()
         ...         self.net = net
-        ...         self.grad_op = GradOperation()
+        ...         self.grad_op = ops.GradOperation()
         ...
         ...     def construct(self, z):
         ...         gradient_function = self.grad_op(self.net)
@@ -114,20 +134,28 @@ def mutable(input_data):
         >>> output = GradNetWrtX(Net())(z)
         >>> print(output)
         (Tensor(shape=[2, 3], dtype=Float32, value=
-        [[ 1.41000009e+00, 1.60000002e+00, 6.59999943e+00],
-         [ 1.41000009e+00, 1.60000002e+00, 6.59999943e+00]]), Tensor(shape=[3, 3], dtype=Float32, value=
-        [[ 1.70000005e+00, 1.70000005e+00, 1.70000005e+00],
-         [ 1.89999998e+00, 1.89999998e+00, 1.89999998e+00],
-         [ 1.50000000e+00, 1.50000000e+00, 1.50000000e+00]]))
+        [[ 1.41000009e+00,  1.60000002e+00,  6.59999943e+00],
+         [ 1.41000009e+00,  1.60000002e+00,  6.59999943e+00]]), Tensor(shape=[3, 3], dtype=Float32, value=
+        [[ 1.70000005e+00,  1.70000005e+00,  1.70000005e+00],
+         [ 1.89999998e+00,  1.89999998e+00,  1.89999998e+00],
+         [ 1.50000000e+00,  1.50000000e+00,  1.50000000e+00]]))
     """
-
-    if not _check_all_tensor(input_data):
+    if not _check_element_type(input_data):
         raise TypeError(
-            f"For 'mutable', the 'input_data' should be one of (Tensor, tuple[Tensor], list[Tensor], dict[Tensor]) "
+            f"For 'mutable', the 'input_data' should be one of (int, float, bool, Tensor, tuple, list, dict) "
             f"or their nested structures, but got {input_data}.")
 
+    if dynamic_len and not isinstance(input_data, (tuple, list)):
+        raise TypeError(
+            f"For mutable, when the variable_len is True, the first input should be list or tuple,"
+            f" but got {input_data}")
+
     ret = input_data
-    if isinstance(input_data, list):
+    if isinstance(input_data, int):
+        ret = _Int(input_data)
+    elif isinstance(input_data, float):
+        ret = _Float(input_data)
+    elif isinstance(input_data, list):
         ret = _List(input_data)
     elif isinstance(input_data, tuple):
         ret = _Tuple(input_data)
@@ -140,4 +168,5 @@ def mutable(input_data):
         ret.set_const_arg(False)
 
     setattr(ret, "__ms_mutable__", True)
+    setattr(ret, "__ms_dynamic_len__", dynamic_len)
     return ret

mindspore/common/parameter.py

@@ -18,6 +18,7 @@ from __future__ import absolute_import
 
 from copy import copy
 import sys
+import math
 import numbers
 import numpy as np
 from mindspore import log as logger
@@ -26,17 +27,17 @@ from mindspore._c_expression import ParamInfo
 from mindspore.common import dtype as mstype
 from mindspore import context
 from mindspore.parallel._utils import _get_parallel_mode
+from mindspore.common._utils import get_slice_num, get_slice_shape
 from mindspore.common.initializer import initializer
 from mindspore.common.tensor import Tensor
-from mindspore._checkparam import Validator
+from mindspore import _checkparam as Validator
+from mindspore._check_jit_forbidden_api import jit_forbidden_register
 from mindspore._c_expression import Tensor as Tensor_
 from mindspore.parallel._tensor import _get_slice_index
 from mindspore.parallel._auto_parallel_context import auto_parallel_context
 from mindspore.parallel._ps_context import _is_role_worker, _is_role_pserver, _is_role_sched, _clone_hash_table, \
-    _enable_distributed_mindrt, _is_ps_mode
-from mindspore.parallel._ps_context import _reinsert_hash_table_size
-from mindspore.parallel._ps_context import _insert_weight_init_info, _insert_accumu_init_info
-from mindspore.common.seed import _get_global_and_op_seed
+                                           _is_ps_mode
+from mindspore.parallel._ps_context import _reinsert_hash_table_size, _insert_accumu_init_info, _cache_enable
 import mindspore.common._monad as monad
 
 __all__ = ['Parameter', 'ParameterTuple']
@@ -220,10 +221,24 @@ class Parameter(Tensor_):
         self.is_in_parallel = _is_in_parallel_mode()
         self.is_in_shard = False
         self._pipeline_stage_list = []
+        self.slice_num = 1
         if -1 in self.shape:
             raise ValueError(f"All shape elements of the Parameter must be positive. But got None.")
         if isinstance(default_input, (Tensor_, Tensor)):
-            Tensor_.__init__(self, default_input.dtype, default_input.shape)
+            # At embedding cache scenes, we need limit the size of memory for parameter.
+            # And save out range data to persistent storage to support TB-Level size parameter.
+            slice_num_of_persistent_data = get_slice_num(default_input.dtype, default_input.shape)
+            if slice_num_of_persistent_data > 1:
+                data_shape = list(default_input.shape)
+                slice_first_dim = math.ceil(data_shape[0] / slice_num_of_persistent_data)
+                data_shape[0] = slice_first_dim
+                self.param_info.use_persistent_storage = True
+                self.param_info.origin_shape = default_input.shape
+                self.slice_num = slice_num_of_persistent_data
+                Tensor_.__init__(self, default_input.dtype, tuple(data_shape))
+            else:
+                Tensor_.__init__(self, default_input.dtype, default_input.shape)
+
         elif isinstance(default_input, int):
             Tensor_.__init__(self, mstype.int64, ())
         elif isinstance(default_input, float):
@@ -234,6 +249,7 @@ class Parameter(Tensor_):
             raise TypeError(f"The type of the argument 'default_input' must be in ['Tensor', 'int', 'float',"
                             f" 'numpy.ndarray', 'list']. But got type {type(default_input)}.")
         self.param_info.parameter_shape = self.shape
+
         import mindspore.ops.operations.other_ops as other_ops
         self.load = other_ops.Load()
 
@@ -263,13 +279,6 @@ class Parameter(Tensor_):
             Parameter._base_type[input_class_name] = new_type
         return new_type
 
-    @staticmethod
-    def _not_init_data():
-        is_worker_or_server = (_is_role_worker() or _is_role_pserver()) and not _enable_distributed_mindrt()
-        if is_worker_or_server or _is_role_sched() or _is_in_parallel_mode():
-            return True
-        return False
-
     @staticmethod
     def _get_parameter_new_args(data, rc):
         """Set `set_data` of current `Parameter`."""
@@ -284,11 +293,10 @@ class Parameter(Tensor_):
                 # make a copy of Tensor to init the parameter.
                 return (Tensor, data.asnumpy())
 
-            is_worker_or_server = (_is_role_worker() or _is_role_pserver()) and not _enable_distributed_mindrt()
-            not_init_data = is_worker_or_server or _is_role_sched() or _is_in_parallel_mode()
+            not_init_data = _is_role_sched() or (_is_role_pserver() and _cache_enable()) or _is_in_parallel_mode()
             if not_init_data:
                 # do not init data while in auto parallel.
-                return (Tensor, None, data.dtype, data.shape, data.init)
+                return (Tensor, None, data.dtype, get_slice_shape(data.dtype, data.shape), data.init)
             return (Tensor, data.init_data())
         if isinstance(data, int):
             return (Tensor, data, mstype.int32)
@@ -303,6 +311,7 @@ class Parameter(Tensor_):
 
         Note:
             It only works when a running task is in the parameter server mode.
+            It is supported only in graph mode.
 
         Args:
             init_in_server (bool): Whether trainable parameter updated by parameter server is
@@ -313,10 +322,23 @@ class Parameter(Tensor_):
                                "1. context.set_ps_context(enable_ps=True) \n"
                                "2. export MS_ROLE environment variable \n"
                                "Please refer to the official website for detailed usage.")
+
+        if context.get_context("mode") == context.PYNATIVE_MODE:
+            raise RuntimeError("Parameter server training is not supported in pynative mode currently."
+                               "Please switch to graph mode and retry.")
         self.is_param_ps = True
         self.init_in_server = init_in_server
         self.param_info.init_in_server = init_in_server
 
+    def copy(self):
+        """
+        Copy the parameter.
+
+        Returns:
+            Parameter, a new parameter.
+        """
+        return self.clone(init='same')
+
     def set_param_fl(self, push_to_server=False, pull_from_server=False, requires_aggr=True):
         """
         Set the way of parameter and server interaction.
@@ -464,13 +486,15 @@ class Parameter(Tensor_):
             Parameter, a new parameter.
         """
         x = copy(self)
-        x.param_info = self.param_info.clone()
+        param_info_clone = self.param_info.clone()
         info = self.param_info
         if hasattr(info, "cloned_obj"):
             info.cloned_obj.append(x)
         else:
             info.cloned_obj = [x]
         self.param_info = info
+        param_info_clone.obj = x
+        x.param_info = param_info_clone
         x.is_init = False
         x.init = self.init
         x.is_param_ps = self.is_param_ps
@@ -482,7 +506,7 @@ class Parameter(Tensor_):
         if self.cache_shape:
             x.cache_shape = self.cache_shape
         if init != 'same':
-            shape = self.shape
+            shape = self.shape if self.slice_num == 1 else self.param_info.origin_shape
             dtype = self.dtype
             x.set_data(initializer(init, shape=shape, dtype=dtype))
         return x
@@ -573,14 +597,15 @@ class Parameter(Tensor_):
 
     def value(self):
         """
-        Return: the value of parameter object.
+        Return the value of parameter object.
+
         Examples:
-        >>> from mindspore import Tensor, Parameter
-        >>> import numpy as np
-        >>> x = Parameter(Tensor(np.array([1, 2], dtype=np.float32)), name="param")
-        >>> x_value = x.value()
-        >>> print(x_value)
-        [1.  2.]
+            >>> from mindspore import Tensor, Parameter
+            >>> import numpy as np
+            >>> x = Parameter(Tensor(np.array([1, 2], dtype=np.float32)), name="param")
+            >>> x_value = x.value()
+            >>> print(x_value)
+            [1.  2.]
         """
         return self.load(self, monad.U)
 
@@ -608,13 +633,13 @@ class Parameter(Tensor_):
 
     @staticmethod
     def _set_data_check_input_valid(current_shape, data_shape, current_tensor_is_init,
-                                    incoming_tensor_is_init, slice_shape=False):
+                                    incoming_tensor_is_init, slice_shape=False, slice_num=1):
         if incoming_tensor_is_init and not current_tensor_is_init:
             raise TypeError("The original tensor data is initialized, but the argument 'data' is not initialized."
                             "Please initialize 'data' before call this method.")
         if tuple(current_shape) != tuple(data_shape):
             # If Slice create Parameter shape can be change.
-            if not slice_shape:
+            if not slice_shape and slice_num == 1:
                 raise ValueError(f"Can not change the shape of Parameter which has been initialized."
                                  f" Current shape is {current_shape}, and incoming is {data_shape}.")
 
@@ -631,6 +656,7 @@ class Parameter(Tensor_):
         Parameter.__init__(param, tensor, *args, **kwargs)
         return param
 
+    @jit_forbidden_register
     def set_data(self, data, slice_shape=False):
         """
         Set Parameter's data.
@@ -654,7 +680,7 @@ class Parameter(Tensor_):
         incoming_tensor_is_init = isinstance(data, Tensor) and not data.has_init
         current_tensor_is_init = isinstance(self, Tensor) and not self.has_init
         Parameter._set_data_check_input_valid(self.shape, data.shape, current_tensor_is_init, incoming_tensor_is_init,
-                                              slice_shape)
+                                              slice_shape, self.slice_num)
         if self.dtype != data.dtype:
             if mstype.implicit_conversion_seq[self.dtype] < mstype.implicit_conversion_seq[data.dtype]:
                 self._raise_type_error(data.dtype)
@@ -686,7 +712,7 @@ class Parameter(Tensor_):
                 raise TypeError("The argument 'layout' should be tuple, but got {}.".format(type(layout)))
             if len(layout) < 6:
                 raise ValueError("The length of 'layout' must be larger than 5, but got {}.".format(len(layout)))
-            slice_index = int(_get_slice_index(layout[0], layout[1]))
+            slice_index = int(_get_slice_index(layout[0], layout[1], layout[5]))
             init_data_args += (slice_index, layout[2], layout[5])
         return init_data_args
 
@@ -719,22 +745,18 @@ class Parameter(Tensor_):
             TypeError: If `layout` is not tuple.
         """
         if self.is_default_input_init and self.is_in_parallel != _is_in_parallel_mode():
-            raise RuntimeError("Must set or change parallel mode before any Tensor created.")
+            raise RuntimeError("Must set or change parallel mode before any initializer Tensor created.")
         if self.init_mode is None:
             return self
         if self.inited_param is not None:
             return self.inited_param
-        if _is_role_worker() and self.cache_enable:
-            global_seed, op_seed = _get_global_and_op_seed()
-            _insert_weight_init_info(self.name, global_seed, op_seed)
 
         init_data_args = self._get_init_data_args(layout)
 
-        if _is_role_pserver() and not _enable_distributed_mindrt():
+        if _is_role_sched():
             return self
-
         if self.init_in_server and self.is_param_ps and isinstance(self.init_mode, Tensor) and \
-                self.init_mode.init is not None and (_is_role_worker() or _is_role_sched()):
+                self.init_mode.init is not None and _is_role_worker():
             if self.cache_enable:
                 data = self.init_mode.init_data(*init_data_args)
             else:
@@ -789,10 +811,11 @@ class ParameterTuple(tuple):
 
                 - If `init` is a `Tensor` , set the new Parameter data to the input Tensor.
                 - If `init` is `numbers.Number` , set the new Parameter data to the input number.
-                - If `init` is a `str`, data will be seted according to the initialization method of the same name in
+                - If `init` is a `str`, data will be set according to the initialization method of the same name in
                   the `Initializer`.
                 - If `init` is 'same', the new Parameter has the same value with the original Parameter.
 
+
         Returns:
             Tuple, the new Parameter tuple.
         """