mindspore 2.3.0__cp39-cp39-win_amd64.whl → 2.4.1__cp39-cp39-win_amd64.whl

mindspore/common/parameter.py

@@ -22,6 +22,7 @@ import os
 import sys
 import math
 import numbers
+from contextlib import contextmanager
 import numpy as np
 from mindspore import log as logger
 from mindspore.log import _LogActionOnce
@@ -41,6 +42,8 @@ from mindspore.parallel._ps_context import _is_role_worker, _is_role_pserver, _i
                                            _is_ps_mode
 from mindspore.parallel._ps_context import _reinsert_hash_table_size, _insert_accumu_init_info, _cache_enable
 from mindspore.common._decorator import deprecated
+from mindspore.communication._comm_helper import _is_initialized
+from mindspore.communication import get_group_size
 import mindspore.common._monad as monad
 
 __all__ = ['Parameter', 'ParameterTuple']
@@ -52,11 +55,32 @@ PARAMETER_NAME_PREFIX_MAX_LEN = 1024
 _GLOBAL_PARAMETER_KEY = -1
 
 
-def _is_in_parallel_mode():
+@contextmanager
+def no_init_parameters():
+    init_class = globals()["Parameter"]
+    setattr(init_class, "init_param", False)
+    try:
+        yield
+    finally:
+        setattr(init_class, "init_param", True)
+
+
+def _is_in_auto_parallel_mode():
     """Get parallel mode."""
     return auto_parallel_context().get_parallel_mode() in ["semi_auto_parallel", "auto_parallel"]
 
 
+def _is_parallel_mode():
+    """ Whether is parallel mode """
+    if not _is_initialized() or context.get_context('mode') == context.PYNATIVE_MODE:
+        return False
+    if os.getenv("RUN_MODE") != "predict":
+        return False
+    if get_group_size() > 1 and _get_parallel_mode() == "stand_alone":
+        return True
+    return False
+
+
 def init_to_value(init):
     """
     Get value of initializer.
@@ -91,6 +115,15 @@ def _get_unique_parameter_key():
     return _GLOBAL_PARAMETER_KEY
 
 
+def _gen_offload_file_path(offload_dir):
+    offload_dir = os.path.relpath(offload_dir)
+    if not os.path.exists(offload_dir):
+        os.makedirs(offload_dir, mode=0o700, exist_ok=True)
+    offload_file_path = offload_dir + "/" + str(_get_global_rank()) + "_" + str(
+        _get_unique_parameter_key()) + "_" + str(time.time()) + ".data"
+    return offload_file_path
+
+
 def _offload_if_config(data):
     """
     Offload parameter(data size > 512) to file when enable memory offload and offload parameter to disk.
@@ -111,11 +144,7 @@ def _offload_if_config(data):
     offload_file_path = data.offload_file_path()
     if offload_file_path is None or offload_file_path == "":
         offload_dir = offload_context.get("offload_path", "./offload")
-        offload_dir = os.path.relpath(offload_dir)
-        if not os.path.exists(offload_dir):
-            os.makedirs(offload_dir)
-        offload_file_path = offload_dir + "/" + str(_get_global_rank()) + "_" + str(
-            _get_unique_parameter_key()) + "_" + str(time.time()) + ".data"
+        offload_file_path = _gen_offload_file_path(offload_dir)
     data.offload(offload_file_path)
 
 
@@ -191,6 +220,12 @@ class Parameter(Tensor_):
         storage_format (str): Only Ascend device target is supported. It is used to specify the format of the weight
             loaded to the device. By default, the format is not changed. The optional values are ``"FRACTAL_NZ"`` ,
             ``"NC1HWC0"`` , ``"FRACTAL_Z"`` , etc. Default: ``""`` .
+        device(str): Only Ascend device target is supported. It is used to specify the device which the parameter is
+            stored. By default, the parameter will be stored on NPU while computing. When the device is specified as
+            ``"CPU"``, the parameter will be loaded into the device when it needs to be used, and unloaded to the CPU
+            after use. It takes effext only when `memory_offload` is ``"ON"``, `jit_level` is not ``"O2"`` and
+            `memory_optimize_level` is ``O0`` in `mindspore.set_context()`. Less device memory is needed when device is
+            specified as ``"CPU"``.
 
     Examples:
         >>> import numpy as np
@@ -219,7 +254,8 @@ class Parameter(Tensor_):
     def __new__(cls, default_input, *args, **kwargs):
         init_data_flag = bool(isinstance(default_input, Tensor) and default_input.has_init)
         rc = sys.getrefcount(default_input)
-        input_class, *class_init_args = Parameter._get_parameter_new_args(default_input, rc)
+        init_param = getattr(cls, "init_param", True)
+        input_class, *class_init_args = Parameter._get_parameter_new_args(default_input, rc, init_param)
         new_type = Parameter._get_base_class(input_class)
         obj = input_class.__new__(new_type)
         input_class.__init__(obj, *class_init_args)
@@ -244,7 +280,7 @@ class Parameter(Tensor_):
             Parameter, (data, self.name, self.requires_grad, self.layerwise_parallel))
 
     def __init__(self, default_input, name=None, requires_grad=True, layerwise_parallel=False, parallel_optimizer=True,
-                 storage_format=""):
+                 storage_format="", device=None):
         self.param_info = ParamInfo()
         self.init_in_server = False
         self.name = name
@@ -263,7 +299,7 @@ class Parameter(Tensor_):
         self.requires_aggr = True
         self._cast_type = None
         self._unique = False
-        self.is_in_parallel = _is_in_parallel_mode()
+        self.is_in_parallel = _is_in_auto_parallel_mode()
         self.is_in_shard = False
         self._pipeline_stage_list = []
         self.slice_num = 1
@@ -296,6 +332,10 @@ class Parameter(Tensor_):
                             f" 'numpy.ndarray', 'list']. But got type {type(default_input)}.")
         self.param_info.parameter_shape = self.shape
         self.param_info.storage_format = storage_format
+        if device is not None:
+            if device != "CPU":
+                raise ValueError(f"Only 'CPU' is supported for device, but got ${device}.")
+            self._set_user_data("parameter_device", device)
 
         import mindspore.ops.operations.other_ops as other_ops
         self.load = other_ops.Load()
@@ -327,7 +367,7 @@ class Parameter(Tensor_):
         return new_type
 
     @staticmethod
-    def _get_parameter_new_args(data, rc):
+    def _get_parameter_new_args(data, rc, init_param=True):
         """Set `set_data` of current `Parameter`."""
         if isinstance(data, bool):
             raise ValueError('Parameter data can not be `bool`')
@@ -342,7 +382,8 @@ class Parameter(Tensor_):
                     return (Tensor, data.asnumpy(), mstype.qint4x2)
                 return (Tensor, data.asnumpy())
 
-            not_init_data = _is_role_sched() or (_is_role_pserver() and _cache_enable()) or _is_in_parallel_mode()
+            not_init_data = not init_param or _is_role_sched() or (_is_role_pserver() and _cache_enable()) \
+                            or _is_in_auto_parallel_mode() or _is_parallel_mode()
             if not_init_data:
                 # do not init data while in auto parallel.
                 return (Tensor, None, data.dtype, get_slice_shape(data.dtype, data.shape), data.init)
@@ -368,7 +409,7 @@ class Parameter(Tensor_):
 
         Tutorial Examples:
             - `Parameter Server Mode
-              <https://www.mindspore.cn/tutorials/experts/en/master/parallel/parameter_server_training.html>`_
+              <https://www.mindspore.cn/docs/en/master/model_train/parallel/parameter_server_training.html>`_
         """
         if not _is_ps_mode() or not (_is_role_worker() or _is_role_pserver() or _is_role_sched()):
             raise RuntimeError("Must complete following two steps before calling set_param_ps: \n"
@@ -616,6 +657,9 @@ class Parameter(Tensor_):
             shape = self.shape if self.slice_num == 1 else self.param_info.origin_shape
             dtype = self.dtype
             x.set_data(initializer(init, shape=shape, dtype=dtype))
+        device = self._get_user_data("parameter_device")
+        if device is not None:
+            x._set_user_data("parameter_device", device)
         return x
 
     @property
@@ -942,8 +986,10 @@ class Parameter(Tensor_):
             >>> x = Parameter(Tensor(np.array([[1, 2], [3, 4]], dtype=np.float32)), name="param")
             >>> x.init_data()
         """
-        if self.is_default_input_init and self.is_in_parallel != _is_in_parallel_mode():
+        if self.is_default_input_init and self.is_in_parallel != _is_in_auto_parallel_mode():
             raise RuntimeError("Must set or change parallel mode before any initializer Tensor created.")
+        if hasattr(self, "init_param") and self.init_param:
+            return self
         if self.init_mode is None:
             return self
         if self.inited_param is not None:
@@ -1026,8 +1072,9 @@ class ParameterTuple(tuple):
             Tuple, the new Parameter tuple.
 
         Tutorial Examples:
-            - `Cell and Parameter - Parameter Tuple
-              <https://mindspore.cn/tutorials/en/master/advanced/modules/layer.html#parameter-tuple>`_
+            - `Tensor and Parameter - Parameter Tuple
+              <https://mindspore.cn/docs/en/master/model_train/model_building/tensor_and_parameter.html
+              #parameter-tuple>`_
         """
         Validator.check_str_by_regular(prefix)
         new = []

mindspore/common/recompute.py

@@ -23,8 +23,10 @@ from mindspore.common.tensor import Tensor
 from mindspore import ops
 from mindspore.ops.composite import GradOperation
 from mindspore.common._register_for_recompute import recompute_registry
-from mindspore.common.api import _pynative_executor
+from mindspore.common.api import _pynative_executor, _no_grad
 from mindspore.common.generator import get_rng_state, set_rng_state
+from mindspore.train.amp import amp_decorator
+from mindspore._c_expression.amp import get_curr_amp_strategy
 
 
 class _WrapCell(Cell):
@@ -34,7 +36,7 @@ class _WrapCell(Cell):
     """
 
     def __init__(self, function):
-        super(_WrapCell, self).__init__()
+        super(_WrapCell, self).__init__(auto_prefix=False)
         self.function = function
 
     def construct(self, *args, **kwargs):
@@ -56,6 +58,7 @@ class _RecomputeCell(Cell):
         self.args = []
         self.kwargs = []
         self.wrap_cell = _WrapCell(block)
+        self.wrap_cell.set_inputs()
 
         self.net = block
         self.internal_params = []
@@ -64,15 +67,18 @@ class _RecomputeCell(Cell):
         self._add_attr("is_cell_recompute", "True")
         self.grad = GradOperation(get_all=True, get_by_list=True, sens_param=True)
         self.init_mixed_precision_type(block)
+        self.amp_strategy = None
 
     def construct(self, *args, **kwargs):
-        _check_input_args_validate(self.net, args)
+        _check_input_args_validate(self.net, args, kwargs)
         self.args.append(args)
         self.kwargs.append(kwargs)
         self.save_rng_state = kwargs.pop("save_rng_state", True)
         if self.save_rng_state:
             self.cpu_rng_state = get_rng_state()
-        return self.net(*args, **kwargs)
+        self.amp_strategy = get_curr_amp_strategy()
+        with _no_grad():
+            return self.net(*args, **kwargs)
 
     def bprop(self, *args):
         """
@@ -86,14 +92,23 @@ class _RecomputeCell(Cell):
         self.args.pop()
         self.kwargs.pop()
         if kwargs:
-            input_args = list(input_args) + list(kwargs.values())
+            input_args_for_check = list(input_args) + list(kwargs.values())
+        else:
+            input_args_for_check = list(input_args)
         # To detach inputs to avoid erasing auto grad meta info of origin inputs.
         input_args = _detach_input(input_args)
+        kwargs = _detach_input(kwargs)
+        kwargs['sens'] = grad_input
         try:
             pre_rng_state = get_rng_state()
             set_rng_state(self.cpu_rng_state)
             _pynative_executor.set_is_run_recompute(True)
-            grads = self.grad(self.net, self.internal_params)(*input_args, grad_input)
+            if self.amp_strategy:
+                with amp_decorator(self.amp_strategy.get_amp_level(), self.amp_strategy.get_amp_dtype(),
+                                   self.amp_strategy.get_white_list(), self.amp_strategy.get_black_list()):
+                    grads = self.grad(self.net, self.internal_params)(*input_args, **kwargs)
+            else:
+                grads = self.grad(self.net, self.internal_params)(*input_args, **kwargs)
             _pynative_executor.set_is_run_recompute(False)
             set_rng_state(pre_rng_state)
         except Exception as err:
@@ -101,7 +116,7 @@ class _RecomputeCell(Cell):
             raise err
         weights = OrderedDict()
         input_grads = list(grads[0])
-        _padding_input_grads(input_args, input_grads)
+        _padding_input_grads(input_args_for_check, input_grads)
         for i, param in enumerate(self.internal_params):
             weights[param] = grads[1][i]
         return tuple(input_grads), weights
@@ -121,6 +136,7 @@ class _RecomputeCell(Cell):
             # To avoid sub cell same name
             block.__self__.check_names_and_refresh_name()
             self.internal_params = block.__self__.trainable_params()
+            self.wrap_cell.mixed_precision_type = block.__self__.get_mixed_precision_type()
             self.wrap_cell.set_mixed_precision_type(block.__self__.get_mixed_precision_type())
             self.net = self.wrap_cell
         else:
@@ -128,13 +144,14 @@ class _RecomputeCell(Cell):
                             "only support Cell object or MethodType function!")
 
 
-def _check_input_args_validate(block, args):
+def _check_input_args_validate(block, args, kwargs):
     """
     Check recompute input args validate
     :param args:
     :return:
     """
-    if not any([isinstance(arg, Tensor) for arg in args]):
+    if not (any([isinstance(arg, Tensor) for arg in args]) or \
+        any([isinstance(arg, Tensor) for arg in kwargs.values()])):
         logger.warning("None of the inputs of function are tensors, which not need use recompute!")
     for arg in args:
         if isinstance(arg, (tuple, list)):
@@ -168,6 +185,11 @@ def _padding_input_grads(args, input_grads):
 
 
 def _detach_input(input_arg):
+    """
+    Detach input
+    :param input_arg:
+    :return: detach output
+    """
     if isinstance(input_arg, Tensor):
         return ops.stop_gradient(input_arg)
     if isinstance(input_arg, (list, tuple)):
@@ -175,6 +197,14 @@ def _detach_input(input_arg):
         for arg in input_arg:
             detach_inputs.append(_detach_input(arg))
         return detach_inputs if isinstance(input_arg, list) else tuple(detach_inputs)
+    if isinstance(input_arg, dict):
+        detach_inputs = {}
+        for key, val in input_arg.items():
+            if isinstance(val, Tensor):
+                detach_inputs[key] = ops.stop_gradient(val)
+            else:
+                detach_inputs[key] = val
+        return detach_inputs
     return input_arg
 
 

mindspore/common/sparse_tensor.py

@@ -97,7 +97,8 @@ class RowTensor(RowTensorInner):
          [0, 0]]
 
     .. warning::
-        This is an experimental API that is subjected to change or deletion.
+        - This is an experimental API that is subjected to change or deletion.
+        - If use PyNative mode, set "export MS_PYNATIVE_CONFIG_STATIC_SHAPE=1".
 
     Args:
         indices (Tensor): A 1-D integer Tensor of shape :math:`(d_0)` . Default: ``None``.
@@ -226,10 +227,11 @@ class COOTensor(COOTensor_):
 
     Common arithmetic operations include: addition (+), subtraction (-), multiplication (*),
     and division (/). For details about operations supported by `COOTensor`, see
-    `operators <https://www.mindspore.cn/docs/en/master/note/static_graph_syntax_support.html#operators>`_.
+    `operators <https://www.mindspore.cn/docs/en/master/model_train/program_form/static_graph.html#operators>`_.
 
     .. warning::
         - This is an experimental API that is subject to change or deletion.
+        - If use PyNative mode, set "export MS_PYNATIVE_CONFIG_STATIC_SHAPE=1".
         - Currently, duplicate coordinates in the indices will not be coalesced.
           If the indices contain out-of-bound values, the result will be undefined.
 
@@ -646,6 +648,7 @@ class CSRTensor(CSRTensor_):
     [1., 2., 3., 4., 5., 6.], shape is (3, 5), then the dense representation of the sparse tensor will be:
 
     .. code-block::
+
         [[1., 0., 0., 2., 0.],
          [0., 3., 4., 0., 5.],
          [0., 0., 6., 0., 0.]]
@@ -668,10 +671,11 @@ class CSRTensor(CSRTensor_):
 
     Common arithmetic operations include: addition (+), subtraction (-), multiplication (*),
     and division (/). For details about operations supported by `CSRTensor`, see
-    `operators <https://www.mindspore.cn/docs/en/master/note/static_graph_syntax_support.html#operators>`_.
+    `operators <https://www.mindspore.cn/docs/en/master/model_train/program_form/static_graph.html#operators>`_.
 
     .. warning::
         - This is an experimental API that is subjected to change.
+        - If use PyNative mode, set "export MS_PYNATIVE_CONFIG_STATIC_SHAPE=1".
         - If the values given by `indptr` or `indices` are invalid, the results may be undefined. Invalid values include
           when the length of `values` or `indices` exceeds the range indicated by `indptr`, and when the columns
           indicated by `indices` are repeated on the same row.