mindspore 2.7.0rc1__cp310-cp310-win_amd64.whl → 2.7.1__cp310-cp310-win_amd64.whl

mindspore/common/parameter.py

@@ -21,7 +21,6 @@ from copy import copy
 import time
 import os
 import sys
-import math
 import numbers
 import numpy as np
 
@@ -29,19 +28,14 @@ from mindspore import log as logger
 from mindspore.log import _LogActionOnce
 from mindspore._c_expression import ParamInfo
 from mindspore.common import dtype as mstype
-from mindspore import context
-from mindspore.common._utils import get_slice_num, get_slice_shape
 from mindspore.common.initializer import initializer
 from mindspore.common.tensor import Tensor, _TensorMeta
+from mindspore.common.hook_handle import _update_hook_version
 from mindspore import _checkparam as Validator
 from mindspore._check_jit_forbidden_api import jit_forbidden_register
 from mindspore._c_expression import TensorPy as Tensor_
 from mindspore.parallel._tensor import _get_slice_index
 from mindspore.parallel._auto_parallel_context import auto_parallel_context
-from mindspore.parallel._ps_context import _is_role_worker, _is_role_pserver, _is_role_sched, _clone_hash_table, \
-                                           _is_ps_mode
-from mindspore.parallel._ps_context import _reinsert_hash_table_size, _insert_accumu_init_info, _cache_enable
-from mindspore.common._decorator import deprecated
 from mindspore.communication._comm_helper import _is_initialized
 from mindspore.communication import get_group_size, get_rank
 import mindspore.common._monad as monad
@@ -71,19 +65,6 @@ def _is_parameter_generated(param_name):
 # Global variable for parameter unique key.
 _GLOBAL_PARAMETER_KEY = -1
 
-# Global variable to mark the hook of parameter is updated
-_parameter_hook_updated = True
-
-
-def set_parameter_hook_updated(value):
-    global _parameter_hook_updated
-    _parameter_hook_updated = value
-
-
-def parameter_hook_updated():
-    global _parameter_hook_updated
-    return _parameter_hook_updated
-
 
 def _is_in_auto_parallel_mode():
     """Get parallel mode."""
@@ -92,7 +73,7 @@ def _is_in_auto_parallel_mode():
 
 def _is_parallel_mode():
     """ Whether is parallel mode """
-    if not _is_initialized() or context.get_context('mode') == context.PYNATIVE_MODE:
+    if not _is_initialized():
         return False
     if os.getenv("RUN_MODE") != "predict":
         return False
@@ -150,11 +131,7 @@ def _offload_if_config(data):
     Args:
         data: The parameter data to offload.
     """
-    if not context.get_context("memory_offload") or data is None:
-        return
-
-    offload_context = context.get_offload_context()
-    if offload_context.get("offload_param", None) != "disk":
+    if data is None:
         return
 
     data_size_threshold = 512
@@ -231,7 +208,10 @@ class Parameter(Tensor_):
                 self.param_a = Parameter(Tensor([1], ms.float32), name="name_a")
                 self.param_tuple = (self.param_a, self.param_a)
 
-        requires_grad (bool): True if the parameter requires gradient. Default: ``True`` .
+        requires_grad (bool): It is Used to filter parameters in :func:`mindspore.nn.Cell.trainable_params()`.
+            If it is ``False``, the filter parameters will not be returned in
+            :func:`mindspore.nn.Cell.trainable_params()`.
+            Default: ``True`` .
         layerwise_parallel (bool): When `layerwise_parallel` is true in data/hybrid parallel mode,
             broadcast and gradients communication would not be applied to the `Parameter`. Default: ``False`` .
         parallel_optimizer (bool): It is used to filter the weight shard operation in parallel mode. It works only when
@@ -242,10 +222,8 @@ class Parameter(Tensor_):
         device(str): Only Ascend device target is supported. It is used to specify the device which the parameter is
             stored. By default, the parameter will be stored on NPU while computing. When the device is specified as
             ``"CPU"``, the parameter will be loaded into the device when it needs to be used, and unloaded to the CPU
-            after use. It takes effext only when `memory_offload` is ``"ON"``, `jit_level` is not ``"O2"`` and
-            `memory_optimize_level` is ``O0`` in :func:`mindspore.set_context`.
-            Less device memory is needed when device is
-            specified as ``"CPU"``.
+            after use. It takes effext only when `jit_level` is not ``"O2"`` and `memory_optimize_level` is ``O0``
+            in :func:`mindspore.set_context`. Less device memory is needed when device is specified as ``"CPU"``.
 
     Examples:
         >>> import numpy as np
@@ -284,8 +262,6 @@ class Parameter(Tensor_):
         obj.is_default_input_init = init_data_flag
         if obj.has_init:
             obj.init_mode = default_input
-        else:
-            _offload_if_config(obj)
         return obj
 
     def __reduce_ex__(self, _):
@@ -301,7 +277,6 @@ class Parameter(Tensor_):
     def __init__(self, default_input, name=None, requires_grad=True, layerwise_parallel=False, parallel_optimizer=True,
                  storage_format="", device=None):
         self.param_info = ParamInfo()
-        self.init_in_server = False
         self.name = name
         self.requires_grad = requires_grad
         self.layerwise_parallel = layerwise_parallel
@@ -312,32 +287,15 @@ class Parameter(Tensor_):
         self.is_init = False
         self._inited_param = None
         self._sliced = False
-        self.is_param_ps = False
-        self.push_weight_to_server = False
-        self.pull_weight_from_server = False
         self.requires_aggr = True
         self._cast_type = None
         self._unique = False
         self.is_in_parallel = _is_in_auto_parallel_mode()
         self._pipeline_stage_list = []
-        self.slice_num = 1
         if -1 in self.shape:
             raise ValueError(f"All shape elements of the Parameter must be positive. But got None.")
         if isinstance(default_input, (Tensor_, Tensor)):
-            # At embedding cache scenes, we need limit the size of memory for parameter.
-            # And save out range data to persistent storage to support TB-Level size parameter.
-            slice_num_of_persistent_data = get_slice_num(default_input.dtype, default_input.shape)
-            if slice_num_of_persistent_data > 1:
-                data_shape = list(default_input.shape)
-                slice_first_dim = math.ceil(data_shape[0] / slice_num_of_persistent_data)
-                data_shape[0] = slice_first_dim
-                self.param_info.use_persistent_storage = True
-                self.param_info.origin_shape = default_input.shape
-                self.slice_num = slice_num_of_persistent_data
-                Tensor_.__init__(self, dtype=default_input.dtype, shape=tuple(data_shape))
-            else:
-                Tensor_.__init__(self, dtype=default_input.dtype, shape=default_input.shape)
-
+            Tensor_.__init__(self, dtype=default_input.dtype, shape=default_input.shape)
         elif isinstance(default_input, int):
             Tensor_.__init__(self, dtype=mstype.int64, shape=())
         elif isinstance(default_input, float):
@@ -399,11 +357,10 @@ class Parameter(Tensor_):
                     return (Tensor, data.asnumpy(), mstype.qint4x2)
                 return (Tensor, data.asnumpy())
 
-            not_init_data = not init_param or _is_role_sched() or (_is_role_pserver() and _cache_enable()) \
-                            or _is_in_auto_parallel_mode() or _is_parallel_mode()
+            not_init_data = not init_param or _is_in_auto_parallel_mode() or _is_parallel_mode()
             if not_init_data:
                 # do not init data while in auto parallel.
-                return (Tensor, None, data.dtype, get_slice_shape(data.dtype, data.shape), data.init)
+                return (Tensor, None, data.dtype, data.shape, data.init)
             return (Tensor, data.init_data())
         if isinstance(data, int):
             return (Tensor, data, mstype.int32)
@@ -411,33 +368,6 @@ class Parameter(Tensor_):
             return (Tensor, data, mstype.float32)
         return (Tensor, data)
 
-    def set_param_ps(self, init_in_server=False):
-        """
-        Set whether the trainable parameter is updated by parameter server and whether the
-        trainable parameter is initialized on server.
-
-        Note:
-            It only works when a running task is in the parameter server mode.
-            It is supported only in graph mode.
-
-        Args:
-            init_in_server (bool): Whether trainable parameter updated by parameter server is
-                initialized on server. Default: ``False``.
-
-        """
-        if not _is_ps_mode() or not (_is_role_worker() or _is_role_pserver() or _is_role_sched()):
-            raise RuntimeError("Must complete following two steps before calling set_param_ps: \n"
-                               "1. context.set_ps_context(enable_ps=True) \n"
-                               "2. export MS_ROLE environment variable \n"
-                               "Please refer to the official website for detailed usage.")
-
-        if context.get_context("mode") == context.PYNATIVE_MODE:
-            raise RuntimeError("Parameter server training is not supported in pynative mode currently."
-                               "Please switch to graph mode and retry.")
-        self.is_param_ps = True
-        self.init_in_server = init_in_server
-        self.param_info.init_in_server = init_in_server
-
     def copy(self):
         """
         Copy the parameter.
@@ -453,16 +383,6 @@ class Parameter(Tensor_):
         """
         return self.clone(init='same')
 
-    @deprecated("1.8", "set_param_fl")
-    def set_param_fl(self, push_to_server=False, pull_from_server=False, requires_aggr=True):
-        if push_to_server:
-            self.push_weight_to_server = True
-        if pull_from_server:
-            self.pull_weight_from_server = True
-        if not requires_aggr:
-            self.requires_aggr = False
-            self.param_info.requires_aggr = False
-
     @property
     def inited_param(self):
         """
@@ -528,8 +448,6 @@ class Parameter(Tensor_):
             raise ValueError("The type of the Parameter's name should be 'string' or 'None', "
                              "but got {}.".format(type(name_)))
 
-        if _is_role_worker() and self.cache_enable:
-            _reinsert_hash_table_size(name_, self.param_info.name)
         self.param_info.name = name_
 
     @property
@@ -658,8 +576,6 @@ class Parameter(Tensor_):
         x.param_info = param_info_clone
         x.is_init = False
         x.init = self.init
-        x.is_param_ps = self.is_param_ps
-        x.init_in_server = self.init_in_server
         x.cache_enable = self.cache_enable
         if x.cache_enable:
             x.key = _get_unique_parameter_key()
@@ -667,7 +583,7 @@ class Parameter(Tensor_):
         if self.cache_shape:
             x.cache_shape = self.cache_shape
         if init != 'same':
-            shape = self.shape if self.slice_num == 1 else self.param_info.origin_shape
+            shape = self.shape
             dtype = self.dtype
             tensor = initializer(init, shape=shape, dtype=dtype)
             x.set_data(tensor)
@@ -812,6 +728,7 @@ class Parameter(Tensor_):
             raise TypeError("The argument `requires_grad` must be bool type")
         Tensor_.wait_pipeline(self)
         self.param_info.requires_grad = value
+        self._requires_grad = value
 
     @property
     def data(self):
@@ -878,20 +795,6 @@ class Parameter(Tensor_):
             raise TypeError("The original tensor data is initialized, but the argument 'data' is not initialized."
                             "Please initialize 'data' before call this method.")
 
-    @staticmethod
-    def _from_tensor(tensor, *args, **kwargs):
-        """Create a `Parameter` that data is shared from a `Tensor`."""
-        if not isinstance(tensor, Tensor_):
-            raise TypeError(f"The type of input must be Tensor, but got {type(tensor)}.")
-        param = Tensor_.__new__(Parameter)
-        Tensor_.__init__(param, tensor)
-        param.init = None
-        param.init_mode = None
-        param.has_init = False
-        param.is_default_input_init = False
-        Parameter.__init__(param, tensor, *args, **kwargs)
-        return param
-
     @jit_forbidden_register
     def set_data(self, data, slice_shape=False):
         """
@@ -997,16 +900,7 @@ class Parameter(Tensor_):
 
         init_data_args = self._get_init_data_args(layout)
 
-        if _is_role_sched():
-            return self
-        if self.init_in_server and self.is_param_ps and isinstance(self.init_mode, Tensor) and \
-                self.init_mode.init is not None and _is_role_worker():
-            if self.cache_enable:
-                data = self.init_mode.init_data(*init_data_args)
-            else:
-                data = self.init_mode.init_data(0, [1])
-        else:
-            data = self.init_mode.init_data(*init_data_args)
+        data = self.init_mode.init_data(*init_data_args)
         origin_dtype = self.dtype
         obj = self._update_tensor_data(data)
         if self.dtype != origin_dtype:
@@ -1015,7 +909,6 @@ class Parameter(Tensor_):
             self._inited_param = obj
         obj.init_mode = None
         obj.sliced = set_sliced
-        _offload_if_config(obj)
         return obj
 
     def register_hook(self, hook_fn):
@@ -1023,11 +916,11 @@ class Parameter(Tensor_):
         For details, please refer to :func:`mindspore.Tensor.register_hook`.
         """
         handle = Tensor.register_hook(self, hook_fn)
-        set_parameter_hook_updated(True)
+        _update_hook_version()
         return handle
 
     def _remove_hook(self):
-        set_parameter_hook_updated(True)
+        _update_hook_version()
 
     def _offload(self):
         r"""
@@ -1170,9 +1063,6 @@ class ParameterTuple(tuple):
             if not x1.cache_enable:
                 continue
 
-            if _is_role_worker():
-                _clone_hash_table(x.name, x.key, x1.name, x1.key)
-                _insert_accumu_init_info(x1.name, init_to_value(init))
         return ParameterTuple(new)
 
     def __parameter_tuple__(self):

mindspore/common/recompute.py

@@ -18,12 +18,11 @@ from collections import OrderedDict
 from types import MethodType
 from mindspore import log as logger
 from mindspore.nn.cell import Cell
-from mindspore import context
 from mindspore.common.tensor import Tensor
 from mindspore import ops
 from mindspore.ops.composite import GradOperation
 from mindspore.common._register_for_recompute import recompute_registry
-from mindspore.common.api import _pynative_executor, _no_grad
+from mindspore.common.api import _pynative_executor, _no_grad, _run_in_jit
 from mindspore.common.generator import get_rng_state, set_rng_state
 from mindspore.train.amp import AmpDecorator
 from mindspore._c_expression.amp import get_curr_amp_strategy
@@ -211,12 +210,6 @@ def _detach_input(input_arg):
 def _check_validation(block):
     if not isinstance(block, Cell):
         raise TypeError("Recompute function now only support block which inherited from Cell!")
-    if context.get_context("mode") != context.PYNATIVE_MODE:
-        raise AssertionError("Recompute function now only support pynative mode, you can use "
-                             "Cell.recompute() in graph mode.")
-    if block.construct.__code__.co_name == "staging_specialize":
-        logger.warning('Block\'s construct method decorated by @jit that recompute '
-                       'function will not come into effect.')
 
 
 def recompute(block, *args, **kwargs):
@@ -225,10 +218,7 @@ def recompute(block, *args, **kwargs):
     storing the intermediate activation computed in forward pass, we will recompute it in backward pass.
 
     Note:
-        - Recompute function only support block which inherited from Cell object.
-        - This function interface now only support pynative mode. you can use Cell.recompute interface
-          in graph mode.
-        - When use recompute function, block object should not decorated by @jit.
+        Recompute function only support block which inherited from Cell object.
 
     Args:
         block (Cell): Block to be recompute.
@@ -240,7 +230,6 @@ def recompute(block, *args, **kwargs):
 
     Raises:
         TypeError: If `block` is not Cell object.
-        AssertionError: If execute mode is not PYNATIVE_MODE.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -274,6 +263,8 @@ def recompute(block, *args, **kwargs):
     """
 
     _check_validation(block)
+    if _run_in_jit():  # @jit.cond: True
+        return ops.recompute_block(block)(*args, **kwargs)
     return _RecomputeCell(block)(*args, **kwargs)