mindspore 2.0.0a0__cp39-cp39-win_amd64.whl → 2.0.0rc1__cp39-cp39-win_amd64.whl

mindspore/amp.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,17 +16,20 @@
 from __future__ import absolute_import
 
 from abc import ABC, abstractmethod
-
-from ._checkparam import Validator as validator
+from mindspore.common import mutable
+from mindspore.ops._primitive_cache import _get_cache_prim
+from mindspore.ops.operations.math_ops import NPUGetFloatStatusV2, NPUClearFloatStatusV2
+from mindspore import _checkparam as validator
 from .common import dtype as mstype
 from . import context
 from . import ops
 from .ops import constexpr
-from .common.api import jit_class
+from .common.api import jit_class, jit
 from .common.parameter import Parameter
 from .common.tensor import Tensor
 from .train.loss_scale_manager import DynamicLossScaleManager, LossScaleManager, FixedLossScaleManager
-from .train.amp import build_train_network, auto_mixed_precision
+from .train.amp import build_train_network, auto_mixed_precision, custom_mixed_precision,\
+    get_white_list, get_black_list
 
 
 _hypermap = ops.HyperMap()
@@ -51,46 +54,29 @@ def _grad_scale(scale, grad):
     return grad * scale.astype(grad.dtype)
 
 
-def _is_finite(inputs):
-    if _gpu_target():
-        return ops.FloatStatus()(inputs)[0] == 0
-    status = ops.isfinite(inputs)
-    return status.all()
-
-
-def init_status():
-    r"""
-    Returns a Tensor indicating initialized status for overflow detection.
-
-    Note:
-        Only Ascend need status to capture overflow status, you can also call
-        this function on GPU or CPU, but the return value is useless.
+@jit
+def _grad_scale_map(scale_value, inputs):
+    return _hypermap(_partial(_grad_scale, scale_value), inputs)
 
-    Returns:
-        Tensor, has the shape of `(8,)`.
 
-    Supported Platforms:
-        ``Ascend`` ``GPU`` ``CPU``
+@jit
+def _grad_unscale_map(scale_value, inputs):
+    return _hypermap(_partial(_grad_unscale, scale_value), inputs)
 
-    Examples:
-        >>> status = amp.init_status()
-    """
-    if _ascend_target():
-        status = ops.NPUAllocFloatStatus()()
-        clear_status = ops.NPUClearFloatStatus()(status)
-        status = ops.depend(status, clear_status)
-    else:
-        status = Tensor([0, 0, 0, 0, 0, 0, 0, 0], mstype.float32)
 
-    return status
+def _overflow(inputs):
+    if _gpu_target():
+        return ops.FloatStatus()(inputs)
+    status = ops.isfinite(inputs)
+    return 1 - status.all()
 
 
-def all_finite(inputs, status=None):
+def all_finite(inputs):
     r"""
     Returns a scalar Tensor indicating whether the inputs are finite.
 
-    Note:
-        This is an experimental interface that is subject to change or deletion.
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
 
         The interface must be used in whole network training scenario to detect
         whether grads are finite, and the results may be different on different
@@ -98,8 +84,6 @@ def all_finite(inputs, status=None):
 
     Args:
         inputs (Union(tuple(Tensor), list(Tensor))): a iterable Tensor.
-        status (Tensor): the status Tensor for overflow detection, only required on
-            Ascend. Default: None.
 
     Returns:
         Tensor, a scalar Tensor and the dtype is bool.
@@ -112,16 +96,18 @@ def all_finite(inputs, status=None):
         >>> output = amp.all_finite(x)
     """
     if _ascend_target():
-        if status is None:
-            raise ValueError("The status must be initialized on Ascend, but get 'None'.")
+        status = Tensor([0] * 8, mstype.int32)
         status = ops.depend(status, inputs)
-        get_status = ops.NPUGetFloatStatus()(status)
+        get_status = _get_cache_prim(NPUGetFloatStatusV2)()(status)
         status = ops.depend(status, get_status)
-        status_finite = status.sum() == 0
-        _ = ops.NPUClearFloatStatus()(status)
+        clear_status = _get_cache_prim(NPUClearFloatStatusV2)()(status)
+        get_status = ops.depend(get_status, clear_status)
+        status_finite = get_status.equal(Tensor(0, mstype.int32)).all()
         return status_finite
-    outputs = _hypermap(_partial(_is_finite), inputs)
-    return ops.stack(outputs).all()
+    outputs = _hypermap(_partial(_overflow), inputs)
+    flag_sum = ops.addn(outputs).reshape(())
+    _all_finite = ops.less(flag_sum, 1)
+    return _all_finite
 
 
 @jit_class
@@ -133,8 +119,11 @@ class LossScaler(ABC):
     to scale and unscale the loss value and gradients to avoid overflow, `adjust` is used to update the
     loss scale value.
 
-    Note:
-        This is an experimental interface that is subject to change or deletion.
+    For more information, refer to the `tutorials  <https://mindspore.cn/tutorials/en/r2.0/advanced/
+    mixed_precision.html#loss-scaling>`_.
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
     """
     @abstractmethod
     def scale(self, inputs):
@@ -173,8 +162,8 @@ class StaticLossScaler(LossScaler):
 
     Scales and unscales loss or gradients by a fixed constant.
 
-    Note:
-        This is an experimental interface that is subject to change or deletion.
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
 
     Args:
         scale_value (Union(float, int)): The initial loss scale value.
@@ -211,7 +200,8 @@ class StaticLossScaler(LossScaler):
         Returns:
             Union(Tensor, tuple(Tensor)), the scaled value.
         """
-        return _hypermap(_partial(_grad_scale, self.scale_value), inputs)
+        inputs = mutable(inputs)
+        return _grad_scale_map(self.scale_value, inputs)
 
     def unscale(self, inputs):
         """
@@ -223,7 +213,8 @@ class StaticLossScaler(LossScaler):
         Returns:
             Union(Tensor, tuple(Tensor)), the unscaled value.
         """
-        return _hypermap(_partial(_grad_unscale, self.scale_value), inputs)
+        inputs = mutable(inputs)
+        return _grad_unscale_map(self.scale_value, inputs)
 
     def adjust(self, grads_finite):
         """
@@ -244,8 +235,8 @@ class DynamicLossScaler(LossScaler):
     `scale_window` steps by `factor` if the grads remain finite, otherwise it reduces
     the loss scale by `1 / factor` and resets the counter.
 
-    Note:
-        This is an experimental interface that is subject to change or deletion.
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
 
     Args:
         scale_value (Union(float, int)): The initial loss scale value.
@@ -286,7 +277,8 @@ class DynamicLossScaler(LossScaler):
         Returns:
             Union(Tensor, tuple(Tensor)), the scaled value.
         """
-        return _hypermap(_partial(_grad_scale, self.scale_value), inputs)
+        inputs = mutable(inputs)
+        return _grad_scale_map(self.scale_value, inputs)
 
     def unscale(self, inputs):
         """
@@ -298,8 +290,10 @@ class DynamicLossScaler(LossScaler):
         Returns:
             Union(Tensor, tuple(Tensor)), the unscaled value.
         """
-        return _hypermap(_partial(_grad_unscale, self.scale_value), inputs)
+        inputs = mutable(inputs)
+        return _grad_unscale_map(self.scale_value, inputs)
 
+    @jit
     def adjust(self, grads_finite):
         """
         Adjust the `scale_value` dependent on whether grads are finite.
@@ -313,7 +307,7 @@ class DynamicLossScaler(LossScaler):
             grads_finite,
             ops.select(
                 self.counter == (self.scale_window - 1),
-                ops.select(_is_finite(scale_mul_factor),
+                ops.select(ops.isfinite(scale_mul_factor),
                            scale_mul_factor,
                            self.scale_value),
                 self.scale_value),
@@ -327,5 +321,6 @@ class DynamicLossScaler(LossScaler):
 __all__ = [
     "DynamicLossScaleManager", "LossScaleManager", "FixedLossScaleManager",
     "build_train_network", "DynamicLossScaler", "StaticLossScaler", "LossScaler",
-    "auto_mixed_precision", "init_status", "all_finite"
+    "auto_mixed_precision", "all_finite", "custom_mixed_precision",
+    "get_white_list", "get_black_list"
 ]

mindspore/boost/boost.py

@@ -156,8 +156,8 @@ class AutoBoost:
 
               Here:
 
-              - pca_mat (array): Shape (k*n), k is part of n_components, n is the size of weight.
-              - bk (array): Shape (k*k), is the symmetric positive definite matrix in Quasi-Newton method.
+              - pca_mat (array): Shape :math:`(k*n)`, k is part of n_components, n is the size of weight.
+              - bk (array): Shape :math:`(k*k)`, is the symmetric positive definite matrix in Quasi-Newton method.
 
               we need to find the m satisfy:
 

mindspore/boost/boost_cell_wrapper.py

@@ -27,6 +27,7 @@ from mindspore.common import Tensor
 from mindspore.common.sparse_tensor import RowTensorInner
 from mindspore.common.parameter import Parameter, ParameterTuple
 from mindspore.nn.wrap.grad_reducer import DistributedGradReducer
+from mindspore.ops.operations.math_ops import NPUGetFloatStatusV2, NPUClearFloatStatusV2
 from mindspore.ops import functional as F
 from mindspore.ops import composite as C
 from mindspore.ops import operations as P
@@ -115,7 +116,7 @@ class BoostTrainOneStepCell(TrainOneStepCell):
         sens (numbers.Number): The scaling number to be filled as the input of backpropagation. Default value is 1.0.
 
     Inputs:
-        - **(\*inputs)** (Tuple(Tensor)) - Tuple of input tensors with shape :math:`(N, \ldots)`.
+        - **\*inputs** (Tuple(Tensor)) - Tuple of input tensors with shape :math:`(N, \ldots)`.
 
     Outputs:
         Tensor, a tensor means the loss value, the shape of which is usually :math:`()`.
@@ -392,7 +393,7 @@ class BoostTrainOneStepWithLossScaleCell(BoostTrainOneStepCell):
                                           is Tensor type, Tensor with shape :math:`()` or :math:`(1,)`.
 
     Inputs:
-        - **(*inputs)** (Tuple(Tensor)) - Tuple of input tensors with shape :math:`(N, \ldots)`.
+        - **\*inputs** (Tuple(Tensor)) - Tuple of input tensors with shape :math:`(N, \ldots)`.
 
     Outputs:
         Tuple of 3 Tensor, the loss, overflow flag and current loss scaling value.
@@ -460,6 +461,11 @@ class BoostTrainOneStepWithLossScaleCell(BoostTrainOneStepCell):
         self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE)
         self.gpu_target = (context.get_context("device_target") == "GPU")
         self.loss_scaling_manager = None
+        self.base0 = Tensor(0, mstype.int32)
+        self.reduce_all = P.ReduceAll(keep_dims=False)
+        self.reduce_any = P.ReduceAny(keep_dims=False)
+        self.equal = P.Equal()
+        self.not_equal = P.NotEqual()
 
         if self.auto_boost.boost_config.get("loss_scale_group", False):
             self.enable_enhanced_amp = True
@@ -535,12 +541,13 @@ class BoostTrainOneStepWithLossScaleCell(BoostTrainOneStepCell):
             bool, overflow value.
             float, update ratio.
         """
-        flag_sum = self.reduce_sum(param, (0,))
+        flag_sum = self.equal(self.base0, param)
         if self.reducer_flag:
             flag_reduce = self.allreduce(flag_sum)
-            overflow = self.less_equal(self.base, flag_reduce)
+            overflow = not self.reduce_all(flag_reduce)
         else:
-            overflow = self.less_equal(self.base, flag_sum)
+            overflow = not self.reduce_all(flag_sum)
+
         if overflow:
             update_ratio = self.reduce_ratio
         else:
@@ -609,13 +616,11 @@ class BoostTrainOneStepWithLossScaleCell(BoostTrainOneStepCell):
             The second value is the same as the input of `compute_input`, but contains some information about the
             execution order.
         """
-        status = False
+        status = Tensor([0] * 8, mstype.int32)
         if not self.gpu_target:
-            # init overflow buffer
-            status = P.NPUAllocFloatStatus()()
             status = F.depend(status, pre_cond)
             # clear overflow buffer
-            clear_status = P.NPUClearFloatStatus()(status)
+            clear_status = NPUClearFloatStatusV2()(status)
             compute_input = F.depend(compute_input, clear_status)
         return status, compute_input
 
@@ -636,22 +641,35 @@ class BoostTrainOneStepWithLossScaleCell(BoostTrainOneStepCell):
         """
         if not self.gpu_target:
             status = F.depend(status, compute_output)
-            get_status = P.NPUGetFloatStatus()(status)
-            status = F.depend(status, get_status)
-            # sum overflow buffer elements, 0:not overflow , >0:overflow
-            flag_sum = self.reduce_sum(status, (0,))
+            get_status = NPUGetFloatStatusV2()(status)
+
+            if self.is_distributed:
+                # sum overflow flag over devices
+                flag_reduce = self.allreduce(get_status)
+                # get_status not equal to [0]*8 means overflow
+                flag = self.not_equal(self.base0, flag_reduce)
+                status = F.depend(status, flag)
+                # distributed needs to skip allreduce to avoid its overflow affecting the next step
+                clear_status = NPUClearFloatStatusV2()(status)
+                flag = F.depend(flag, clear_status)
+            else:
+                status = F.depend(status, get_status)
+                clear_status = NPUClearFloatStatusV2()(status)
+                get_status = F.depend(get_status, clear_status)
+                flag = self.not_equal(self.base0, get_status)
+            overflow = self.reduce_any(flag)
         else:
             flag_sum = self.hyper_map(F.partial(_grad_overflow), compute_output)
             flag_sum = P.AddN()(flag_sum)
             # convert flag_sum to scalar
             flag_sum = P.Reshape()(flag_sum, (()))
 
-        if self.is_distributed:
-            # sum overflow flag over devices
-            flag_reduce = self.allreduce(flag_sum)
-            overflow = self.less_equal(self.base, flag_reduce)
-        else:
-            overflow = self.less_equal(self.base, flag_sum)
+            if self.is_distributed:
+                # sum overflow flag over devices
+                flag_reduce = self.allreduce(flag_sum)
+                overflow = self.less_equal(self.base, flag_reduce)
+            else:
+                overflow = self.less_equal(self.base, flag_sum)
         return overflow
 
     def _process_loss_scale(self, overflow):
@@ -688,7 +706,7 @@ class BoostTrainOneStepWithLossScaleCell(BoostTrainOneStepCell):
         self.optimizer_loss_scale = [self.parent.count(x) for x in parent_set]
         self.reduce_ratio = Tensor(1.0 / (2 ** 0.5), mstype.float32)
         self.growth_ratio = Tensor(2 ** (1.0 / 1000.0), mstype.float32)
-        self.overflow_status_list = ParameterTuple(Parameter(Tensor(np.zeros(shape=[8]), mstype.float32),
+        self.overflow_status_list = ParameterTuple(Parameter(Tensor(np.zeros(shape=[8]), mstype.int32),
                                                              name='mix_layer_status_{}'.format(x), requires_grad=False)
                                                    for x in range(loss_scale_number))
         self.loss_scaling_manager.set_loss_scale_status(loss_scale_number, self.loss_scaling_manager.get_loss_scale())

mindspore/boost/dim_reduce.py

@@ -102,8 +102,8 @@ class DimReduce(Cell):
 
     Here:
 
-    - pca_mat (array): Shape (k*n), k is part of n_components, n is the size of weight.
-    - bk (array): Shape (k*k), is the symmetric positive definite matrix in Quasi-Newton method.
+    - pca_mat (array): Shape :math:`(k*n)`, k is part of n_components, n is the size of weight.
+    - bk (array): Shape :math:`(k*k)`, is the symmetric positive definite matrix in Quasi-Newton method.
 
     we need to find the m satisfy:
 
@@ -138,7 +138,7 @@ class DimReduce(Cell):
         - **old_grad** (Tuple(Tensor)) - Tuple of gradient tensors.
         - **weight** (Tuple(Tensor)) - Tuple of parameters.
         - **weight_clone** (Tuple(Tensor)) - clone of weight
-        - **(\*inputs)** (Tuple(Tensor)) - Tuple of input tensors with shape :math:`(N, \ldots)`.
+        - **\*inputs** (Tuple(Tensor)) - Tuple of input tensors with shape :math:`(N, \ldots)`.
 
     Outputs:
         - **loss** (Tensor) - Tensor with shape :math:`()`.

mindspore/boost/group_loss_scale_manager.py

@@ -93,7 +93,7 @@ class GroupLossScaleManager(Cell):
         >>>               boost_level="O1", boost_config_dict=boost_config_dict)
         >>> # For details about how to build the dataset, please refer to the variable `dataset_train` in tutorial
         >>> # document on the official website:
-        >>> # https://www.mindspore.cn/tutorials/zh-CN/r2.0.0-alpha/beginner/quick_start.html
+        >>> # https://www.mindspore.cn/tutorials/zh-CN/r2.0/beginner/quick_start.html
         >>> dataset = create_custom_dataset()
         >>> model.train(2, dataset)
     """

mindspore/common/__init__.py

@@ -15,13 +15,13 @@
 """Top-level reference to dtype of common module."""
 from __future__ import absolute_import
 from mindspore.common import dtype
-from mindspore.common.api import no_recursive, ms_function, ms_memory_recycle, ms_class, jit, jit_class
+from mindspore.common.api import ms_function, ms_memory_recycle, ms_class, jit, jit_class
 from mindspore.common.dtype import Type, int8, byte, int16, short, int32, intc, int64, intp, \
     uint8, ubyte, uint16, ushort, uint32, uintc, uint64, uintp, float16, half, \
     float32, single, float64, double, bool_, float_, list_, tuple_, int_, \
     uint, number, tensor, string, type_none, tensor_type, Int, \
     complex64, complex128, dtype_to_nptype, _null, _null_type, \
-    dtype_to_pytype, pytype_to_dtype, get_py_obj_dtype
+    dtype_to_pytype, pytype_to_dtype, get_py_obj_dtype, QuantDtype
 from mindspore.common.dump import set_dump
 from mindspore.common.parameter import Parameter, ParameterTuple
 from mindspore.common.seed import set_seed, get_seed
@@ -29,7 +29,6 @@ from mindspore.common.tensor import Tensor
 from mindspore.common.sparse_tensor import RowTensor, RowTensorInner, SparseTensor, COOTensor, CSRTensor
 from mindspore.common.mutable import mutable
 from mindspore.common.jit_config import JitConfig
-from mindspore.common._utils import update_and_return_dict
 
 # symbols from dtype
 __all__ = [
@@ -50,7 +49,7 @@ __all__ = [
     "number", "tensor",
     "string", "type_none",
     "_null",
-    "tensor_type",
+    "tensor_type", "QuantDtype",
     "Type", "Int", "_null_type",
     "complex64", "complex128",
     # __method__ from dtype
@@ -60,12 +59,11 @@ __all__ = [
 
 __all__.extend([
     "Tensor", "RowTensor", "SparseTensor", "COOTensor", "CSRTensor", # tensor
-    "no_recursive", "ms_function", "ms_class", 'jit', 'jit_class',  # api
+    "ms_function", "ms_class", 'jit', 'jit_class',  # api
     "Parameter", "ParameterTuple",  # parameter
     "dtype",
     "set_seed", "get_seed",  # random seed
     "set_dump",
     "ms_memory_recycle",
     "mutable", "JitConfig",
-    "update_and_return_dict",
 ])

mindspore/common/_decorator.py

@@ -15,6 +15,7 @@
 """Providing decorators."""
 
 from __future__ import absolute_import
+from functools import wraps
 from mindspore import log
 
 
@@ -31,6 +32,7 @@ def deprecated(version, substitute, use_substitute_name=False):
     """
 
     def decorate(func):
+        @wraps(func)
         def wrapper(*args, **kwargs):
             cls = getattr(args[0], "__class__", None) if args else None
             name = cls.__name__ if cls else func.__name__

mindspore/common/_register_for_adapter.py

@@ -0,0 +1,55 @@
+# Copyright 2022 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""Registry MSAdapter config."""
+
+from mindspore.common.tensor import Tensor
+
+
+class Registry:
+    """Registry class for ms adapter."""
+
+    def __init__(self):
+        self._tensor = None
+        self._convert_map = {}
+
+    @property
+    def tensor(self):
+        """Return the registered tensor."""
+        if self._tensor is None:
+            raise ValueError("Before using Tensor in MSAdapter, please call 'set_adapter_config'.")
+        return self._tensor
+
+    @property
+    def convert_map(self):
+        """Return the registered convert map."""
+        return self._convert_map
+
+    def register_tensor(self, value):
+        """Register the tensor of ms adapter."""
+        if self._tensor is not None:
+            raise ValueError("Repeated registration of tensor in ms adapter config.")
+        if not issubclass(value, Tensor):
+            raise ValueError(f"The tensor definition here should be a subclass of ms.Tensor, but got {value}.")
+        self._tensor = value
+
+    def register_convert_map(self, value):
+        """Register the convert map of ms adapter."""
+        if not isinstance(value, dict):
+            raise ValueError(f"Expect a dict type, but got {type(value)}.")
+        self._convert_map = value
+
+
+ms_adapter_registry = Registry()