mindspore 2.1.0__cp39-cp39-win_amd64.whl → 2.2.11__cp39-cp39-win_amd64.whl

mindspore/amp.py

@@ -15,11 +15,13 @@
 """ms function for mixed precision."""
 from __future__ import absolute_import
 
+import os
 from abc import ABC, abstractmethod
 from mindspore.common import mutable
 from mindspore.ops._primitive_cache import _get_cache_prim
 from mindspore.ops.operations.math_ops import NPUGetFloatStatusV2, NPUClearFloatStatusV2
 from mindspore import _checkparam as validator
+from mindspore._c_expression import MSContext
 from .common import dtype as mstype
 from . import context
 from . import ops
@@ -37,8 +39,13 @@ _partial = ops.Partial()
 
 
 @constexpr
-def _ascend_target():
-    return context.get_context("device_target") == "Ascend"
+def _ascend_910A_target():
+    return MSContext.get_instance().get_ascend_soc_version() == "ascend910"
+
+
+@constexpr
+def _ascend_910B_target():
+    return MSContext.get_instance().get_ascend_soc_version() == "ascend910b"
 
 
 @constexpr
@@ -72,9 +79,10 @@ def _overflow(inputs):
 
 
 @jit
-def _all_finite(inputs):
+def _all_finite(inputs, check_overflow_mode):
     """all finite check"""
-    if _ascend_target():
+    if (_ascend_910A_target()) or \
+       (_ascend_910B_target() and check_overflow_mode != "INFNAN_MODE"):
         status = Tensor([0] * 8, mstype.int32)
         status = ops.depend(status, inputs)
         get_status = _get_cache_prim(NPUGetFloatStatusV2)()(status)
@@ -83,6 +91,7 @@ def _all_finite(inputs):
         get_status = ops.depend(get_status, clear_status)
         status_finite = get_status.equal(Tensor(0, mstype.int32)).all()
         return status_finite
+
     outputs = _hypermap(_partial(_overflow), inputs)
     flag_sum = ops.addn(outputs).reshape(())
     status_finite = ops.less(flag_sum, 1)
@@ -117,10 +126,11 @@ def all_finite(inputs):
 
     Tutorial Examples:
         - `Automatic Mix Precision - Loss Scaling
-          <https://mindspore.cn/tutorials/en/r2.1/advanced/mixed_precision.html#loss-scaling>`_
+          <https://mindspore.cn/tutorials/en/r2.2/advanced/mixed_precision.html#loss-scaling>`_
     """
     inputs = mutable(inputs)
-    return _all_finite(inputs)
+    _check_overflow_mode = os.environ.get('MS_ASCEND_CHECK_OVERFLOW_MODE')
+    return _all_finite(inputs, _check_overflow_mode)
 
 
 @jit_class
@@ -132,11 +142,36 @@ class LossScaler(ABC):
     to scale and unscale the loss value and gradients to avoid overflow, `adjust` is used to update the
     loss scale value.
 
-    For more information, refer to the `tutorials  <https://mindspore.cn/tutorials/en/r2.1/advanced/
+    For more information, refer to the `tutorials  <https://mindspore.cn/tutorials/en/r2.2/advanced/
     mixed_precision.html#loss-scaling>`_.
 
     .. warning::
         This is an experimental API that is subject to change or deletion.
+
+    Examples:
+        >>> from mindspore.amp import LossScaler, _grad_scale_map, _grad_unscale_map
+        >>> from mindspore import ops, Parameter, Tensor
+        >>> from mindspore.common import dtype as mstype
+        >>>
+        >>> class MyLossScaler(LossScaler):
+        ...     def __init__(self, scale_value):
+        ...         self.scale_value = Parameter(Tensor(scale_value, dtype=mstype.float32), name="scale_value")
+        ...
+        ...     def scale(self, inputs):
+        ...         inputs = mutable(inputs)
+        ...         return _grad_scale_map(self.scale_value, inputs)
+        ...
+        ...     def unscale(self, inputs):
+        ...         inputs = mutable(inputs)
+        ...         return _grad_unscale_map(self.scale_value, inputs)
+        ...
+        ...     def adjust(self, grads_finite):
+        ...         scale_mul_factor = self.scale_value * self.scale_factor
+        ...         scale_value = ops.select(grads_finite, scale_mul_factor, self.scale_value)
+        ...         ops.assign(self.scale_value, scale_value)
+        ...         return True
+        >>>
+        >>> loss_scaler = MyLossScaler(1024)
     """
     @abstractmethod
     def scale(self, inputs):
@@ -234,7 +269,8 @@ class StaticLossScaler(LossScaler):
 
     def adjust(self, grads_finite):
         """
-        `scale_value` is fixed.
+        Adjust `scale_value` in `LossScaler`. `scale_value` is fixed in `StaticLossScaler`, so this method
+        return False directly.
 
         Args:
             grads_finite (Tensor): a scalar bool Tensor indicating whether the grads are finite.
@@ -298,7 +334,7 @@ class DynamicLossScaler(LossScaler):
 
         Tutorial Examples:
             - `Automatic Mix Precision - Loss Scaling
-              <https://mindspore.cn/tutorials/en/r2.1/advanced/mixed_precision.html#loss-scaling>`_
+              <https://mindspore.cn/tutorials/en/r2.2/advanced/mixed_precision.html#loss-scaling>`_
         """
         inputs = mutable(inputs)
         return _grad_scale_map(self.scale_value, inputs)
@@ -315,7 +351,7 @@ class DynamicLossScaler(LossScaler):
 
         Tutorial Examples:
             - `Automatic Mix Precision - Loss Scaling
-              <https://mindspore.cn/tutorials/en/r2.1/advanced/mixed_precision.html#loss-scaling>`_
+              <https://mindspore.cn/tutorials/en/r2.2/advanced/mixed_precision.html#loss-scaling>`_
         """
         inputs = mutable(inputs)
         return _grad_unscale_map(self.scale_value, inputs)
@@ -329,7 +365,7 @@ class DynamicLossScaler(LossScaler):
 
         Tutorial Examples:
             - `Automatic Mix Precision - Loss Scaling
-              <https://mindspore.cn/tutorials/en/r2.1/advanced/mixed_precision.html#loss-scaling>`_
+              <https://mindspore.cn/tutorials/en/r2.2/advanced/mixed_precision.html#loss-scaling>`_
         """
         one = ops.ones((), self.scale_value.dtype)
         scale_mul_factor = self.scale_value * self.scale_factor

mindspore/boost/boost.py

@@ -85,9 +85,6 @@ class AutoBoost:
                         "freeze_p": 0.7,
                         "total_steps": 65536
                     }
-                    "grad_accumulation": {
-                        "grad_accumulation_step": 1
-                    },
                     "dim_reduce": {
                         "rho": 0.55,
                         "gamma": 0.9,
@@ -139,10 +136,6 @@ class AutoBoost:
               - freeze_p (float): Gradient freezing probability. Default: ``0.7`` .
               - total_steps (int): Total training steps. Default: ``65536`` .
 
-            - grad_accumulation:
-
-              - grad_accumulation_step (int): Steps to accumulate gradients. Default: ``1`` .
-
             - dim_reduce:
 
               The leading principles of dim_reduce:
@@ -204,7 +197,7 @@ class AutoBoost:
         >>> import json
         >>> boost_json = "/path/boost_config.json"
         >>> with open(boost_json, 'r') as fp:
-        >>>     boost_config_dict = json.load(fp)
+        ...     boost_config_dict = json.load(fp)
         >>> boost = AutoBoost("O1", boost_config_dict)
     """
     _instance_lock = threading.Lock()

mindspore/boost/boost_cell_wrapper.py

@@ -136,7 +136,7 @@ class BoostTrainOneStepCell(TrainOneStepCell):
         >>> from mindspore import boost
         >>> from mindspore import nn
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits()
         >>> optim = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9)
@@ -412,7 +412,8 @@ class BoostTrainOneStepWithLossScaleCell(BoostTrainOneStepCell):
         network (Cell): The training network. The network only supports single output.
         optimizer (Cell): Optimizer for updating the weights.
         scale_sense (Union[Tensor, Cell]): If this value is Cell type, the loss scaling update logic cell.If this value
-                                          is Tensor type, Tensor with shape :math:`()` or :math:`(1,)`.
+            is Tensor type, :func:`mindspore.nn.TrainOneStepWithLossScaleCell.set_sense_scale` can be called to update
+            loss scale factor, Tensor with shape :math:`()` or :math:`(1,)`.
 
     Inputs:
         - **\*inputs** (Tuple(Tensor)) - Tuple of input tensors with shape :math:`(N, \ldots)`.

mindspore/boost/grad_accumulation.py

@@ -71,7 +71,7 @@ class GradientAccumulation(Cell):
 
         if self._accumulation_step >= self._max_accumulation_step:
             loss = F.depend(loss, self.optimizer(self._grad_accumulation))
-            self._accumulation_step = 0
+            F.assign(self._accumulation_step, 0)
 
         if self._accumulation_step == 0:
             loss = F.depend(loss, self.hyper_map(F.partial(gradient_clear_op), self._grad_accumulation))

mindspore/boost/group_loss_scale_manager.py

@@ -27,7 +27,7 @@ __all__ = ["GroupLossScaleManager"]
 
 
 class GroupLossScaleManager(Cell):
-    """
+    r"""
     Enhanced hybrid precision algorithm supports multi-layer application of different loss scales and
     dynamic updating of loss scales.
 
@@ -41,7 +41,8 @@ class GroupLossScaleManager(Cell):
         - **layer2** (Int) - Last network layer value.
 
     Outputs:
-        - **x** (Tensor) - The output of `_DynamicLossScale` operator.
+        - **out** (Tensor) - A tensor with a group of loss scale tags that marks
+          the loss scale group number of the current tensor.
 
     Supported Platforms:
         ``Ascend``
@@ -80,10 +81,10 @@ class GroupLossScaleManager(Cell):
         >>> param_group1 = []
         >>> param_group2 = []
         >>> for param in net.trainable_params():
-        >>>     if 'conv' in param.name:
-        >>>         param_group1.append(param)
-        >>>     else:
-        >>>         param_group2.append(param)
+        ...     if 'conv' in param.name:
+        ...         param_group1.append(param)
+        ...     else:
+        ...         param_group2.append(param)
         >>> loss_scale_manager.loss_scale_groups = [param_group1, param_group2]
         >>> loss = nn.SoftmaxCrossEntropyWithLogits()
         >>> optim = nn.Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9)
@@ -93,7 +94,7 @@ class GroupLossScaleManager(Cell):
         ...                        loss_scale_manager=loss_scale_manager,
         ...                        boost_level="O1", boost_config_dict=boost_config_dict)
         >>> # Create the dataset taking MNIST as an example. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/mnist.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
         >>> dataset = create_dataset()
         >>> model.train(2, dataset)
     """

mindspore/common/__init__.py

@@ -18,7 +18,7 @@ from mindspore.common import dtype
 from mindspore.common.api import ms_function, ms_memory_recycle, ms_class, jit, jit_class, _no_grad
 from mindspore.common.dtype import Type, int8, byte, int16, short, int32, intc, int64, intp, \
     uint8, ubyte, uint16, ushort, uint32, uintc, uint64, uintp, float16, half, \
-    float32, single, float64, double, bool_, float_, list_, tuple_, int_, \
+    float32, single, float64, bfloat16, double, bool_, float_, list_, tuple_, int_, \
     uint, number, tensor_type, string, type_none, TensorType, Int, \
     complex64, complex128, dtype_to_nptype, _null, _NullType, \
     dtype_to_pytype, pytype_to_dtype, get_py_obj_dtype, QuantDtype
@@ -30,6 +30,7 @@ from mindspore.common.sparse_tensor import RowTensor, RowTensorInner, SparseTens
 from mindspore.common.mutable import mutable
 from mindspore.common.jit_config import JitConfig
 from mindspore.common.lazy_inline import lazy_inline
+from mindspore.common.mindir_util import load_mindir, save_mindir
 
 # symbols from dtype
 __all__ = [
@@ -55,7 +56,8 @@ __all__ = [
     "complex64", "complex128",
     # __method__ from dtype
     "dtype_to_nptype", "dtype_to_pytype",
-    "pytype_to_dtype", "get_py_obj_dtype"
+    "pytype_to_dtype", "get_py_obj_dtype",
+    "bfloat16",
 ]
 
 __all__.extend([
@@ -67,5 +69,5 @@ __all__.extend([
     "set_dump",
     "ms_memory_recycle",
     "mutable", "JitConfig",
-    "lazy_inline"
+    "lazy_inline", "load_mindir", "save_mindir"
 ])

mindspore/common/_jit_fallback_utils.py

@@ -103,3 +103,9 @@ def list_inplace_clear(list_obj):
         list_obj = []
     list_obj.clear()
     return list_obj
+
+
+def dict_inplace_setitem(dict_obj, key, target):
+    """Inplace dictionary setitem operation for dict_obj."""
+    dict_obj[key] = target
+    return dict_obj

mindspore/common/_register_for_adapter.py

@@ -20,6 +20,7 @@ class Registry:
     """Registry class for ms adapter."""
 
     def __init__(self):
+        self.is_registered = False
         self._tensor = None
         self._parameter = None
         self._convert_map = {}
@@ -48,6 +49,7 @@ class Registry:
         if self._tensor is not None:
             raise ValueError("Repeated registration of tensor in ms adapter config.")
         self._tensor = value
+        self.is_registered = True
 
     def register_parameter(self, value):
         """Register the parameter of ms adapter."""

mindspore/common/_register_for_tensor.py

@@ -17,7 +17,7 @@
 
 from __future__ import absolute_import
 from collections import UserDict
-from mindspore import context
+from mindspore._c_expression import Tensor as Tensor_
 
 
 class Registry(UserDict):
@@ -31,7 +31,7 @@ class Registry(UserDict):
         """Get the value by str."""
         if not isinstance(obj_str, str):
             raise TypeError("key for tensor registry must be string.")
-        if context.get_context("enable_ge"):
+        if Tensor_._is_test_stub() is True:  # pylint: disable=W0212
             def wrap(*args):
                 new_args = list(args)
                 new_args.append(obj_str)

mindspore/common/_stub_tensor.py

@@ -144,6 +144,8 @@ class StubTensor:
     slice_num_of_persistent_data = _stub_method(Tensor.slice_num_of_persistent_data)
     slice_shape_of_persistent_data = _stub_method(Tensor.slice_shape_of_persistent_data)
     flush_from_cache = _stub_method(Tensor.flush_from_cache)
+    contiguous = _stub_method(Tensor.contiguous)
+    is_contiguous = _stub_method(Tensor.is_contiguous)
 
     def stub_sync(self):
         """sync real tensor."""
@@ -156,6 +158,17 @@ class StubTensor:
             self.stub = None
         return self.tensor
 
+    def __getstate__(self):
+        state = {}
+        value = self.stub.get_value() if self.stub else self.tensor.__getstate__()
+        state["value"] = value
+        return state
+
+    def __setstate__(self, state):
+        value = state.pop("value")
+        self.stub = None
+        self.tensor = Tensor(value, internal=True)
+
 
 def _init_stub_tensor_api():
     """adapt to python tensor and cpp tensor api"""

mindspore/common/_utils.py

@@ -18,9 +18,11 @@
 
 import os
 import math
+import ctypes
 import functools
 
 import mindspore
+from mindspore import log as logger
 from mindspore.common import dtype as mstype
 from mindspore.parallel._ps_context import _is_ps_mode, _is_role_pserver, _is_role_sched
 
@@ -103,3 +105,30 @@ def ones_like(x):
 def zeros_like(x):
     """Implement `zeroslike`."""
     return mindspore.ops.composite.zeros_like(x)
+
+
+def load_lib(lib_path):
+    """load specified library."""
+    try:
+        ctypes.CDLL(lib_path)
+    # pylint: disable=broad-except
+    except Exception:
+        logger.warning(f'Loading {lib_path} lib error.')
+        return False
+    return True
+
+
+def _jit_fallback_next_func(xs):
+    """Generate ms_next for xs"""
+    if hasattr(xs, "__next__"):
+        # Convert an iterator to tuple first.
+        xs = tuple(xs)
+    return xs[0], xs[1:]
+
+
+def _jit_fallback_has_next_func(xs):
+    """Determine whether xs has next value"""
+    if hasattr(xs, "__next__"):
+        # Convert an iterator to tuple first.
+        xs = tuple(xs)
+    return len(xs) > 0