mindspore 2.6.0rc1__cp311-cp311-win_amd64.whl → 2.7.0__cp311-cp311-win_amd64.whl

mindspore/experimental/optim/rmsprop.py

@@ -1,48 +1,37 @@
-# Copyright 2023 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# The code implementation refers to the following files from pytorch:
+# - https://github.com/pytorch/pytorch/blob/v1.13.0/torch/optim/rmsprop.py
+# Additional modifications are made by Huawei Technologies Co., Ltd in 2023.
 # ============================================================================
 """rmsprop"""
 from __future__ import absolute_import
 
-from mindspore.ops import functional as F, composite as C, operations as P
 import mindspore.common.dtype as mstype
 from mindspore.experimental.optim.optimizer import Optimizer, check_not_less_than, check_not_less_than_without_equal
 from mindspore import ops
 from mindspore import jit
 
-_rmsprop_opt = C.MultitypeFuncGraph("rmsprop_opt")
+_rmsprop_opt = ops.MultitypeFuncGraph("rmsprop_opt")
 
-op_mul = P.Mul()
-op_sqrt = P.Sqrt()
+op_mul = ops.Mul()
+op_sqrt = ops.Sqrt()
 
 
 @_rmsprop_opt.register("Bool", "Number", "Number", "Number", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor")
 def _run_rmsprop_opt(centered, alpha, eps, momentum, lr, weight, mean_square, mean_grad, mom, grad):
     """Apply rmsprop optimizer to the weight parameter using dynamic learning rate."""
-    F.assign(mean_square, ops.addcmul(op_mul(mean_square, alpha), grad, grad, 1 - alpha))
+    ops.assign(mean_square, ops.addcmul(op_mul(mean_square, alpha), grad, grad, 1 - alpha))
 
     if centered:
-        F.assign(mean_grad, op_mul(mean_grad, alpha) + op_mul(grad, 1 - alpha))
+        ops.assign(mean_grad, op_mul(mean_grad, alpha) + op_mul(grad, 1 - alpha))
         avg = op_sqrt(ops.addcmul(mean_square, mean_grad, mean_grad, -1.)) + eps
     else:
         avg = op_sqrt(mean_square) + eps
 
     if momentum > 0:
-        F.assign(mom, op_mul(mom, momentum) + grad / avg)
-        F.assign(weight, weight - mom * lr)
+        ops.assign(mom, op_mul(mom, momentum) + grad / avg)
+        ops.assign(weight, weight - mom * lr)
     else:
-        F.assign(weight, weight - lr * grad / avg)
+        ops.assign(weight, weight - lr * grad / avg)
     return True
 
 
@@ -124,7 +113,7 @@ class RMSprop(Optimizer):
         self.mean_grad = self.parameters.clone(prefix="mean_grad", init='zeros')
         self.mean_square = self.parameters.clone(prefix="mean_square", init='zeros')
         self.moment = self.parameters.clone(prefix="moment", init='zeros')
-        self.op_cast = P.Cast()
+        self.op_cast = ops.Cast()
 
     @jit
     def implementation(self, group_id, lr, gradients, maximize, weight_decay, centered, alpha, eps, momentum):
@@ -132,12 +121,12 @@ class RMSprop(Optimizer):
         start_id = self.group_start_id[group_id]
         end_id = self.group_start_id[group_id + 1]
         params = self.parameters[start_id: end_id]
-        grads = tuple([grad if not maximize else F.neg(grad) for grad in gradients[start_id: end_id]])
+        grads = tuple([grad if not maximize else ops.neg(grad) for grad in gradients[start_id: end_id]])
         grads = self._decay_weight(weight_decay, params, grads)
         mean_grad = self.mean_grad[start_id: end_id]
         mean_square = self.mean_square[start_id: end_id]
         moment = self.moment[start_id: end_id]
-        self.hyper_map(F.partial(_rmsprop_opt, centered, alpha, eps, momentum, lr),
+        self.hyper_map(ops.partial(_rmsprop_opt, centered, alpha, eps, momentum, lr),
                        params, mean_square, mean_grad, moment, grads)
         return True
 

mindspore/experimental/optim/rprop.py

@@ -1,37 +1,26 @@
-# Copyright 2023 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# The code implementation refers to the following files from pytorch:
+# - https://github.com/pytorch/pytorch/blob/v1.13.0/torch/optim/rprop.py
+# Additional modifications are made by Huawei Technologies Co., Ltd in 2023.
 # ============================================================================
 """rprop"""
 from __future__ import absolute_import
 
-from mindspore.ops import functional as F, composite as C, operations as P
+from mindspore import ops
 from mindspore.common import Tensor, Parameter
 import mindspore.common.dtype as mstype
 from mindspore import _checkparam as validator
 from mindspore.experimental.optim.optimizer import Optimizer, check_not_less_than_without_equal
-from mindspore import ops
 from mindspore import jit
 
-_rprop_opt = C.MultitypeFuncGraph("rprop_opt")
+_rprop_opt = ops.MultitypeFuncGraph("rprop_opt")
 
-op_sign = P.Sign()
-op_fill = P.FillV2()
-op_assign = P.Assign()
-op_assignadd = P.AssignAdd()
-op_cast = P.Cast()
-op_select = P.Select()
-op_oneslike = P.OnesLike()
+op_sign = ops.Sign()
+op_fill = ops.FillV2()
+op_assign = ops.Assign()
+op_assignadd = ops.AssignAdd()
+op_cast = ops.Cast()
+op_select = ops.Select()
+op_oneslike = ops.OnesLike()
 
 
 @_rprop_opt.register("Tensor", "Tensor", "Number", "Number", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor")
@@ -131,7 +120,7 @@ class Rprop(Optimizer):
         self.step_size = self.parameters.clone(prefix="step_size", init='zeros')
         self.step_t = Parameter(Tensor(0, mstype.int32), "step_t")
         self.increase_tensor = Tensor(1, mstype.int32)
-        self.op_cast = P.Cast()
+        self.op_cast = ops.Cast()
 
     @jit(backend="ms_backend")
     def implementation(self, etaminus, etaplus, group_id, lr, gradients, maximize, step_size_min, step_size_max):
@@ -141,10 +130,10 @@ class Rprop(Optimizer):
         end_id = self.group_start_id[group_id + 1]
 
         params = self.parameters[start_id: end_id]
-        grads = tuple([grad if not maximize else F.neg(grad) for grad in gradients[start_id: end_id]])
+        grads = tuple([grad if not maximize else ops.neg(grad) for grad in gradients[start_id: end_id]])
         prev = self.prev[start_id: end_id]
         step_size = self.step_size[start_id: end_id]
-        self.hyper_map(F.partial(_rprop_opt, etaminus, etaplus, step_size_min, step_size_max, self.step_t, lr),
+        self.hyper_map(ops.partial(_rprop_opt, etaminus, etaplus, step_size_min, step_size_max, self.step_t, lr),
                        params, prev, step_size, grads)
         return True
 

mindspore/experimental/optim/sgd.py

@@ -1,28 +1,18 @@
-# Copyright 2023 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# The code implementation refers to the following files from pytorch:
+# - https://github.com/pytorch/pytorch/blob/v1.13.0/torch/optim/sgd.py
+# Additional modifications are made by Huawei Technologies Co., Ltd in 2023.
 # ============================================================================
 """sgd"""
 from __future__ import absolute_import
 
-from mindspore.ops import functional as F, composite as C, operations as P
+from mindspore import ops
 from mindspore.common.tensor import Tensor
 import mindspore.common.dtype as mstype
 from mindspore import _checkparam as Validator
 from mindspore.experimental.optim.optimizer import Optimizer
 from mindspore import jit
 
-_sgd_opt = C.MultitypeFuncGraph("sgd_opt")
+_sgd_opt = ops.MultitypeFuncGraph("sgd_opt")
 
 
 @_sgd_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor")
@@ -129,7 +119,7 @@ class SGD(Optimizer):
                              "equal to 0.0, but got 'momentum' {}, 'dampening' {}".format(momentum, dampening))
         self.accum = self.parameters.clone(prefix="accum", init='zeros')
         self.stat = self.parameters.clone(prefix="stat", init='ones')
-        self.op_cast = P.Cast()
+        self.op_cast = ops.Cast()
 
     @jit
     def implementation(self, momentum, lr, group_id, gradients, maximize, dampening, weight_decay, nesterov):
@@ -137,9 +127,9 @@ class SGD(Optimizer):
         start_id = self.group_start_id[group_id]
         end_id = self.group_start_id[group_id + 1]
         momentum = self.op_cast(momentum, mstype.float32)
-        opt = P.SGD(dampening, weight_decay, nesterov)
-        grads = tuple([grad if not maximize else F.neg(grad) for grad in gradients[start_id: end_id]])
-        self.hyper_map(F.partial(_sgd_opt, opt, momentum, lr), grads,
+        opt = ops.SGD(dampening, weight_decay, nesterov)
+        grads = tuple([grad if not maximize else ops.neg(grad) for grad in gradients[start_id: end_id]])
+        self.hyper_map(ops.partial(_sgd_opt, opt, momentum, lr), grads,
                        self.parameters[start_id: end_id], self.accum[start_id: end_id],
                        self.stat[start_id: end_id])
         return True

mindspore/hal/__init__.py

@@ -19,13 +19,13 @@ MindSpore abstracts the preceding modules from different backends and allows use
 resources at the Python layer. Currently, these interfaces take effect only in PyNative mode.
 """
 
-from mindspore.hal.device import is_initialized, is_available, device_count, get_device_capability,\
+from mindspore.hal.device import is_initialized, is_available, device_count, get_device_capability, \
                                  get_device_properties, get_device_name, get_arch_list
-from mindspore.hal.stream import Stream, synchronize, set_cur_stream, current_stream, default_stream,\
+from mindspore.hal.stream import Stream, synchronize, set_cur_stream, current_stream, default_stream, \
                                  communication_stream, StreamCtx
 from mindspore.hal.event import Event
-from mindspore.hal.memory import memory_stats, memory_reserved, max_memory_reserved, empty_cache,\
-                                 reset_peak_memory_stats, memory_summary, memory_allocated,\
+from mindspore.hal.memory import memory_stats, memory_reserved, max_memory_reserved, empty_cache, \
+                                 reset_peak_memory_stats, memory_summary, memory_allocated, \
                                  max_memory_allocated, reset_max_memory_reserved, reset_max_memory_allocated
 
 __all__ = [

mindspore/hal/contiguous_tensors_handle.py

@@ -27,7 +27,7 @@ def combine_tensor_list_contiguous(tensor_list, enable_mem_align=True):
     Return a contiguous memory handle where contiguous memory has been requested and slicing functionality is provided.
 
     Args:
-        tensor_list (list[Tensor], Tuple[Tensor]): The tensor list to be stored.
+        tensor_list (list[Tensor], tuple[Tensor]): The tensor list to be stored.
         enable_mem_align (bool, optional): Whether to enable the memory alignment function.
             False is not supported. Default ``True`` .
 
@@ -57,7 +57,7 @@ class ContiguousTensorsHandle:
     ContiguousTensorsHandle is a handle manage continuous memory.
 
     Args:
-        tensor_list (list[Tensor], Tuple[Tensor]): The tensor list to be stored.
+        tensor_list (list[Tensor], tuple[Tensor]): The tensor list to be stored.
         enable_mem_align (bool, optional): Whether to enable the memory alignment function.
             False is not supported. Default ``True`` .
 

mindspore/hal/memory.py

@@ -14,14 +14,18 @@
 # ============================================================================
 
 """Hardware memory interfaces."""
-from mindspore._c_expression import _memory_stats, _reset_max_mem_reserved, _reset_max_mem_allocated
+from mindspore._c_expression import _memory_stats, _reset_max_mem_reserved, _reset_max_mem_allocated, _empty_cache, \
+    DeviceContextManager
 from mindspore import log as logger
+import mindspore as ms
 from .device import _check_inputs_validation, is_initialized
 
+
 function_memory_status = {'memory_stats': False, 'memory_reserved': False, 'max_memory_reserved': False,
                           'empty_cache': False, 'reset_peak_memory_stats': False, 'memory_summary': False,
                           'memory_allocated': False, 'max_memory_allocated': False,
                           'reset_max_memory_reserved': False, 'reset_max_memory_allocated': False}
+_device_context_mgr = DeviceContextManager.get_instance()
 
 
 @_check_inputs_validation
@@ -131,20 +135,36 @@ def max_memory_reserved(device_target=None):
     return _memory_stats(device_target).get("max_reserved_memory", 0)
 
 
+def _is_initialized(device_target):
+    """
+    Returns whether specified backend is initialized.
+    """
+    _device_context = _device_context_mgr.get_device_context(device_target)
+    if _device_context is None:
+        return False
+    return _device_context.initialized()
+
+
 @_check_inputs_validation
 def empty_cache():
     """
-    Release all memory fragments in the memory pool, so that memory arrangement
-    will be optimized, this api will be deprecated and removed in future versions, please use
-    the api :func:`mindspore.runtime.empty_cache` instead.
+    Empty cache in the memory pool, this api will be deprecated and removed in future versions.
+    Please use the api :func:`mindspore.runtime.empty_cache` instead.
 
     Note:
-        Currently, the MindSpore memory pool does not have the function of releasing memory fragments.
-        This interface is reserved but implemented as an empty method and prompted in log mode when using.
+        - Empty cache help reduce the fragmentation of device memory.
+        - Support Atlas A2 series products.
+
+    Supported Platforms:
+        ``Ascend``
     """
     if not function_memory_status['empty_cache']:
         function_memory_status['empty_cache'] = True
-        logger.warning(f"The empty_cache operation is currently not supported.")
+    device_target = ms.context.get_context("device_target")
+    if not _is_initialized(device_target):
+        logger.warning(f"Backend {device_target} is not initialized yet.")
+        return
+    _empty_cache(device_target)
 
 
 @_check_inputs_validation

mindspore/include/api/cell.h

@@ -31,10 +31,26 @@ using Output = InputAndOutput;
 
 class MS_API CellBase {
  public:
+  /// \brief Constructor of Cellbase.
   CellBase() = default;
+  /// \brief Destructor of Cellbase.
   virtual ~CellBase() = default;
+  /// \brief Construct using inputs.
+  ///
+  /// \param[in] inputs Vector of inputs.
+  ///
+  /// \return Vector of outputs.
   virtual std::vector<Output> Construct(const std::vector<Input> &inputs) { return {}; }
+  /// \brief Clone a cellbase.
+  ///
+  /// \return Shared pointer of Cellbase.
   virtual std::shared_ptr<CellBase> Clone() const = 0;
+  /// \brief Run a cellbase.
+  ///
+  /// \param[in] inputs Vector of MSTensor as inputs.
+  /// \param[in] outputs Vector of MSTensor as outputs.
+  ///
+  /// \return Status of the operation.
   virtual Status Run(const std::vector<MSTensor> &inputs, std::vector<MSTensor> *outputs) { return kSuccess; }
   std::vector<Output> operator()(const std::vector<Input> &inputs) const;
 };
@@ -49,19 +65,50 @@ class MS_API Cell : public CellBase {
 class MS_API GraphCell final : public Cell<GraphCell> {
  public:
   class GraphImpl;
-
+  /// \brief Constructor of GraphCell.
   GraphCell() = default;
+  /// \brief Destructor of GraphCell.
   ~GraphCell() override = default;
-
+  /// \brief Constructor of GraphCell.
+  ///
+  /// \param[in] graph Graph to construct.
   explicit GraphCell(const Graph &graph);
+  /// \brief Constructor of GraphCell.
+  ///
+  /// \param[in] graph Graph to construct.
   explicit GraphCell(Graph &&graph);
+  /// \brief Constructor of GraphCell.
+  ///
+  /// \param[in] graph Graph to construct.
   explicit GraphCell(const std::shared_ptr<Graph> &graph);
-
+  /// \brief Set a context.
+  ///
+  /// \param[in] context Context to be set.
   void SetContext(const std::shared_ptr<Context> &context);
+  /// \brief Get back the graph.
+  ///
+  /// \return Graph of the graphcell.
   const std::shared_ptr<Graph> &GetGraph() const { return graph_; }
+  /// \brief Run the graphcell.
+  ///
+  /// \param[in] inputs Vector of MSTensor as inputs.
+  /// \param[in] outputs Vector of MSTensor as outputs.
+  ///
+  /// \return Status of the operation.
   Status Run(const std::vector<MSTensor> &inputs, std::vector<MSTensor> *outputs) override;
+  /// \brief Get the inputs.
+  ///
+  /// \return Inputs.
   std::vector<MSTensor> GetInputs();
+  /// \brief Get the outputs.
+  ///
+  /// \return Outputs.
   std::vector<MSTensor> GetOutputs();
+  /// \brief Load the device.
+  ///
+  /// \param[in] device_id Device id to be loaded.
+  ///
+  /// \return Status of the operation.
   Status Load(uint32_t device_id);
 
  private:
@@ -73,12 +120,25 @@ class MS_API GraphCell final : public Cell<GraphCell> {
 
 class MS_API InputAndOutput {
  public:
+  /// \brief Constructor of InputAndOutput.
   InputAndOutput();
+  /// \brief Destructor of InputAndOutput.
   ~InputAndOutput() = default;
-
+  /// \brief Constructor of InputAndOutput.
+  ///
+  /// \param[in] cell The cellbase.
+  ///
+  /// \param[in] prev The previous inputs/outputs.
+  ///
+  /// \param[in] index Index of inputs/outputs.
   InputAndOutput(const std::shared_ptr<CellBase> &cell, const std::vector<InputAndOutput> &prev, int32_t index);
-
+  /// \brief Get index.
+  ///
+  /// \return index Index of inputs/outputs.
   int32_t GetIndex() const { return index_; }
+  /// \brief Set index.
+  ///
+  /// \param[in] index Index to be set.
   void SetIndex(int32_t index) { index_ = index; }
 
  private:

mindspore/include/api/cfg.h

@@ -28,35 +28,50 @@ namespace mindspore {
 constexpr int iter_th = 1000;
 class MS_API MixPrecisionCfg {
  public:
+  /// \brief Constructor of mix precision training config.
   MixPrecisionCfg() {
     this->dynamic_loss_scale_ = false;
     this->loss_scale_ = 128.0f;
     this->keep_batchnorm_fp32_ = true;
     this->num_of_not_nan_iter_th_ = iter_th;
   }
+  /// \brief Constructor of mix precision training config.
+  ///
+  /// \param[in] rhs The config of mix precision.
   MixPrecisionCfg(const MixPrecisionCfg &rhs) {
     this->dynamic_loss_scale_ = rhs.dynamic_loss_scale_;
     this->loss_scale_ = rhs.loss_scale_;
     this->keep_batchnorm_fp32_ = rhs.keep_batchnorm_fp32_;
     this->num_of_not_nan_iter_th_ = rhs.num_of_not_nan_iter_th_;
   }
+  /// \brief Destructor of mix precision config.
   ~MixPrecisionCfg() = default;
 
-  bool dynamic_loss_scale_ = false;   /**< Enable/disable dynamic loss scale during mix precision training */
-  float loss_scale_;                  /**< Initial loss scale factor  */
-  bool keep_batchnorm_fp32_ = true;   /**< Keep batch norm in FP32 while training */
-  uint32_t num_of_not_nan_iter_th_;   /**< a threshold for modifying loss scale when dynamic loss scale is enabled */
-  bool is_raw_mix_precision_ = false; /**< Is mix precision model export from mindspore  */
+  /// \brief Enable/disable dynamic loss scale during mix precision training.
+  bool dynamic_loss_scale_ = false;
+  /// \brief Initial loss scale factor.
+  float loss_scale_;
+  /// \brief Keep batch norm in FP32 while training.
+  bool keep_batchnorm_fp32_ = true;
+  /// \brief A threshold for modifying loss scale when dynamic loss scale is enabled.
+  uint32_t num_of_not_nan_iter_th_;
+  /// \brief Is mix precision model export from mindspore.
+  bool is_raw_mix_precision_ = false;
 };
 
 class MS_API TrainCfg {
  public:
+  /// \brief Constructor of training config.
   TrainCfg() = default;
+  /// \brief Constructor of training config.
+  ///
+  /// \param[in] rhs The training config.
   TrainCfg(const TrainCfg &rhs) {
     this->loss_name_ = rhs.loss_name_;
     this->mix_precision_cfg_ = rhs.mix_precision_cfg_;
     this->accumulate_gradients_ = rhs.accumulate_gradients_;
   }
+  /// \brief Destructor of training config.
   ~TrainCfg() = default;
 
   /// \brief obtain part of the name that identify a loss kernel.
@@ -67,9 +82,11 @@ class MS_API TrainCfg {
   ///
   /// \param[in] loss_name define part of the name that identify a loss kernel.
   inline void SetLossName(const std::vector<std::string> &loss_name);
-
+  /// \brief Optimization level.
   OptimizationLevel optimization_level_ = kO0;
-  MixPrecisionCfg mix_precision_cfg_; /**< Mix precision configuration */
+  /// \brief Mix precision configuration.
+  MixPrecisionCfg mix_precision_cfg_;
+  /// \brief If accumulate gradients is used.
   bool accumulate_gradients_ = false;
 
  private:

mindspore/include/api/context.h

@@ -50,6 +50,7 @@ class DeviceInfoContext;
 /// \brief Context is used to store environment variables during execution.
 class MS_API Context {
  public:
+  /// \brief Data of context.
   struct Data;
   Context();
   ~Context() = default;

mindspore/include/api/delegate.h

@@ -129,12 +129,20 @@ class Delegate : public IDelegate<LiteDelegateGraph, kernel::Kernel, kernel::Ker
   ///
   /// \return Status. If Status is kLiteNotSupport, the program will return to the MindSpore Lite inner inference.
   virtual Status Init() = 0;
-
+  /// \brief Create kernel.
+  ///
+  /// \param[in] node The kernel to be created.
+  ///
+  /// \return Created kernel.
   std::shared_ptr<kernel::Kernel> CreateKernel(const std::shared_ptr<kernel::Kernel> &node) override {
     // return node as kernel since they are same one.
     return node;
   }
-
+  /// \brief Check if the node is delegate node.
+  ///
+  /// \param[in] node The kernel to verify.
+  ///
+  /// \return True if the node is delegate.
   bool IsDelegateNode(const std::shared_ptr<kernel::Kernel> &node) override { return false; }
 
   /// \brief Replace the nodes in model with delegate nodes, delegate will create kernels by its delegate nodes.