mindspore 2.7.0__cp310-cp310-win_amd64.whl → 2.7.1__cp310-cp310-win_amd64.whl

mindspore/ops/_grad_experimental/grad_comm_ops.py

@@ -29,7 +29,7 @@ from mindspore.common.sparse_tensor import RowTensorInner
 from mindspore.ops.composite.multitype_ops.zeros_like_impl import zeros_like
 from mindspore.ops.operations.comm_ops import (AllGather, _MiniStepAllGather, _HostAllGather, AllReduce,
                                                NeighborExchange, AlltoAll, AlltoAllV, NeighborExchangeV2,
-                                               Broadcast, AllGatherV, ReduceScatterV,
+                                               Broadcast, AlltoAllVC, AllGatherV, ReduceScatterV,
                                                _GetTensorSlice, _MirrorOperator, _MirrorMiniStepOperator, ReduceOp,
                                                ReduceScatter, _HostReduceScatter, _VirtualDiv, _VirtualAdd, _AllSwap,
                                                _VirtualAssignAdd, _VirtualAccuGrad, _MirrorMicroStepOperator,
@@ -37,6 +37,7 @@ from mindspore.ops.operations.comm_ops import (AllGather, _MiniStepAllGather, _H
                                                _VirtualAssignKvCache)
 from mindspore.ops._grad_experimental.grad_base import bprop_getters
 from mindspore.ops.operations import _grad_ops as G
+from mindspore.tools.sdc_detect import _sdc_detector
 import mindspore as ms
 
 _squared_device_local_norm = None
@@ -277,6 +278,8 @@ def get_bprop_mirror_micro_step_operator(self):
     if dump_device_local_norm:
         # init _squared _squared_device_local_norm
         squared_device_local_norm = get_squared_device_local_norm_param()
+    # feature value sampling for sdc detect
+    feat_value_dump_name = _sdc_detector.get_dump_name(param_name) if _sdc_detector.need_sample() else None
 
     def bprop(x, z, out, dout):
         if dump_local_norm or dump_device_local_norm:
@@ -289,6 +292,9 @@ def get_bprop_mirror_micro_step_operator(self):
             if dump_device_local_norm:
                 z = F.depend(z, F.assign_add(squared_device_local_norm,
                                              cast(squared_norm, squared_device_local_norm.dtype)))
+        if feat_value_dump_name and z.ndim > 1:
+            feat_value = square(F.max(F.abs(z))[0])
+            z = F.depend(z, tensor_dump(feat_value_dump_name, feat_value))
         real_grad = z
         assign_out = dout
         if issubclass_(F.typeof(dout), mstype.tensor_type):
@@ -343,14 +349,16 @@ def get_bprop_all_gather(self):
         ln_print = P.Print()
         tensor_dump = P.TensorDump()
         reduce_sum = P.ReduceSum(keep_dims=False)
-        square = P.Square()
         sqrt = P.Sqrt()
+    square = P.Square()
     if dump_local_norm_path:
         global_rank = get_rank()
         file = os.path.join(dump_local_norm_path, "rank_" + str(global_rank), "local_norm__" + param_name)
     if dump_device_local_norm:
         # init _squared _squared_device_local_norm
         squared_device_local_norm = get_squared_device_local_norm_param()
+    # feature value sampling for sdc detect
+    feat_value_dump_name = _sdc_detector.get_dump_name(param_name) if _sdc_detector.need_sample() else None
 
     def bprop(x, out, dout):
         if param_name and (dump_local_norm or dump_device_local_norm):
@@ -363,6 +371,9 @@ def get_bprop_all_gather(self):
             if dump_device_local_norm:
                 dout = F.depend(dout, F.assign_add(squared_device_local_norm,
                                                    cast(squared_norm, squared_device_local_norm.dtype)))
+        if param_name and feat_value_dump_name and dout.ndim > 1:
+            feat_value = square(F.max(F.abs(dout))[0])
+            dout = F.depend(dout, tensor_dump(feat_value_dump_name, feat_value))
 
         dx = reduce_scatter(dout)
         if mean_flag:
@@ -452,6 +463,8 @@ def get_bprop_micro_step_all_gather(self):
     if dump_device_local_norm:
         # init _squared _squared_device_local_norm
         squared_device_local_norm = get_squared_device_local_norm_param()
+    # feature value sampling for sdc detect
+    feat_value_dump_name = _sdc_detector.get_dump_name(param_name) if _sdc_detector.need_sample() else None
 
     def bprop(x, z, out, dout):
         if with_mirror_operator:
@@ -472,6 +485,9 @@ def get_bprop_micro_step_all_gather(self):
             if dump_device_local_norm:
                 z = F.depend(z, F.assign_add(squared_device_local_norm,
                                              cast(squared_norm, squared_device_local_norm.dtype)))
+        if feat_value_dump_name and z.ndim > 1:
+            feat_value = square(F.max(F.abs(z))[0])
+            z = F.depend(z, tensor_dump(feat_value_dump_name, feat_value))
         if not do_mirror:
             return (z, cast(out_tensor, dtype(z)))
         real_grad = reduce_scatter(z)
@@ -655,6 +671,21 @@ def get_bprop_all_to_all_v(self):
     return bprop
 
 
+@bprop_getters.register(AlltoAllVC)
+def get_bprop_all_to_all_v_c(self):
+    """Generate bprop for AlltoAllVC."""
+    all_to_all_v_c_grad = AlltoAllVC(self.group, self.block_size, transpose=True)
+    if hasattr(self, "instance_name") and self.instance_name:
+        instance_name = "grad" + self.instance_name
+        all_to_all_v_c_grad.set_prim_instance_name(instance_name)
+
+    def bprop(x, send_count_matrix, out, dout):
+        dx = all_to_all_v_c_grad(dout, send_count_matrix)
+        return (dx, zeros_like(send_count_matrix))
+
+    return bprop
+
+
 @bprop_getters.register(AllGatherV)
 def get_bprop_all_gather_v(self):
     """Generate bprop for AllGatherV."""
@@ -728,6 +759,8 @@ def get_bprop_mirror_operator(self):
     if dump_device_local_norm:
         # init _squared _squared_device_local_norm
         squared_device_local_norm = get_squared_device_local_norm_param()
+    # feature value sampling for sdc detect
+    feat_value_dump_name = _sdc_detector.get_dump_name(param_name) if _sdc_detector.need_sample() else None
     if dev_num > 1:
         dev_num_r = 1.0 / dev_num
         all_reduce = AllReduce(group=group)
@@ -762,6 +795,9 @@ def get_bprop_mirror_operator(self):
             if dump_device_local_norm:
                 dout = F.depend(dout, F.assign_add(squared_device_local_norm,
                                                    cast(squared_norm, squared_device_local_norm.dtype)))
+        if feat_value_dump_name and dout.ndim > 1:
+            feat_value = square(F.max(F.abs(dout))[0])
+            dout = F.depend(dout, tensor_dump(feat_value_dump_name, feat_value))
 
         if dev_num == 1:
             return (dout,)

mindspore/ops/_op_impl/aicpu/__init__.py

@@ -214,13 +214,8 @@ from .cumsum import _cumsum_aicpu
 from .round import _round_aicpu
 from .stft import _stft_aicpu
 from .floor_div import _floor_div_aicpu
-from .priority_replay_buffer import _prb_create_op_cpu
-from .priority_replay_buffer import _prb_push_op_cpu
 from .conjugate_transpose import _conjugate_transpose_aicpu
-from .priority_replay_buffer import _prb_sample_op_cpu
-from .priority_replay_buffer import _prb_update_op_cpu
 from .equal import _equal_aicpu
-from .priority_replay_buffer import _prb_destroy_op_cpu
 from .right_shift import _right_shift_aicpu
 from .tril import _tril_aicpu
 from .linspace import _lin_space_aicpu
@@ -242,10 +237,6 @@ from .sparse_tensor_to_csr_sparse_matrix import _sparse_tensor_to_csr_sparse_mat
 from .csr_sparse_matrix_to_sparse_tensor import _csr_sparse_matrix_to_sparse_tensor_aicpu
 from .linear_sum_assignment import _linear_sum_assignment_aicpu
 from .random_shuffle import _random_shuffle_aicpu
-from .reservoir_replay_buffer import _rrb_create_op_cpu
-from .reservoir_replay_buffer import _rrb_push_op_cpu
-from .reservoir_replay_buffer import _rrb_sample_op_cpu
-from .reservoir_replay_buffer import _rrb_destroy_op_cpu
 from .concat_offset import _concat_offset_aicpu
 from .range import _range_aicpu
 from .range_v2 import _range_v2_aicpu
@@ -414,7 +405,6 @@ from .segment_prod import _segment_prod_aicpu
 from .segment_sum import _segment_sum_aicpu
 from .set_size import _set_size_aicpu
 from .slice import _slice_aicpu
-from .sparse_cross import _sparse_cross_aicpu
 from .sparse_slice import _sparse_slice_aicpu
 from .sparse_softmax import _sparse_softmax_aicpu
 from .sparse_tensor_dense_add import _sparse_tensor_dense_add_aicpu

mindspore/ops/_op_impl/cpu/__init__.py

@@ -69,12 +69,7 @@ from .tensor_copy_slices import _tensor_copy_slices_cpu
 from .l2loss import _l2loss_cpu
 from .pyexecute import _pyexecute_cpu
 from .pyfunc import _pyfunc_cpu
-from .buffer_append import _buffer_append_cpu
-from .buffer_get import _buffer_get_cpu
 from .raise_op import _raise_cpu
 from .joinedstr_op import _joinedstr_cpu
-from .buffer_sample import _buffer_sample_cpu
-from .priority_replay_buffer import _prb_push_op_cpu
-from .priority_replay_buffer import _prb_sample_op_cpu
 from .space_to_batch_nd import _space_to_batch_nd_cpu
 from .sspaddmm import _sspaddmm_cpu

mindspore/ops/auto_generate/cpp_create_prim_instance_helper.py

@@ -105,6 +105,7 @@ op_args_default_value = {
     "DCTN": {"type": 2, "s": None, "axes": None, "norm": None},
     "DCT": {"type": 2, "n": None, "axis": -1, "norm": None},
     "Dense": {"bias": None},
+    "DequantSwigluQuant": {"bias": None, "quant_scale": None, "quant_offset": None, "group_index": None, "activate_left": False, "quant_mode": 'static'},
     "Diagonal": {"offset": 0, "dim1": 0, "dim2": 1},
     "DiagonalView": {"offset": 0, "dim1": 0, "dim2": 1},
     "DiagExt": {"diagonal": 0},
@@ -115,20 +116,10 @@ op_args_default_value = {
     "EluExt": {"alpha": 1.0},
     "EluGradExt": {"alpha": 1.0, "is_result": False},
     "Elu": {"alpha": 1.0},
-    "EmbeddingApplyAdamW": {"ams_grad": (0,), "mask_zero": (0,), "padding_key": (0,), "padding_key_mask": (1,), "completion_key": (0,), "completion_key_mask": (1,), "_embedding_dim": 1, "_max_key_num": 1},
-    "EmbeddingApplyAdam": {"mask_zero": (0,), "padding_key": (0,), "padding_key_mask": (1,), "completion_key": (0,), "completion_key_mask": (1,), "_embedding_dim": 1, "_max_key_num": 1},
-    "EmbeddingApplyAdaGrad": {"mask_zero": (0,), "padding_key": (0,), "padding_key_mask": (1,), "completion_key": (0,), "completion_key_mask": (1,), "_embedding_dim": 1, "_max_key_num": 1},
-    "EmbeddingApplyFtrl": {"mask_zero": (0,), "padding_key": (0,), "padding_key_mask": (1,), "completion_key": (0,), "completion_key_mask": (1,), "_embedding_dim": 1, "_max_key_num": 1},
-    "EmbeddingApplyRmsprop": {"mask_zero": (0,), "padding_key": (0,), "padding_key_mask": (1,), "completion_key": (0,), "completion_key_mask": (1,), "_embedding_dim": 1, "_max_key_num": 1},
-    "EmbeddingApplySgd": {"mask_zero": (0,), "padding_key": (0,), "padding_key_mask": (1,), "completion_key": (0,), "completion_key_mask": (1,), "_embedding_dim": 1, "_max_key_num": 1},
     "EmbeddingDenseBackward": {"padding_idx": None, "scale_grad_by_freq": False},
-    "EmbeddingFeatureMappingFileSize": {"only_offset_flag": True},
-    "EmbeddingFeatureMappingFind": {"num": 1},
-    "EmbeddingFeatureMappingImport": {"only_offset_flag": True, "num": 1},
     "Embedding": {"padding_idx": None, "max_norm": None, "norm_type": 2.0, "scale_grad_by_freq": False},
-    "EmbeddingTableEvict": {"steps_to_live": 0},
-    "EmptyLike": {"dtype": None, "device": None},
-    "Empty": {"dtype": None, "device": None},
+    "EmptyLike": {"dtype": None, "device": None, "pin_memory": False},
+    "Empty": {"dtype": None, "device": None, "pin_memory": False},
     "ExtractImagePatches": {"padding": 'VALID'},
     "FFNExt": {"expertTokens": None, "bias1": None, "bias2": None, "scale": None, "offset": None, "deqScale1": None, "deqScale2": None, "antiquant_scale1": None, "antiquant_scale2": None, "antiquant_offset1": None, "antiquant_offset2": None, "activation": 'fastgelu', "inner_precise": 0},
     "FFT2": {"s": None, "dim": (-2, -1), "norm": None},
@@ -179,6 +170,7 @@ op_args_default_value = {
     "IndexAddExt": {"alpha": 1},
     "InnerInplaceIndexPut": {"accumulate": False},
     "InnerMoeTokenUnpermute": {"probs": None, "padded_mode": False, "restore_shape": None},
+    "InnerUnique": {"sorted": True, "return_inverse": False},
     "InplaceAddmm": {"beta": 1, "alpha": 1},
     "InplaceAddsExt": {"alpha": 1},
     "InplaceAddExt": {"alpha": 1},
@@ -353,6 +345,9 @@ op_args_default_value = {
     "TopKRouter": {"drop_type": 0},
     "TopkExt": {"dim": -1, "largest": True, "sorted": True},
     "TopPRouter": {"drop_type": 0, "threshold": 0.0, "router_prob": 0.0},
+    "ToDevice": {"device": None, "dtype": None, "non_blocking": False, "copy": False},
+    "ToDtype": {"dtype": None, "non_blocking": False, "copy": False},
+    "ToOther": {"non_blocking": False, "copy": False},
     "TraceV2Grad": {"offset": 0, "axis1": 1, "axis2": 0},
     "TraceV2": {"offset": 0, "axis1": 1, "axis2": 0, "dtype": None},
     "TriangularSolve": {"upper": True, "transpose": False, "unitriangular": False},
@@ -387,7 +382,7 @@ op_args_default_value = {
     "FusedInferAttentionScore": {"pse_shift": None, "attn_mask": None, "actual_seq_lengths": None, "actual_seq_lengths_kv": None, "dequant_scale1": None, "quant_scale1": None, "dequant_scale2": None, "quant_scale2": None, "quant_offset2": None, "antiquant_scale": None, "antiquant_offset": None, "block_table": None, "query_padding_size": None, "kv_padding_size": None, "key_antiquant_scale": None, "key_antiquant_offset": None, "value_antiquant_scale": None, "value_antiquant_offset": None, "key_shared_prefix": None, "value_shared_prefix": None, "actual_shared_prefix_len": None, "num_heads": 1, "scale_value": 1.0, "pre_tokens": 2147483647, "next_tokens": 2147483647, "input_layout": 'BSH', "num_key_value_heads": 0, "sparse_mode": 0, "inner_precise": 1, "block_size": 0, "antiquant_mode": 0, "softmax_lse_flag": False, "key_antiquant_mode": 0, "value_antiquant_mode": 0},
     "GroupedMatmul": {"bias": None, "scale": None, "offset": None, "antiquant_scale": None, "antiquant_offset": None, "group_list": None, "split_item": 0, "group_type": -1, "transpose_a": False, "transpose_b": False},
     "GroupedMatmulV2": {"bias": None, "scale": None, "offset": None, "antiquant_scale": None, "antiquant_offset": None, "group_list": None, "split_item": 0, "group_type": -1},
-    "GroupedMatmulV4": {"bias": None, "scale": None, "offset": None, "antiquant_scale": None, "antiquant_offset": None, "pre_token_scale": None, "group_list": None, "activation_input": None, "activation_quant_scale": None, "activation_quant_offset": None, "split_item": 0, "group_type": -1, "group_list_type": 0, "act_type": 0},
+    "GroupedMatmulV4": {"bias": None, "scale": None, "offset": None, "antiquant_scale": None, "antiquant_offset": None, "pre_token_scale": None, "group_list": None, "activation_input": None, "activation_quant_scale": None, "activation_quant_offset": None, "split_item": 0, "group_type": -1, "group_list_type": 0, "act_type": 0, "output_dtype": None},
     "KVCacheScatterUpdate": {"reduce": 'none'},
     "MatmulAllReduceAddRmsNorm": {"reduce_op": 'sum', "comm_turn": 0, "stream_mode": 1},
     "MoeFinalizeRouting": {"x2": None, "bias": None, "scales": None, "expanded_row_idx": None, "expanded_expert_idx": None},
@@ -435,11 +430,13 @@ op_args_default_value = {
     "DeprecatedMedian": {"axis": -1, "keepdims": False},
     "DeprecatedMin": {"axis": None, "keepdims": False, "initial": None, "where": True, "return_indices": False},
     "DeprecatedNansum": {"axis": None, "keepdims": False, "dtype": None},
+    "DeprecatedPermute": {"axis": None},
     "DeprecatedProd": {"dim": None, "keepdim": False, "dtype": None},
     "DeprecatedRepeatInterleave": {"dim": None},
     "DeprecatedRoll": {"dims": None},
     "DeprecatedSort": {"axis": -1, "descending": False},
     "DeprecatedSplit": {"axis": 0},
+    "DeprecatedSqueeze": {"axis": None},
     "DeprecatedStd": {"axis": None, "ddof": 0, "keepdims": False},
     "DeprecatedSum": {"axis": None, "dtype": None, "keepdims": False, "initial": None},
     "DeprecatedTake": {"axis": None, "mode": 'clip'},
@@ -463,19 +460,10 @@ op_labels = {
     "BroadcastToView": {"side_effect_mem": True},
     "ChunkView": {"side_effect_mem": True},
     "DiagonalView": {"side_effect_mem": True},
-    "DistCommAllReduce": {"side_effect_mem": True},
-    "DistCommReduce": {"side_effect_mem": True},
     "DropoutExt": {"side_effect_hidden": True},
     "DropoutGenMaskExt": {"side_effect_hidden": True},
     "Dropout": {"side_effect_hidden": True},
-    "EmbeddingApplyAdamW": {"_process_node_engine_id": 'PS'},
-    "EmbeddingApplyAdam": {"_process_node_engine_id": 'PS'},
-    "EmbeddingApplyAdaGrad": {"_process_node_engine_id": 'PS'},
-    "EmbeddingApplyFtrl": {"_process_node_engine_id": 'PS'},
-    "EmbeddingApplyRmsprop": {"_process_node_engine_id": 'PS'},
-    "EmbeddingApplySgd": {"_process_node_engine_id": 'PS'},
     "Embedding": {"side_effect_mem": True},
-    "EmbeddingTableEvict": {"_process_node_engine_id": 'PS'},
     "ExpandDimsView": {"side_effect_mem": True},
     "Generator": {"side_effect_mem": True},
     "GroupTopk": {"side_effect_mem": True},
@@ -502,10 +490,12 @@ op_labels = {
     "InplaceGroupedMatmulAdd": {"side_effect_mem": True},
     "InplaceHardtanh": {"side_effect_mem": True},
     "InplaceIndexAddExt": {"side_effect_mem": True},
+    "InplaceIndexCopy": {"side_effect_mem": True},
     "InplaceIndexPut": {"side_effect_mem": True},
     "InplaceLog": {"side_effect_mem": True},
     "InplaceMaskedFillScalar": {"side_effect_mem": True},
     "InplaceMaskedFillTensor": {"side_effect_mem": True},
+    "InplaceMaskedScatter": {"side_effect_mem": True},
     "InplaceMatmulAdd": {"side_effect_mem": True},
     "InplaceMuls": {"side_effect_mem": True},
     "InplaceMul": {"side_effect_mem": True},
@@ -520,6 +510,8 @@ op_labels = {
     "InplaceScatterSrcReduce": {"side_effect_mem": True},
     "InplaceScatterValue": {"side_effect_mem": True},
     "InplaceScatterValueReduce": {"side_effect_mem": True},
+    "InplaceSigmoid": {"side_effect_mem": True},
+    "InplaceSign": {"side_effect_mem": True},
     "InplaceSiLU": {"side_effect_mem": True},
     "InplaceSubExt": {"side_effect_mem": True},
     "InplaceSubScalar": {"side_effect_mem": True},
@@ -542,5 +534,7 @@ op_labels = {
     "TransposeView": {"side_effect_mem": True},
     "UnstackExtView": {"side_effect_mem": True},
     "KVCacheScatterUpdate": {"side_effect_mem": True},
+    "DistCommAllReduce": {"side_effect_mem": True},
+    "DistCommReduce": {"side_effect_mem": True},
     "InplaceExponential": {"side_effect_mem": True},
 }

mindspore/ops/auto_generate/gen_extend_func.py

@@ -905,7 +905,6 @@ def histc(input, bins=100, min=0, max=0):
     Elements lower than min or higher than max are ignored.
     
     .. warning::
-        This is an experimental API that is subject to change or deletion.
         If input is int64, valid values fit within int32; exceeding this may cause precision errors.
     
     Args:
@@ -1732,9 +1731,6 @@ def outer(input, vec2):
     Return outer product of `input` and `vec2`. If `input` is a vector of size :math:`n`
     and `vec2` is a vector of size :math:`m` , then output must be a matrix of shape :math:`(n, m)` .
     
-    .. warning::
-        This is an experimental API that is subject to change or deletion.
-    
     .. note::
         This function does not broadcast.
     
@@ -1957,14 +1953,13 @@ def stack(tensors, dim=0):
     :math:`(x_1, x_2, ..., x_{dim}, N, x_{dim+1}, ..., x_R)`.
     
     Args:
-        tensors (Union[tuple, list]): A Tuple or list of Tensor objects with the same shape and type.
+        tensors (Union[tuple, list]): A Tuple or list of Tensor objects with the same shape.
         dim (int, optional): Dimension to stack. The range is [-(R+1), R+1). Default: ``0`` .
     
     Returns:
-        Tensor. A stacked Tensor with the same type as `tensors`.
+        A stacked Tensor.
     
     Raises:
-        TypeError: If the data types of elements in `tensors` are not the same.
         ValueError: If `dim` is out of the range [-(R+1), R+1);
                     or if the shapes of elements in `tensors` are not the same.