mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/numpy/utils.py

@@ -18,6 +18,7 @@ from __future__ import absolute_import
 import types
 
 from mindspore.common import Tensor
+from mindspore._c_expression import Tensor as Tensor_
 from mindspore.ops import functional as F
 from mindspore.common import dtype as mstype
 
@@ -127,6 +128,8 @@ def _to_tensor(*args):
     res = ()
     for arg in args:
         if isinstance(arg, (int, float, bool, list, tuple)):
+            if isinstance(arg, (list, tuple)) and not arg:
+                arg = Tensor_(arg)
             arg = _convert_64_to_32(_type_convert(Tensor, arg))
         elif not isinstance(arg, Tensor):
             _raise_type_error("Expect input to be array like.")

mindspore/numpy/utils_const.py

@@ -208,7 +208,7 @@ def _check_shape_aligned(shape1, shape2):
 @_primexpr
 def _tile_size(shape, out_shape, ndim):
     """Returns tile_size such that shape*tile_size = out_shape"""
-    size = [1]*ndim
+    size = [1] * ndim
     for idx, (i, j) in enumerate(zip(shape, out_shape)):
         if i != j:
             size[idx] = j
@@ -441,7 +441,7 @@ def _add_unit_axes(shape, ndim, append=False):
 
 
 @constexpr
-def  _check_element_int(lst):
+def _check_element_int(lst):
     """
     Check whether each element in `lst` is an integer.
     """
@@ -508,7 +508,7 @@ def _iota(dtype, num, increasing=True):
         raise ValueError("zero shape Tensor is not currently supported.")
     if increasing:
         return Tensor(list(range(int(num))), dtype)
-    return Tensor(list(range(int(num)-1, -1, -1)), dtype)
+    return Tensor(list(range(int(num) - 1, -1, -1)), dtype)
 
 
 @constexpr
@@ -520,7 +520,7 @@ def _ceil(number):
 @_primexpr
 def _seq_prod(seq1, seq2):
     """Returns the element-wise product of seq1 and seq2."""
-    return tuple(map(lambda x, y: x*y, seq1, seq2))
+    return tuple(map(lambda x, y: x * y, seq1, seq2))
 
 
 @constexpr

mindspore/ops/__init__.py

@@ -23,31 +23,34 @@ Examples:
 from __future__ import absolute_import
 
 from mindspore.common import Tensor
-from mindspore.ops.primitive import Primitive, PrimitiveWithInfer, PrimitiveWithCheck, prim_attr_register
+from mindspore.ops.primitive import Primitive, PrimitiveWithInfer, PrimitiveWithCheck, prim_attr_register, \
+    prim_arg_register
 from mindspore.ops.vm_impl_registry import get_vm_impl_fn, vm_impl_registry
 from mindspore.ops.op_info_register import op_info_register, custom_info_register, AkgGpuRegOp, AkgAscendRegOp, \
     AiCPURegOp, TBERegOp, CpuRegOp, CustomRegOp, DataType
 from mindspore.ops.primitive import constexpr
 from mindspore.ops import composite, operations, functional, function
 from mindspore.ops import signature
+from mindspore.ops.auto_generate import cpp_create_prim_instance_helper, gen_arg_dtype_cast, gen_arg_handler, \
+    gen_extend_func, gen_ops_def, gen_ops_prim, pyboost_inner_prim
 from mindspore.ops.composite import *
 from mindspore.ops.operations import *
 from mindspore.ops.function import *
 from mindspore.ops.functional import *
-from mindspore.ops.silent_check import _silent_check
 
 __primitive__ = [
-    "prim_attr_register", "Primitive", "PrimitiveWithInfer", "PrimitiveWithCheck", "signature"
+    "prim_attr_register", "prim_arg_register", "Primitive", "PrimitiveWithInfer", "PrimitiveWithCheck", "signature"
 ]
 
 __all__ = ["get_vm_impl_fn", "vm_impl_registry",
            "op_info_register", "custom_info_register", "AkgGpuRegOp", "AkgAscendRegOp", "AiCPURegOp", "TBERegOp",
            "CpuRegOp", "CustomRegOp", "DataType",
-           "constexpr"]
+           "constexpr", "reshard",
+           "cpp_create_prim_instance_helper", "gen_arg_dtype_cast", "gen_arg_handler", "gen_extend_func", "gen_ops_def",
+           "gen_ops_prim", "pyboost_inner_prim"]
 __all__.extend(__primitive__)
 __all__.extend(composite.__all__)
 __all__.extend(operations.__all__)
 __all__.extend(functional.__all__)
 __all__.extend(function.__all__)
-
-_silent_check()
+__all__.extend(auto_generate.__all__)

mindspore/ops/_grad_experimental/grad_array_ops.py

@@ -201,7 +201,7 @@ def get_bprop_index_put(self):
         if is_ascend:
             indices_ms = [convert_idx_positive(indices_ms[i], x1.shape[i]) for i in range(len(indices_ms))]
         indices_me = stack(indices_ms)
-        indices_grad = F.transpose(indices_me, F.make_range(F.rank(indices_me) - 1, -1, -1))
+        indices_grad = F.transpose(indices_me, F.make_range(F.rank(indices_me)-1, -1, -1))
         values_grad = gather_nd(dout, indices_grad)
         if equal(cast(x2.shape[0], mstype.int32), Tensor(1)):
             values_grad = values_grad.sum().reshape(1)
@@ -358,99 +358,6 @@ def get_bprop_resize_nearest_neighbor_v2(self):
     return bprop
 
 
-@bprop_getters.register(P.ExtractVolumePatches)
-def get_bprop_extract_volume_patches(self):
-    """Generate bprop for ExtractVolumePatches"""
-    extract_volume_patches = P.ExtractVolumePatches(kernel_size=self.kernel_size,
-                                                    strides=self.strides, padding=self.padding)
-    concat = P.Concat(axis=-1)
-    expend_dims = P.ExpandDims()
-    scatter_nd = P.ScatterNd()
-    slice_op = P.Slice()
-    dtype = P.DType()
-    cast = P.Cast()
-    matmul = P.MatMul()
-    _, _, ksize_d, ksize_h, ksize_w = self.kernel_size
-    range_ = P.Range()
-    ones_like = P.OnesLike()
-
-    def _dyn_extract_volume_patches(x, out, dout):
-        x_shape = shape_op(x)
-        out_shape = shape_op(out)
-        x_n, x_c, x_d, x_h, x_w = x_shape[0], x_shape[1], x_shape[2], x_shape[3], x_shape[4]
-        x_indices_num = 1 + x_d * x_h * x_w
-        x_idx = range_(cast(1, mstype.float32), cast(x_indices_num, mstype.float32), cast(1, mstype.float32))
-        x_idx = cast(x_idx, mstype.float16)
-        x_idx = P.Reshape()(x_idx, create_tensor_by_element((1, 1, x_d, x_h, x_w)))
-        x_idx_patched = extract_volume_patches(x_idx)
-        x_idx_patched = P.Transpose()(x_idx_patched, (0, 2, 3, 4, 1))
-        x_idx_patched = cast(x_idx_patched, mstype.int32)
-
-        out_d, out_h, out_w = out_shape[2], out_shape[3], out_shape[4]
-        out_indices_num = out_d * out_h * out_w * ksize_d * ksize_h * ksize_w
-        out_idx_ori = range_(cast(0, mstype.int32), cast(out_indices_num, mstype.int32), cast(1, mstype.int32))
-        out_idx = P.Reshape()(out_idx_ori,
-                              create_tensor_by_element((1, out_d, out_h, out_w, ksize_d * ksize_h * ksize_w)))
-
-        idx_tensor = concat((expend_dims(x_idx_patched, -1), expend_dims(out_idx, -1)))
-        idx_map = P.Reshape()(idx_tensor, (-1, 2))
-        sp_shape = create_tensor_by_element((x_indices_num, out_indices_num))
-        update = cast(ones_like(out_idx_ori), dtype(dout))
-        sp_mat_full = scatter_nd(idx_map, update, sp_shape)
-        begin = create_tensor_by_element((1, 0))
-        size = create_tensor_by_element((x_indices_num - 1, out_indices_num))
-        sp_tensor = slice_op(sp_mat_full, begin, size)
-
-        grad = P.Transpose()(dout, (0, 2, 3, 4, 1))
-        grad = P.Reshape()(grad, create_tensor_by_element((x_n, out_d, out_h, out_w, ksize_d,
-                                                           ksize_h, ksize_w, x_c)))
-        grad_expended = P.Transpose()(grad, (1, 2, 3, 4, 5, 6, 0, 7))
-        grad_flat = P.Reshape()(grad_expended,
-                                create_tensor_by_element((out_d * out_h * out_w * ksize_d * ksize_h * ksize_w,
-                                                          x_n * x_c)))
-        jac = matmul(sp_tensor, grad_flat)
-        dx = P.Reshape()(jac, create_tensor_by_element((x_d, x_h, x_w, x_n, x_c)))
-        dx = P.Transpose()(dx, (3, 4, 0, 1, 2))
-        return (dx,)
-
-    def bprop(x, out, dout):
-        x_shape = P.Shape()(x)
-        out_shape = P.Shape()(out)
-        if F.is_sequence_value_unknown(x_shape) or F.is_sequence_value_unknown(out_shape):
-            return _dyn_extract_volume_patches(x, out, dout)
-        x_n, x_c, x_d, x_h, x_w = x_shape
-        x_indices_num = 1 + x_d * x_h * x_w
-        x_idx = cast(F.tuple_to_array(range(1, x_indices_num)), mstype.float16)
-        x_idx = P.Reshape()(x_idx, (1, 1, x_d, x_h, x_w))
-        x_idx_patched = extract_volume_patches(x_idx)
-        x_idx_patched = P.Transpose()(x_idx_patched, (0, 2, 3, 4, 1))
-        x_idx_patched = cast(x_idx_patched, mstype.int32)
-
-        _, _, out_d, out_h, out_w = out_shape
-        out_indices_num = out_d * out_h * out_w * ksize_d * ksize_h * ksize_w
-        out_idx = F.tuple_to_array(range(0, out_indices_num))
-        out_idx = P.Reshape()(out_idx, (1, out_d, out_h, out_w, ksize_d * ksize_h * ksize_w))
-
-        idx_tensor = concat((expend_dims(x_idx_patched, -1), expend_dims(out_idx, -1)))
-        idx_map = P.Reshape()(idx_tensor, (-1, 2))
-        sp_shape = (x_indices_num, out_indices_num)
-        sp_mat_full = scatter_nd(idx_map, F.fill(dtype(dout), (out_indices_num,), 1), sp_shape)
-        sp_tensor = slice_op(sp_mat_full, (1, 0), (x_indices_num - 1, out_indices_num))
-
-        grad = P.Transpose()(dout, (0, 2, 3, 4, 1))
-        grad = P.Reshape()(grad, (x_n, out_d, out_h, out_w, ksize_d,
-                                  ksize_h, ksize_w, x_c))
-        grad_expended = P.Transpose()(grad, (1, 2, 3, 4, 5, 6, 0, 7))
-        grad_flat = P.Reshape()(grad_expended, (-1, x_n * x_c))
-
-        jac = matmul(sp_tensor, grad_flat)
-        dx = P.Reshape()(jac, (x_d, x_h, x_w, x_n, x_c))
-        dx = P.Transpose()(dx, (3, 4, 0, 1, 2))
-        return (dx,)
-
-    return bprop
-
-
 @bprop_getters.register(SegmentSum)
 def get_bprop_segment_sum(self):
     """Generate bprop for SegmentSum"""
@@ -784,26 +691,6 @@ def get_bprop_segment_mean(self):
     return bprop
 
 
-@bprop_getters.register(P.Ones)
-def get_bprop_ones(self):
-    """Generate bprop for Ones"""
-
-    def bprop(dims, dtype, out, dout):
-        return zeros_like(dims)
-
-    return bprop
-
-
-@bprop_getters.register(P.Zeros)
-def get_bprop_zeros(self):
-    """Generate bprop for Zeros"""
-
-    def bprop(dims, dtype, out, dout):
-        return zeros_like(dims)
-
-    return bprop
-
-
 @bprop_getters.register(P.EmbeddingLookup)
 def get_bprop_embedding_lookup(self):
     """Generate bprop for EmbeddingLookup"""
@@ -816,14 +703,14 @@ def get_bprop_embedding_lookup(self):
             raise RuntimeError("Now, EmbeddingLookup op's grad don't support Dynamic Sense!")
         new_indices = sub_op(indices, offset)
         indices_size = size_op(new_indices)
+        x_shp_tail = x_shp[1:]
         if indices_size > 0:
             # Reshape the 'new_indices'
             new_indices_shape_changed = (indices_size,)
             new_indices = reshape_op(new_indices, new_indices_shape_changed)
+            actual_dout_shape_changed = new_indices_shape_changed + x_shp_tail
         else:
-            new_indices_shape_changed = ()
-        x_shp_tail = x_shp[1:]
-        actual_dout_shape_changed = new_indices_shape_changed + x_shp_tail
+            actual_dout_shape_changed = x_shp_tail
         # Reshape the 'actual_dout' on device
         actual_dout = reshape_op(dout, actual_dout_shape_changed)
         return RowTensorInner(new_indices, actual_dout, x_shp), zeros_like(indices), zeros_like(offset)
@@ -931,18 +818,6 @@ def get_bprop_scatter_update(self):
     return bprop
 
 
-@bprop_getters.register(P.TransShape)
-def get_bprop_trans_shape(self):
-    """Generate bprop for TransShape"""
-    op = P.TransShape()
-
-    def bprop(x, shape, out, dout):
-        dx = op(dout, shape_op(x))
-        return (dx, zeros_like(shape))
-
-    return bprop
-
-
 @bprop_getters.register(P.Unique)
 def get_bprop_unique(self):
     """Generate bprop for Unique"""

mindspore/ops/_grad_experimental/grad_comm_ops.py

@@ -22,7 +22,8 @@ from mindspore.ops import functional as F
 from mindspore.communication import get_rank, get_group_size
 from mindspore.parallel._utils import _get_enable_parallel_optimizer, _get_grad_accumulation_shard
 from mindspore.ops import operations as P
-from mindspore.ops.operations._inner_ops import Send, Receive, issubclass_
+from mindspore.ops import Send, Receive
+from mindspore.ops.operations._inner_ops import issubclass_
 from mindspore.common.sparse_tensor import RowTensorInner
 from mindspore.ops.composite.multitype_ops.zeros_like_impl import zeros_like
 from mindspore.ops.operations.comm_ops import (AllGather, _MiniStepAllGather, _HostAllGather, AllReduce,
@@ -30,9 +31,10 @@ from mindspore.ops.operations.comm_ops import (AllGather, _MiniStepAllGather, _H
                                                _GetTensorSlice, _MirrorOperator, _MirrorMiniStepOperator, ReduceOp,
                                                ReduceScatter, _HostReduceScatter, _VirtualDiv, _VirtualAdd, _AllSwap,
                                                _VirtualAssignAdd, _VirtualAccuGrad, _MirrorMicroStepOperator,
-                                               _MicroStepAllGather)
+                                               _MicroStepAllGather, Reduce, CollectiveGather, CollectiveScatter)
 from mindspore.ops._grad_experimental.grad_base import bprop_getters
 from mindspore.ops.operations import _grad_ops as G
+import mindspore as ms
 
 
 @bprop_getters.register(AllReduce)
@@ -41,7 +43,7 @@ def get_bprop_all_reduce(self):
 
     all_reduce_grad = AllReduce(ReduceOp.SUM, self.group)
     all_gather = AllGather(group=self.group)
-    if self.instance_name:
+    if hasattr(self, "instance_name") and self.instance_name:
         instance_name = "grad" + self.instance_name
         all_reduce_grad.set_prim_instance_name(instance_name)
     equal = P.Equal()
@@ -94,6 +96,12 @@ def get_bprop_send(self):
     dtype = self.get_attr_dict()["dtype"]
     tag = self.get_attr_dict()["sr_tag"]
     send_grad = Receive(tag, self.rank, shape, dtype, self.group_back)
+    if "dst_global_rank" in self.get_attr_dict():
+        dst_global_rank = self.get_attr_dict().get("dst_global_rank")
+        send_grad.add_prim_attr("src_global_rank", dst_global_rank)
+    if "RING_ATTENTION_INDEX" in self.get_attr_dict():
+        ringattention = self.get_attr_dict().get("RING_ATTENTION_INDEX")
+        send_grad.add_prim_attr("RING_ATTENTION_INDEX", ringattention)
     virtual_input = Tensor(0.0, dtype)
 
     def bprop(x, out, dout):
@@ -107,7 +115,16 @@ def get_bprop_send(self):
 def get_bprop_receive(self):
     """Generate bprop for Receive."""
     tag = self.get_attr_dict()["sr_tag"]
+    flash_tag = self.get_attr_dict().get("flash_tag")
     receive_grad = Send(tag, self.rank, self.group_back)
+    shape = self.get_attr_dict()["shape"]
+    receive_grad.add_prim_attr("shape", shape)
+    if "src_global_rank" in self.get_attr_dict():
+        src_global_rank = self.get_attr_dict().get("src_global_rank")
+        receive_grad.add_prim_attr("dst_global_rank", src_global_rank)
+    if "RING_ATTENTION_INDEX" in self.get_attr_dict():
+        ringattention = self.get_attr_dict().get("RING_ATTENTION_INDEX")
+        receive_grad.add_prim_attr("RING_ATTENTION_INDEX", ringattention)
     depend = P.Depend()
     cast = P.Cast()
     out_tensor = Tensor(0.0, mstype.float16)
@@ -115,7 +132,7 @@ def get_bprop_receive(self):
 
     def bprop(x, out, dout):
         send_out = receive_grad(dout)
-        if is_opt_shard:
+        if is_opt_shard or (flash_tag == "True"):
             dx = depend(F.zeros_like(x), send_out)
         else:
             dx = depend(cast(out_tensor, F.dtype(x)), send_out)
@@ -184,6 +201,9 @@ def get_bprop_mirror_micro_step_operator(self):
     group = self.group
     dev_num = self.dev_num
     mean_flag = self.mean_flag
+    param_name = " "
+    if 'mirror_user_id' in self.get_attr_dict():
+        param_name = self.get_attr_dict()['mirror_user_id']
     scale = 1 / dev_num
 
     all_reduce = AllReduce(group=group)
@@ -194,7 +214,6 @@ def get_bprop_mirror_micro_step_operator(self):
     if hasattr(self, 'parameter'):
         parameter = self.parameter
         all_reduce.add_prim_attr("parameter", parameter)
-
     if self.instance_name:
         instance_name = "grad_mirror" + self.instance_name
         all_reduce.set_prim_instance_name(instance_name)
@@ -205,25 +224,27 @@ def get_bprop_mirror_micro_step_operator(self):
         assign.add_prim_attr("parameter_micro", 0)
     out_tensor = Tensor(1.0, mstype.float16)
     opt_shard = _get_enable_parallel_optimizer()
+    ln_print = P.Print()
+    reduce_sum = P.ReduceSum(keep_dims=False)
+    square = P.Square()
+    dump_local_norm = ms.get_auto_parallel_context("dump_local_norm")
 
     def bprop(x, z, out, dout):
+        if dump_local_norm:
+            z = F.depend(z, ln_print("dump local norm: ", param_name, reduce_sum(square((z)))))
         real_grad = z
         assign_out = dout
-        if mean_flag:
-            if issubclass_(F.typeof(dout), mstype.tensor_type):
-                z = F.depend(z, dout)
-                real_grad = all_reduce(z)
-                real_grad = F.tensor_mul(real_grad, scale)
-                if opt_shard:
-                    return (real_grad, cast(out_tensor, dtype(z)))
-                return F.depend((cast(out_tensor, dtype(x)), cast(out_tensor, dtype(z))), assign(z, real_grad))
-        else:
-            if issubclass_(F.typeof(dout), mstype.tensor_type):
-                z = F.depend(z, dout)
+        if issubclass_(F.typeof(dout), mstype.tensor_type):
+            z = F.depend(z, dout)
+            if dev_num > 1:
                 real_grad = all_reduce(z)
-                if opt_shard:
-                    return (real_grad, cast(out_tensor, dtype(z)))
-                return F.depend((cast(out_tensor, dtype(x)), cast(out_tensor, dtype(z))), assign(z, real_grad))
+                if mean_flag:
+                    real_grad = F.tensor_mul(real_grad, scale)
+            else:
+                real_grad = z
+            if opt_shard:
+                return (real_grad, cast(out_tensor, dtype(z)))
+            return F.depend((cast(out_tensor, dtype(x)), cast(out_tensor, dtype(z))), assign(z, real_grad))
         return F.depend((cast(out_tensor, dtype(x)), cast(out_tensor, dtype(z))), assign_out)
 
     return bprop
@@ -243,11 +264,13 @@ def get_bprop_broad_cast(self):
 def get_bprop_all_gather(self):
     """Generate bprop for AllGather"""
     fusion = self.get_attr_dict()["fusion"]
+    self.group = self.get_attr_dict()["group"]
     reduce_scatter = ReduceScatter(ReduceOp.SUM, self.group).add_prim_attr("fusion", fusion)
-    if self.instance_name:
+    if hasattr(self, "instance_name") and self.instance_name:
         instance_name = "grad_" + self.instance_name
         reduce_scatter.set_prim_instance_name(instance_name)
     mean_flag = self.get_attr_dict()["mean_flag"]
+    self.rank_size = self.get_attr_dict()["rank_size"]
     if self.rank_size == 0:
         raise ValueError(f"The 'rank_size' can not be zero, but got {self.rank_size}.")
     scale = 1.0 / self.rank_size
@@ -272,7 +295,7 @@ def get_bprop_mini_step_all_gather(self):
     scale = 1 / self.rank_size
     all_reduce = AllReduce(ReduceOp.SUM, self.group).add_prim_attr("fusion", fusion)
     assign_add = P.AssignAdd()
-    if self.instance_name:
+    if hasattr(self, "instance_name") and self.instance_name:
         instance_name = "grad_" + self.instance_name
         all_reduce.set_prim_instance_name(instance_name)
     rank = get_rank(self.group)
@@ -309,39 +332,43 @@ def get_bprop_micro_step_all_gather(self):
     """Generate bprop for _MicroStepAllGather"""
     fusion = self.get_attr_dict()["fusion"]
     mean_flag = self.get_attr_dict()["mean_flag"]
+    param_name = " "
+    if 'mirror_user_id' in self.get_attr_dict():
+        param_name = self.get_attr_dict()['mirror_user_id']
     do_mirror = False
     if self.group != "":
         do_mirror = self.get_attr_dict()["do_mirror"]
     if do_mirror:
         scale = 1.0 / self.rank_size
-        all_reduce = AllReduce(ReduceOp.SUM, self.group).add_prim_attr("fusion", fusion)
+        reduce_scatter = ReduceScatter(ReduceOp.SUM, self.group).add_prim_attr("fusion", fusion)
         if "segment" in self.get_attr_dict():
-            all_reduce.add_prim_attr("segment", self.get_attr_dict()["segment"])
-        rank = get_rank(self.group)
-        dev_num = get_group_size(self.group)
-        split = P.Split(output_num=dev_num)
+            reduce_scatter.add_prim_attr("segment", self.get_attr_dict()["segment"])
         if self.instance_name:
             instance_name = "grad_" + self.instance_name
-            all_reduce.set_prim_instance_name(instance_name)
+            reduce_scatter.set_prim_instance_name(instance_name)
     cast = P.Cast()
     dtype = P.DType()
     out_tensor = Tensor(1.0, mstype.float16)
     with_mirror_operator = self.get_attr_dict()["with_mirror_operator"]
+    ln_print = P.Print()
+    reduce_sum = P.ReduceSum(keep_dims=False)
+    square = P.Square()
+    dump_local_norm = ms.get_auto_parallel_context("dump_local_norm")
 
     def bprop(x, z, out, dout):
         if with_mirror_operator:
             if not do_mirror:
                 return (dout, cast(out_tensor, dtype(z)))
-            real_grad = all_reduce(dout)
-            real_grad = split(real_grad)[rank]
+            real_grad = reduce_scatter(dout)
             if mean_flag:
                 real_grad = F.tensor_mul(real_grad, scale)
             return (real_grad, cast(out_tensor, dtype(z)))
         z = F.depend(z, dout)
+        if dump_local_norm:
+            z = F.depend(z, ln_print("dump local norm: ", param_name, reduce_sum(square((z)))))
         if not do_mirror:
             return (z, cast(out_tensor, dtype(z)))
-        real_grad = all_reduce(z)
-        real_grad = split(real_grad)[rank]
+        real_grad = reduce_scatter(z)
         if mean_flag:
             real_grad = F.tensor_mul(real_grad, scale)
         return (real_grad, cast(out_tensor, dtype(z)))
@@ -353,7 +380,7 @@ def get_bprop_micro_step_all_gather(self):
 def get_bprop_host_all_gather(self):
     """Generate bprop for _HostAllGather"""
     host_all_gather_grad = _HostReduceScatter(ReduceOp.SUM, self.group)
-    if self.instance_name:
+    if hasattr(self, "instance_name") and self.instance_name:
         instance_name = "grad" + self.instance_name
         host_all_gather_grad.set_prim_instance_name(instance_name)
 
@@ -368,7 +395,7 @@ def get_bprop_host_all_gather(self):
 def get_bprop_reduce_scatter(self):
     """Generate bprop for ReduceScatter"""
     reduce_scatter_grad = AllGather(self.group)
-    if self.instance_name:
+    if hasattr(self, "instance_name") and self.instance_name:
         instance_name = "grad" + self.instance_name
         reduce_scatter_grad.set_prim_instance_name(instance_name)
 
@@ -382,11 +409,71 @@ def get_bprop_reduce_scatter(self):
     return bprop
 
 
+@bprop_getters.register(Reduce)
+def get_bprop_reduce(self):
+    """Generate bprop for Reduce"""
+    dest_rank = self.get_attr_dict()["dest_rank"]
+    group = self.get_attr_dict()["group"]
+    reduce_grad = Broadcast(dest_rank, group)
+    if hasattr(self, "instance_name") and self.instance_name:
+        instance_name = "grad" + self.instance_name
+        reduce_grad.set_prim_instance_name(instance_name)
+
+    def bprop(x, out, dout):
+        dx = reduce_grad((dout,))
+        return (dx[0],)
+
+    return bprop
+
+
+@bprop_getters.register(CollectiveGather)
+def get_bprop_collective_gather(self):
+    """Generate bprop for CollectiveGather"""
+    group = self.get_attr_dict()["group"]
+    dest_rank = self.get_attr_dict()["dest_rank"]
+    collective_gather_grad = Broadcast(dest_rank, group)
+    rank = get_rank(group)
+    dev_num = self.rank_size
+    split = P.Split(output_num=dev_num)
+    if hasattr(self, "instance_name") and self.instance_name:
+        instance_name = "grad" + self.instance_name
+        collective_gather_grad.set_prim_instance_name(instance_name)
+
+    def bprop(x, out, dout):
+        grad = collective_gather_grad((dout,))
+        dx = split(grad[0])[rank]
+        return (dx,)
+
+    return bprop
+
+
+@bprop_getters.register(CollectiveScatter)
+def get_bprop_collective_scatter(self):
+    """Generate bprop for CollectiveScatter"""
+    group = self.get_attr_dict()["group"]
+    dest_rank = self.get_attr_dict()["src_rank"]
+    rank = get_rank(group)
+    collective_scatter_grad = CollectiveGather(dest_rank, group)
+    if hasattr(self, "instance_name") and self.instance_name:
+        instance_name = "grad" + self.instance_name
+        collective_scatter_grad.set_prim_instance_name(instance_name)
+
+    def bprop(x, out, dout):
+        dx_out = collective_scatter_grad(dout)
+        if rank == dest_rank:
+            dx = dx_out
+        else:
+            dx = F.depend(F.zeros_like(x), dx_out)
+        return (dx,)
+
+    return bprop
+
+
 @bprop_getters.register(_AllSwap)
 def get_bprop_allswap(self):
     """Generate bprop for _AllSwap."""
     all_swap_grad = _AllSwap(self.group)
-    if self.instance_name:
+    if hasattr(self, "instance_name") and self.instance_name:
         instance_name = "grad" + self.instance_name
         all_swap_grad.set_prim_instance_name(instance_name)
 
@@ -401,7 +488,7 @@ def get_bprop_allswap(self):
 def get_bprop_host_reduce_scatter(self):
     """Generate bprop for _HostReduceScatter"""
     host_reduce_scatter_grad = _HostAllGather(self.group)
-    if self.instance_name:
+    if hasattr(self, "instance_name") and self.instance_name:
         instance_name = "grad" + self.instance_name
         host_reduce_scatter_grad.set_prim_instance_name(instance_name)
 
@@ -436,7 +523,7 @@ def get_bprop_neighborexchange(self):
 def get_bprop_all_to_all(self):
     """Generate bprop for AlltoAll."""
     all_to_all_grad = AlltoAll(self.split_count, self.concat_dim, self.split_dim, self.group)
-    if self.instance_name:
+    if hasattr(self, "instance_name") and self.instance_name:
         instance_name = "grad" + self.instance_name
         all_to_all_grad.set_prim_instance_name(instance_name)
 
@@ -474,16 +561,25 @@ def get_bprop_mirror_operator(self):
     group = self.get_attr_dict()['group']
     dev_num = self.get_attr_dict()['dev_num']
     mean_flag = self.get_attr_dict()['mean_flag']
+    param_name = " "
+    if 'mirror_user_id' in self.get_attr_dict():
+        param_name = self.get_attr_dict()['mirror_user_id']
+
     dev_num_r = 1.0
+    dump_local_norm = ms.get_auto_parallel_context("dump_local_norm")
     if dev_num > 1:
         dev_num_r = 1.0 / dev_num
         all_reduce = AllReduce(group=group)
         all_gather = AllGather(group=group)
         mul = P.Mul()
         cast = P.Cast()
+        ln_print = P.Print()
+        reduce_sum = P.ReduceSum(keep_dims=False)
+        square = P.Square()
 
         fusion = self.get_attr_dict()["fusion"]
         all_reduce.add_prim_attr("fusion", fusion)
+        parameter = " "
         if hasattr(self, 'parameter'):
             parameter = self.parameter
             all_reduce.add_prim_attr("parameter", parameter)
@@ -493,6 +589,9 @@ def get_bprop_mirror_operator(self):
             all_reduce.set_prim_instance_name(instance_name)
 
     def bprop(x, out, dout):
+        if dump_local_norm:
+            dout = F.depend(dout, ln_print("dump local norm: ", param_name, reduce_sum(square((dout)))))
+
         if dev_num == 1:
             return (dout,)
         if mean_flag: