mindspore 2.4.1__cp311-cp311-manylinux1_x86_64.whl → 2.5.0__cp311-cp311-manylinux1_x86_64.whl

mindspore/runtime/thread_bind_core.py

@@ -0,0 +1,401 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""Executor manager interfaces."""
+import subprocess
+from dataclasses import dataclass
+from typing import Union
+import re
+from mindspore import log as logger
+from mindspore import context
+
+
+def execute_command(cmd_list):
+    try:
+        with subprocess.Popen(cmd_list, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+            out, _ = p.communicate(timeout=1000)
+        res = out.decode()
+        return res
+    except FileNotFoundError as e:
+        raise RuntimeError(f"Failed to execute command, because {e}.")
+
+
+def _validate_affinity_cpu_list(affinity_cpu_list):
+    """
+    Validate the user-configured affinity_cpu_list.
+
+    Args:
+        affinity_cpu_list (dict): Customized bind-core policy to be validated.
+
+    Returns:
+        None.
+    """
+    device_pattern = re.compile(r'^device\d+$')
+    range_pattern = re.compile(r'^\d+-\d+$')
+
+    for key, value in affinity_cpu_list.items():
+        if not isinstance(key, str):
+            raise ValueError(f"The key of affinity_cpu_list: {key} should be a string.")
+        if not device_pattern.match(key):
+            raise ValueError(f"The key of affinity_cpu_list: {key} should be in format 'deviceX'.")
+        if not isinstance(value, list):
+            raise ValueError(f"The value of affinity_cpu_list: {value} should be a list.")
+        for item in value:
+            if not isinstance(item, str):
+                raise ValueError(f"The value of affinity_cpu_list: {item} should be a string.")
+            if not range_pattern.match(item):
+                raise ValueError(f"The value of affinity_cpu_list: {item} should be in format 'cpuidX-cpuidY'.")
+
+
+def _get_cpu_available():
+    """
+    Get the CPU resources available on the environment.
+
+    Returns:
+        list: List of available CPUs on the environment.
+    """
+    available_cpu_str = execute_command(["cat", "/sys/fs/cgroup/cpuset/cpuset.cpus"]).strip().split(",")
+    available_cpus = list()
+    for range_str in available_cpu_str:
+        endpoints = range_str.split("-")
+        if len(endpoints) != 2:
+            raise RuntimeError("'cat /sys/fs/cgroup/cpuset/cpuset.cpus' command output error, please check!")
+        available_cpus += [cid for cid in range(int(endpoints[0]), int(endpoints[1]) + 1)]
+    return available_cpus
+
+
+@dataclass
+class DeviceInfo:
+    """
+    A class to represent information about an Ascend device.
+
+    Attributes:
+        _info_line (str): A raw string containing device information.
+        npu_id (int): The ID of the NPU.
+        chip_id (int): The ID of the chip.
+        chip_logic_id (Union[int, str]): The logical ID of the chip, which can be an integer or a string.
+        chip_name (str): The name of the chip.
+
+    Methods:
+        __post_init__(): Initializes the attributes based on input.
+    """
+    _info_line: str = ""
+    npu_id: int = 0
+    chip_id: int = 0
+    chip_logic_id: Union[int, str] = 0
+    chip_name: str = ""
+
+    def __post_init__(self):
+        self.npu_id, self.chip_id, self.chip_logic_id, self.chip_name = \
+            self._info_line.strip().split(None, 3)
+        self.npu_id = int(self.npu_id)
+        self.chip_id = int(self.chip_id)
+        if self.chip_logic_id.isnumeric():
+            self.chip_logic_id = int(self.chip_logic_id)
+
+
+def _get_device_map_info():
+    """
+    Get abbreviated information about all NPUs on the environment.
+
+    Returns:
+        dict: Mapping of NPU logical ID to its details.
+        set: Contains all available NPU logical ids on the environment.
+    """
+    device_map_info = {}
+    available_devices = set()
+    device_map = \
+        execute_command(["npu-smi", "info", "-m"]).strip().split("\n")[1:]
+    for line in device_map:
+        device_info = DeviceInfo(line.strip())
+        if isinstance(device_info.chip_logic_id, int):
+            device_map_info[device_info.chip_logic_id] = device_info
+            available_devices.add(device_info.chip_logic_id)
+    return device_map_info, available_devices
+
+
+def _get_pcie_info(device_map_info, available_devices, keyword="PCIeBusInfo"):
+    """
+    Get the PCIe number of the NPU device.
+
+    Args:
+        device_map_info (dict): A map of NPU logical ID to its details.
+        available_devices (set): All available NPU logical ids on the environment.
+
+    Returns:
+        dict: Mapping of NPU logical ID to its PCIe number.
+    """
+    device_pcie_map = {}
+    for device in available_devices:
+        device_info = device_map_info.get(device)
+        if not device_info:
+            raise RuntimeError("Can not get device info, binding cpu will skip.")
+        pcie_info = \
+            execute_command(["npu-smi", "info", "-t", "board", "-i", f"{device_info.npu_id}",
+                             "-c", f"{device_info.chip_id}"]).strip().split("\n")
+        for _ in pcie_info:
+            line = ''.join(_.split())
+            if line.startswith(keyword):
+                device_pcie_map[device] = line[len(keyword) + 1:]
+                break
+    return device_pcie_map
+
+
+def _get_numa_info(device_pcie_map, keyword="NUMAnode"):
+    """
+    Get NUNA node affinity for device based on PCIe.
+
+    Args:
+        device_pcie_map (dict): A map of NPU logical ID to its PCIe number.
+
+    Returns:
+        dict: Mapping of device ID to its affinity NUMA nodes.
+        dict: Mapping of NUMA node to its affinity device IDs.
+    """
+    device_to_numa_map = {}
+    numa_to_device_map = {}
+
+    for device, pcie_no in device_pcie_map.items():
+        numa_info = execute_command(["lspci", "-s", f"{pcie_no}", "-vvv"]).strip().split("\n")
+        for _ in numa_info:
+            line = ''.join(_.split())
+            if line.startswith(keyword):
+                numa_id = int(line[len(keyword) + 1:])
+                device_to_numa_map[device] = numa_id
+
+                devices = numa_to_device_map.get(numa_id, None)
+                if devices is None:
+                    numa_to_device_map[numa_id] = list()
+                numa_to_device_map[numa_id].append(device)
+                break
+    numa_to_device_map[-1] = list(device_pcie_map.keys())
+    return device_to_numa_map, numa_to_device_map
+
+
+def _get_cpu_info(numa_ids, available_cpus, keyword1="NUMAnode", keyword2="CPU(s)"):
+    """
+    Get information about the CPUs on the NUMA nodes on the environment.
+
+    Args:
+        numa_ids (list): A list of NUMA nodes need to get related CPU information.
+        available_cpus (list): A list of available CPUs on the environment.
+
+    Returns:
+        dict: Mapping of NUMA node to its affinity CPUs.
+    """
+    numa_to_cpu_map = dict()
+
+    cpu_info = execute_command(["lscpu"]).strip().split("\n")
+    for _ in cpu_info:
+        line = ''.join(_.split())
+        if line.startswith(keyword1):
+            pattern = re.escape(keyword1) + r'(\d+)' + re.escape(keyword2)
+            match = re.search(pattern, line)
+            if match:
+                numa_id = int(match.group(1))
+                split_info = line.split(":")
+                cpu_id_ranges = split_info[-1].split(",")
+                ranges = list()
+                for range_str in cpu_id_ranges:
+                    endpoints = range_str.split("-")
+                    if len(endpoints) != 2:
+                        raise RuntimeError("'lscpu' command output error, please check!")
+                    ranges += [cid for cid in range(int(endpoints[0]), int(endpoints[1]) + 1) if cid in available_cpus]
+                if numa_id not in numa_ids:
+                    numa_id = int(-1)
+                if numa_id not in numa_to_cpu_map:
+                    numa_to_cpu_map[numa_id] = list()
+                numa_to_cpu_map[numa_id].extend(ranges)
+    return numa_to_cpu_map
+
+
+def _auto_generate_policy(available_devices, available_cpus, affinity_flag, numa_to_cpu_map, device_to_numa_map):
+    """
+    Automatically generate bind-core policy based on CPU affinity.
+
+    Args:
+        available_devices (list): All available NPU logical ids on the environment.
+        available_cpus (list): A list of available CPUs on the environment.
+        affinity_flag (bool): Whether or not it satisfies generating CPU affinity bind-core policy based on the
+          resources on the environment.
+        numa_to_cpu_map (dict): A map of NUMA node to its affinity CPUs.
+        device_to_numa_map (dict): A map of device ID to its affinity NUMA nodes.
+
+    Returns:
+        dict: Mapping of device to its affinity CPUs.
+    """
+    device_to_cpu_map = {}
+    for device_id in available_devices:
+        device_to_cpu_map[device_id] = list()
+
+    available_cpu_num = len(available_cpus)
+    available_device_num = len(available_devices)
+    cpu_num_per_device = available_cpu_num // available_device_num
+    if cpu_num_per_device < 7:
+        raise RuntimeError(f"Cpu num available for each device is {cpu_num_per_device}, "
+                           "which is less than the minimum cpu num need. Will not enable bind core feature.")
+
+    if affinity_flag:
+        device_to_cpu_idx = {}
+        for numa_id in numa_to_cpu_map:
+            device_to_cpu_idx[numa_id] = 0
+        for device_id in available_devices:
+            numa_id = device_to_numa_map.get(device_id)
+            affinity_cpu_num = 0
+            # Prioritize the use of affinity cpu resources.
+            affinity_cpu_start_idx = device_to_cpu_idx[numa_id]
+            if len(numa_to_cpu_map[numa_id][affinity_cpu_start_idx:]) >= cpu_num_per_device:
+                affinity_cpu = numa_to_cpu_map[numa_id][
+                    affinity_cpu_start_idx:(affinity_cpu_start_idx + cpu_num_per_device)]
+            else:
+                affinity_cpu = numa_to_cpu_map[numa_id][affinity_cpu_start_idx:]
+            affinity_cpu_num = len(affinity_cpu)
+            device_to_cpu_map[device_id].extend(affinity_cpu)
+            device_to_cpu_idx[numa_id] = affinity_cpu_start_idx + affinity_cpu_num
+            # If the affinity cpu resources are insufficient then use resources from the non-affinity cpu pool.
+            if -1 in device_to_cpu_idx:
+                unaffinity_cpu_start_idx = device_to_cpu_idx[-1]
+                unaffinity_cpu_num = cpu_num_per_device - affinity_cpu_num
+                unaffinity_cpu = numa_to_cpu_map[-1][
+                    unaffinity_cpu_start_idx:(unaffinity_cpu_start_idx + unaffinity_cpu_num)]
+                device_to_cpu_map[device_id].extend(unaffinity_cpu)
+                device_to_cpu_idx[-1] = unaffinity_cpu_start_idx + unaffinity_cpu_num
+    else:
+        device_rank = 0
+        for device_id in available_devices:
+            cpu_start = device_rank * cpu_num_per_device
+            device_to_cpu_map[device_id] = available_cpus[cpu_start:(cpu_start + cpu_num_per_device)]
+            device_rank += 1
+    return device_to_cpu_map
+
+
+def _customize_generate_policy(affinity_cpu_list, available_cpus):
+    """
+    Generate customized bind-core policy based on user-configured inputs.
+
+    Args:
+        affinity_cpu_list (dict): User-configured inputs to generate customized bind-core policy.
+        available_cpus (list): A list of available CPUs on the environment.
+
+    Returns:
+        dict: Mapping of device to its affinity CPUs.
+    """
+    device_to_cpu_map = {}
+    _validate_affinity_cpu_list(affinity_cpu_list)
+    for device, cpu_id_ranges in affinity_cpu_list.items():
+        ranges = list()
+        for range_str in cpu_id_ranges:
+            endpoints = range_str.split("-")
+            for cid in range(int(endpoints[0]), int(endpoints[1]) + 1):
+                if cid not in available_cpus:
+                    raise RuntimeError(f"CPU id: {cid} set in affinity_cpu_list is not available.")
+                ranges.append(cid)
+        if len(ranges) < 7:
+            raise RuntimeError(f"cpu num available for {device} is less than 7, which is the minimum cpu num need.")
+        device_id = int(device.replace("device", ""))
+        device_to_cpu_map[device_id] = ranges
+    return device_to_cpu_map
+
+
+def _assign_cpu_to_module(device_to_cpu_map):
+    """
+    Assign specific CPUs to modules.
+
+    Args:
+        device_to_cpu_map (dict): A map of device to its affinity CPUs.
+
+    Returns:
+        dict: Mapping of device to its affinity CPUs based on module segmentation.
+    """
+    module_bind_core_policy = {}
+    for device, cpu_list in device_to_cpu_map.items():
+        thread_to_cpu_map = {}
+        thread_to_cpu_map["main"] = [cpu_list[0]]
+        thread_to_cpu_map["runtime"] = cpu_list[1:6]
+        thread_to_cpu_map["pynative"] = cpu_list[1:5]
+        thread_to_cpu_map["minddata"] = cpu_list[6:]
+        module_bind_core_policy[device] = thread_to_cpu_map
+    return module_bind_core_policy
+
+
+def _get_cpu_affinity_policy(affinity_cpu_list=None):
+    """
+    The entry to get bind-core policy.
+
+    Args:
+        affinity_cpu_list (dict, optional): User-configured inputs to generate customized bind-core policy.
+          Default: ``None``.
+
+    Returns:
+        dict: Mapping of device to its affinity CPUs based on module segmentation.
+        bool: Whether the generated bind-core policy is based on cpu affinity.
+    """
+    device_target = context.get_context("device_target")
+    device_pcie_map = {}
+    device_to_numa_map = {}
+    numa_to_device_map = {}
+    numa_to_cpu_map = {}
+    affinity_flag = False
+    bind_policy_flag = False
+
+    # Get the CPU resources in the environment. If this fails, the binding core feature will not be enabled.
+    try:
+        available_cpus = _get_cpu_available()
+    except RuntimeError as e:
+        logger.warning(f"Failed to acquire available cpu info, error: {e} Will not enable bind core feature.")
+        return {}, False
+    # Automatic generation of binding core policy based on resources on the environment.
+    if (affinity_cpu_list is None) or (not affinity_cpu_list):
+        # If the device target is Ascend, the affinity between the device and NUMA node is taken into account
+        # to generate the binding core policy.
+        if device_target == "Ascend":
+            # Get the hardware resources in the environment. If this fails, will bind core not based on device.
+            try:
+                device_map_info, available_devices = _get_device_map_info()
+            except RuntimeError as e:
+                logger.warning(f"Failed to acquire device to numa affinity info, error: {e} "
+                               "Will not bind core based on affinity. Module bind core policy "
+                               f"generated: {available_cpus}.")
+                return available_cpus, bind_policy_flag
+            # Get the affinity resources in the environment. If this fails, will bind core not based on affinity.
+            try:
+                device_pcie_map = _get_pcie_info(device_map_info, available_devices)
+                device_to_numa_map, numa_to_device_map = _get_numa_info(device_pcie_map)
+                numa_to_cpu_map = _get_cpu_info(list(numa_to_device_map.keys()), available_cpus)
+            except RuntimeError as e:
+                logger.warning(f"Failed to acquire device to numa affinity info, error: {e} "
+                               "Will not bind core based on affinity.")
+                affinity_flag = False
+            if device_pcie_map and device_to_numa_map and numa_to_device_map and numa_to_cpu_map:
+                affinity_flag = True
+            # Auto-generation of bind core policy for Ascned.
+            try:
+                device_to_cpu_map = _auto_generate_policy(available_devices, available_cpus, affinity_flag,
+                                                          numa_to_cpu_map, device_to_numa_map)
+            except (RuntimeError, ZeroDivisionError) as e:
+                logger.warning(f"Failed to auto generate bind core policy, error: {e}. "
+                               "Will not enable bind core feature.")
+                return {}, False
+            module_bind_core_policy = _assign_cpu_to_module(device_to_cpu_map)
+            bind_policy_flag = True
+        else:
+            module_bind_core_policy = available_cpus
+    # User configured binding core policy.
+    else:
+        device_to_cpu_map = _customize_generate_policy(affinity_cpu_list, available_cpus)
+        module_bind_core_policy = _assign_cpu_to_module(device_to_cpu_map)
+        bind_policy_flag = True
+    logger.warning(f"Module bind core policy generated: {module_bind_core_policy}.")
+    return module_bind_core_policy, bind_policy_flag

mindspore/train/__init__.py

@@ -27,7 +27,7 @@ from mindspore.train.loss_scale_manager import LossScaleManager, FixedLossScaleM
 from mindspore.train.serialization import save_checkpoint, load_checkpoint, load_param_into_net, export, \
     load, parse_print, build_searched_strategy, merge_sliced_parameter, load_distributed_checkpoint, \
     async_ckpt_thread_status, restore_group_info_list, convert_model, obfuscate_model, export_split_mindir, \
-    load_checkpoint_async, check_checkpoint, get_ckpt_path_with_strategy
+    load_checkpoint_async, check_checkpoint, get_ckpt_path_with_strategy, ckpt_to_safetensors, safetensors_to_ckpt
 from mindspore.train.callback import Callback, LossMonitor, TimeMonitor, ModelCheckpoint, SummaryCollector, \
     CheckpointConfig, RunContext, LearningRateScheduler, SummaryLandscape, FlopsUtilizationCollector, \
     History, LambdaCallback, ReduceLROnPlateau, EarlyStopping, OnRequestExit, BackupAndRestore, TFTRegister
@@ -41,7 +41,7 @@ __all__ = ["Model", "DatasetHelper", "connect_network_with_dataset", "build_trai
            "load_param_into_net", "export", "load", "export_split_mindir", "parse_print", "build_searched_strategy",
            "merge_sliced_parameter", "load_distributed_checkpoint", "async_ckpt_thread_status",
            "restore_group_info_list", "convert_model", "data_sink", "obfuscate_model", "load_checkpoint_async",
-           "get_ckpt_path_with_strategy"]
+           "get_ckpt_path_with_strategy", "ckpt_to_safetensors", "safetensors_to_ckpt"]
 __all__.extend(callback.__all__)
 __all__.extend(summary.__all__)
 __all__.extend(train_thor.__all__)

mindspore/train/_utils.py

@@ -16,15 +16,15 @@
 from __future__ import absolute_import
 
 import os
-import threading
-from datetime import datetime
 import json
 from collections.abc import Iterable
 
+import time
 import numpy as np
 
 from mindspore.common.tensor import Tensor
 from mindspore._c_expression import Tensor as Tensor_
+from mindspore._c_expression import MSContext, ms_ctx_param
 from mindspore.common.dtype import dtype_to_nptype, pytype_to_dtype
 from mindspore.common import dtype as mstype
 from mindspore import context
@@ -64,6 +64,10 @@ def _get_types_and_shapes(dataset):
     dataset_shapes = dataset.output_shapes()
     return dataset_types, dataset_shapes
 
+def enable_data_broadcast():
+    """Get status to indicate if enable dataset broadcast."""
+    return MSContext.get_instance().get_param(ms_ctx_param.dataset_broadcast_opt_level) > 0
+
 
 def _exec_datagraph(exec_dataset, dataset_size, phase='dataset', create_data_info_queue=False):
     """Initialize and execute the dataset graph."""
@@ -77,15 +81,12 @@ def _exec_datagraph(exec_dataset, dataset_size, phase='dataset', create_data_inf
     if queue_name is None:
         queue_name = str("")
 
+    # Don't enable dynamic shape(multi-subgraph) feature in pp/data_broadcast mode,
+    # otherwise get_data_info will stuck since some rank do not consume data.
     use_pipeline_parallel = (context.get_auto_parallel_context("pipeline_stages") > 1)
+    data_broadcast = enable_data_broadcast()
 
-    # temp env to disable dynamic feature of sink size 1
-    dynamic_sink1_env = os.getenv("MS_DEV_DYNAMIC_SINK1", None)
-    dynamic_sink1 = True
-    if dynamic_sink1_env and dynamic_sink1_env.strip() in ['False', 'false']:
-        dynamic_sink1 = False
-
-    if use_pipeline_parallel or not dynamic_sink1:
+    if use_pipeline_parallel or data_broadcast:
         create_data_info_queue = False
 
     exec_dataset = exec_dataset.device_que(send_epoch_end=send_epoch_end,
@@ -514,12 +515,46 @@ def parse_hccl_file(hccl_file_path):
     return rankid_dict
 
 
-def vlog_print(level, module, file, line, message):
-    '''Read environment variable VLOG_v and print to log'''
-    if os.environ.get("VLOG_v") == level:
-        now = datetime.now()
-        formatted_time = now.strftime("%Y-%m-%d-%H:%M:%S.%f")[:-3] + f".{now.microsecond // 1000}"
-        path = 'mindspore' + file.split("mindspore")[-1]
-        pid = os.getpid()
-        thread_id = threading.get_ident()
-        print(f"[V{level}] {module}({pid},{thread_id},python):{formatted_time} [{path}:{line}] {message}", flush=True)
+def _progress_bar(iterable, total=None):
+    """
+    Decorate an iterable object, returning an iterator which acts exactly
+    like the original iterable, but prints a dynamically updating
+    progressbar every time a value is requested.
+    """
+    if total is None:
+        total = len(iterable)
+
+    start_time = time.time()
+
+    def print_progress_bar(iteration):
+        percent = f"{100 * (iteration / float(total)):.1f}"
+        bar_length = 40
+        filled_length = int(bar_length * iteration // total)
+        bar = '█' * filled_length + '-' * (bar_length - filled_length)
+
+        elapsed_time = time.time() - start_time
+        estimated_total_time = elapsed_time / iteration * total
+        remaining_time = estimated_total_time - elapsed_time
+
+        elapsed_time_str = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
+        remaining_time_str = time.strftime("%H:%M:%S", time.gmtime(remaining_time))
+
+        print(f'\r{percent}%|{bar}|[{elapsed_time_str}<{remaining_time_str}]', end='')
+        if iteration == total:
+            print()
+
+    for i, item in enumerate(iterable, start=1):
+        yield item
+        print_progress_bar(i)
+
+
+def _load_and_transform(path, name_map, load_func, transform_func):
+    if load_func is not None:
+        param_dict = load_func(path)
+    else:
+        param_dict = path
+    transform_dict = {}
+    for k, v in param_dict.items():
+        new_name = name_map.get(k, k) if name_map is not None else k
+        transform_dict[new_name] = transform_func(v, new_name)
+    return transform_dict

mindspore/train/amp.py

@@ -101,6 +101,7 @@ AMP_AUTO_BLACK_LIST = [
     P.LayerNorm,
     gen.LayerNormExt,
     P.BatchNorm,
+    gen.BatchNormExt,
     gen.GroupNorm,
     P.KLDivLoss,
     P.SmoothL1Loss,
@@ -112,6 +113,7 @@ AMP_AUTO_BLACK_LIST = [
     P.Pdist,
     P.Cdist,
     P.Renorm,
+    gen.MSELossExt,
 ]
 
 # Indicates which inputs of primitives need to be converted
@@ -428,15 +430,15 @@ def auto_mixed_precision(network, amp_level="O0", dtype=mstype.float16):
 
     ``Pow``, ``ACos``, ``Asin``, ``Cosh``, ``Erfinv``, ``Exp``, ``Expm1``, ``Log``, ``Log1p``, ``Reciprocal``,
     ``Rsqrt``, ``Sinh``, ``Tan``, ``Softplus``, ``SoftplusExt``, ``LayerNorm``, ``LayerNormExt``, ``BatchNorm``,
-    ``GroupNorm``, ``KLDivLoss``, ``SmoothL1Loss``, ``MultilabelMarginLoss``, ``SoftMarginLoss``,
+    ``BatchNormExt``, ``GroupNorm``, ``KLDivLoss``, ``SmoothL1Loss``, ``MultilabelMarginLoss``, ``SoftMarginLoss``,
     ``TripletMarginLoss``, ``MultiMarginLoss``, ``BCEWithLogitsLoss``, ``Pdist``, ``Cdist``, ``Renorm``,
     ``ReduceProd``, ``Softmax``, ``LogSoftmax``, ``CumProd``, ``CumSum``, ``CumsumExt``, ``ProdExt``, ``SumExt``,
-    ``Norm``
+    ``Norm``, ``MSELossExt``
 
     Operators in `promote_list` are:
 
     ``Addcdiv``, ``Addcmul``, ``Cross``, ``_PyboostCrossPrim``, ``Dot``, ``GridSampler2D``, ``GridSampler3D``,
-    ``BiasAdd``
+    ``BiasAdd``, ``AddN``, ``Concat``
 
     For details on automatic mixed precision, refer to
     `Automatic Mix Precision <https://www.mindspore.cn/tutorials/en/master/beginner/mixed_precision.html>`_ .
@@ -837,12 +839,14 @@ def custom_mixed_precision(network, *, white_list=None, black_list=None, dtype=m
         - Repeatedly calling mixed-precision interfaces, such as `custom_mixed_precision` and `auto_mixed_precision`,
           can result in a larger network hierarchy and slower performance.
         - If interfaces like `Model` and `build_train_network` is used to train the network which is converted by
-          mixed-precision interfaces such as `custom_mixed_precision` and `auto_mixed_precision`, `amp_level`
+          mixed-precision interfaces such as `custom_mixed_precision` and `auto_mixed_precision`, `amp_level` or `level`
           need to be configured to ``O0`` to avoid the duplicated accuracy conversion.
         - Primitives for blacklist is not support yet.
 
     Args:
         network (Cell): Definition of the network.
+
+    Keyword Args:
         white_list (list[Primitive, Cell], optional): White list of custom mixed precision. Defaults: ``None`` , means
             white list is not used.
         black_list (list[Cell], optional): Black list of custom mixed precision. Defaults: ``None`` , means