mindspore 2.3.0__cp39-none-any.whl → 2.3.0rc2__cp39-none-any.whl

mindspore/train/serialization.py

@@ -1,4 +1,4 @@
-# Copyright 2020-2024 Huawei Technologies Co., Ltd
+# Copyright 2020-2021 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,7 +17,6 @@
 from __future__ import absolute_import
 from __future__ import division
 
-import binascii
 import copy
 import json
 import os
@@ -31,7 +30,6 @@ from io import BytesIO
 import math
 import sys
 import time
-import google
 import numpy as np
 
 from mindspore.train.checkpoint_pb2 import Checkpoint
@@ -54,7 +52,6 @@ from mindspore.common.parameter import Parameter, _offload_if_config
 from mindspore.common.tensor import Tensor
 from mindspore._c_expression import Tensor as Tensor_
 from mindspore.common._utils import is_shape_unknown
-from mindspore.common.file_system import FileSystem, _register_basic_file_system, _register_mindio_file_system
 from mindspore.communication.management import get_rank, get_group_size
 from mindspore.experimental import MapParameter
 from mindspore.ops import Cast
@@ -70,9 +67,6 @@ from mindspore.parallel.checkpoint_transform import sync_pipeline_shared_paramet
 from mindspore.train._utils import read_proto
 from mindspore._c_expression import load_mindir, _encrypt, _decrypt, _is_cipher_file, dynamic_obfuscate_mindir, \
     split_mindir, split_dynamic_mindir
-from mindspore.common.generator import Generator
-from mindspore.train._utils import get_parameter_redundancy, remove_param_redundancy
-from mindspore.parallel.parameter_broadcast import parameter_broadcast
 from ..ops.operations._opaque_predicate_registry import add_opaque_predicate, clean_funcs
 
 tensor_to_ms_type = {"Int8": mstype.int8, "UInt8": mstype.uint8, "Int16": mstype.int16, "UInt16": mstype.uint16,
@@ -102,19 +96,6 @@ INT_64_MAX = 9223372036854775807
 
 cpu_cast = Cast().set_device("CPU")
 
-_ckpt_fs = FileSystem()
-
-
-def init_ckpt_file_system(fs: FileSystem):
-    """Initialize checkpoint file system"""
-    if _register_mindio_file_system(fs):
-        return
-    _register_basic_file_system(fs)
-
-
-# Initialize checkpoint file system
-init_ckpt_file_system(_ckpt_fs)
-
 
 class ParamDictFuture:
     def __init__(self, executor, param_dict_future):
@@ -252,19 +233,18 @@ def _save_weight(checkpoint_dir, model_name, iteration, params):
         logger.warning(f"Checkpoint dir: '{checkpoint_dir}' is not existed.")
 
 
-def _exec_save(ckpt_file_name, data_list, enc_key=None, enc_mode="AES-GCM", map_param_inc=False, crc_check=False):
+def _exec_save(ckpt_file_name, data_list, enc_key=None, enc_mode="AES-GCM", map_param_inc=False):
     """Execute the process of saving checkpoint into file."""
     try:
         with _ckpt_mutex:
             if os.path.exists(ckpt_file_name):
                 os.chmod(ckpt_file_name, stat.S_IWUSR)
                 os.remove(ckpt_file_name)
-            with _ckpt_fs.create(ckpt_file_name, *_ckpt_fs.create_args) as f:
+            with open(ckpt_file_name, "ab") as f:
                 plain_data = None
                 if enc_key is not None:
                     plain_data = BytesIO()
 
-                crc_num = 0
                 for name, value in data_list.items():
                     if name == "random_op":
                         _write_random_seed(name, value, f)
@@ -279,16 +259,16 @@ def _exec_save(ckpt_file_name, data_list, enc_key=None, enc_mode="AES-GCM", map_
                         _offload_if_config(value[3])
                         continue
                     if value[1] == "str":
-                        crc_num = _write_parameter_data(name, value, f, enc_key, plain_data, crc_num, crc_check)
+                        _write_parameter_data(name, value, f, enc_key, plain_data)
                         continue
                     if isinstance(value[2], np.ndarray):
-                        crc_num = _write_parameter_data(name, value, f, enc_key, plain_data, crc_num, crc_check)
+                        _write_parameter_data(name, value, f, enc_key, plain_data)
                         continue
                     if isinstance(value[2], Tensor) and hasattr(value[2], "slice_num") and value[2].slice_num > 1:
                         _write_hugeparameter(name, value, f)
                         continue
 
-                    crc_num = _write_parameter_bytes_data(name, value, f, enc_key, plain_data, crc_num, crc_check)
+                    _write_parameter_bytes_data(name, value, f, enc_key, plain_data)
 
                 if enc_key is not None:
                     plain_data.seek(0)
@@ -298,10 +278,7 @@ def _exec_save(ckpt_file_name, data_list, enc_key=None, enc_mode="AES-GCM", map_
                         f.write(_encrypt(block_data, len(block_data), enc_key, len(enc_key), enc_mode))
                         block_data = plain_data.read(max_block_size)
 
-                if crc_check:
-                    f.write('crc_num'.encode() + crc_num.to_bytes(10, byteorder='big'))
-
-            os.chmod(ckpt_file_name, stat.S_IRUSR)
+                os.chmod(ckpt_file_name, stat.S_IRUSR)
 
     except BaseException as e:
         logger.critical("Failed to save the checkpoint file %s. Maybe don't have the permission to write files, "
@@ -321,7 +298,7 @@ def _write_random_seed(name, value, f):
     f.write(checkpoint_list.SerializeToString())
 
 
-def _write_parameter_data(name, value, f, enc_key, plain_data, crc_num=0, crc_check=False):
+def _write_parameter_data(name, value, f, enc_key, plain_data):
     """Write parameter data into protobuf file."""
     data_size = value[2].nbytes / 1024
     if data_size > SLICE_SIZE:
@@ -340,17 +317,12 @@ def _write_parameter_data(name, value, f, enc_key, plain_data, crc_num=0, crc_ch
         param_tensor.tensor_content = param_slice.tobytes()
 
         if enc_key is None:
-            output_data = checkpoint_list.SerializeToString()
-            if crc_check:
-                crc_num = binascii.crc32(output_data, crc_num)
-            f.write(output_data)
+            f.write(checkpoint_list.SerializeToString())
         else:
             plain_data.write(checkpoint_list.SerializeToString())
 
-    return crc_num
-
 
-def _write_parameter_bytes_data(name, value, f, enc_key, plain_data, crc_num=0, crc_check=False):
+def _write_parameter_bytes_data(name, value, f, enc_key, plain_data):
     """Write parameter bytes data into protobuf file."""
     bytes_value = value[2].get_bytes()
     chunk_size = 1024 * SLICE_SIZE
@@ -365,15 +337,10 @@ def _write_parameter_bytes_data(name, value, f, enc_key, plain_data, crc_num=0,
         param_tensor.tensor_content = bytes_value[i:i + chunk_size]
 
         if enc_key is None:
-            output_data = checkpoint_list.SerializeToString()
-            if crc_check:
-                crc_num = binascii.crc32(output_data, crc_num)
-            f.write(output_data)
+            f.write(checkpoint_list.SerializeToString())
         else:
             plain_data.write(checkpoint_list.SerializeToString())
 
-    return crc_num
-
 
 def _write_mapparameter(name, value, f, map_param_inc=False):
     """Write map parameter into protobuf file."""
@@ -434,14 +401,10 @@ def _check_save_obj_and_ckpt_file_name(save_obj, ckpt_file_name):
 
 
 def save_checkpoint(save_obj, ckpt_file_name, integrated_save=True,
-                    async_save=False, append_dict=None, enc_key=None, enc_mode="AES-GCM", choice_func=None,
-                    crc_check=False, **kwargs):
+                    async_save=False, append_dict=None, enc_key=None, enc_mode="AES-GCM", choice_func=None, **kwargs):
     r"""
     Save checkpoint to a specified file.
 
-    Note:
-        The `enc_mode` and `crc_check` parameters are mutually exclusive and cannot be configured simultaneously.
-
     Args:
         save_obj (Union[Cell, list, dict]): The object to be saved. The data type can be :class:`mindspore.nn.Cell`,
             list, or dict. If a list, it can be the returned value of `Cell.trainable_params()`, or a list of dict
@@ -463,8 +426,6 @@ def save_checkpoint(save_obj, ckpt_file_name, integrated_save=True,
                                  If returns ``True`` , the Parameter that matching the custom condition will be saved.
                                  If returns ``False`` , the Parameter that not matching the custom condition will not
                                  be saved. Default: ``None`` .
-        crc_check (bool) : Whether to perform crc32 calculation when saving checkpoint and save the calculation
-            result to the file. Default: ``False`` .
         kwargs (dict): Configuration options dictionary.
 
     Raises:
@@ -504,32 +465,24 @@ def save_checkpoint(save_obj, ckpt_file_name, integrated_save=True,
     append_dict = _check_append_dict(append_dict)
     enc_key = Validator.check_isinstance('enc_key', enc_key, (type(None), bytes))
     enc_mode = Validator.check_isinstance('enc_mode', enc_mode, str)
-    crc_check = Validator.check_isinstance('crc_check', crc_check, bool)
     map_param_inc = kwargs.get('incremental', False)
     logger.info("Execute the process of saving checkpoint files.")
-    global_step_num = kwargs.get('global_step_num', None)
 
     save_obj = _convert_save_obj_to_param_list(save_obj, integrated_save, append_dict, choice_func)
 
     if append_dict:
         append_info_list = []
         for k_name, value in append_dict.items():
-            if isinstance(value, Generator):
-                value = value.get_state()
-            elif not isinstance(value, str):
+            if not isinstance(value, str):
                 value = Tensor(value)
             append_info_list.append({"name": k_name, "data": value})
         save_obj.extend(append_info_list)
 
     data_list = OrderedDict()
-    data_list_np = OrderedDict()
     with _ckpt_mutex:
         for param in save_obj:
             if param["name"] == "random_op":
-                if os.getenv("AITURBO") == "1":
-                    data_list_np["random_op"] = param["data"]
-                else:
-                    data_list["random_op"] = param["data"]
+                data_list["random_op"] = param["data"]
                 continue
             key = param["name"]
             data_list[key] = []
@@ -545,39 +498,28 @@ def save_checkpoint(save_obj, ckpt_file_name, integrated_save=True,
                     _save_param_list_data(data_list, key, param)
 
             if isinstance(param["data"], str):
-                if os.getenv("AITURBO") == "1":
-                    data_list_np[key] = np.array(param["data"])
-                else:
-                    data_list[key].append([0])
-                    data_list[key].append('str')
-                    data = np.array(param["data"])
-                    data_list[key].append(data)
+                data_list[key].append([0])
+                data_list[key].append('str')
+                data = np.array(param["data"])
+                data_list[key].append(data)
             else:
                 if isinstance(param["data"], Parameter):
                     param["data"].init_data()
-                if os.getenv("AITURBO") == "1":
-                    data_list_np[key] = param["data"].asnumpy()
-                else:
-                    dims = []
-                    for dim in param['data'].shape:
-                        dims.append(dim)
-                    data_list[key].append(dims)
-                    tensor_type = str(param["data"].dtype)
-                    data_list[key].append(tensor_type)
-                    data = param["data"]
-                    data_list[key].append(data)
-
-    if os.getenv("AITURBO") == "1":
-        import aiturbo
-        ckpt_name = os.path.basename(ckpt_file_name)
-        aiturbo.save_ckpt(ckpt_name, global_step_num, data_list_np)
-    elif async_save:
+                dims = []
+                for dim in param['data'].shape:
+                    dims.append(dim)
+                data_list[key].append(dims)
+                tensor_type = str(param["data"].dtype)
+                data_list[key].append(tensor_type)
+                data = param["data"]
+                data_list[key].append(data)
+
+    if async_save:
         data_copy = copy.deepcopy(data_list)
-        thr = Thread(target=_exec_save, args=(ckpt_file_name, data_copy, enc_key, enc_mode, map_param_inc, crc_check),
-                     name="asyn_save_ckpt")
+        thr = Thread(target=_exec_save, args=(ckpt_file_name, data_copy, enc_key, enc_mode), name="asyn_save_ckpt")
         thr.start()
     else:
-        _exec_save(ckpt_file_name, data_list, enc_key, enc_mode, map_param_inc, crc_check)
+        _exec_save(ckpt_file_name, data_list, enc_key, enc_mode, map_param_inc)
 
     logger.info("Saving checkpoint process is finished.")
 
@@ -736,9 +678,9 @@ def _check_append_dict(append_dict):
         raise TypeError("For 'save_checkpoint', the argument 'append_dict' must be dict, but got "
                         "{}.".format(type(append_dict)))
     for key, value in append_dict.items():
-        if not isinstance(key, str) or not isinstance(value, (int, float, bool, str, Parameter, Tensor, Generator)):
+        if not isinstance(key, str) or not isinstance(value, (int, float, bool, str, Parameter, Tensor)):
             raise TypeError(f"For 'save_checkpoint', the type of dict 'append_info' must be key: string, "
-                            f"value: int, float, bool or Generator, but got key: {type(key)}, value: {type(value)}")
+                            f"value: int, float or bool, but got key: {type(key)}, value: {type(value)}")
     return append_dict
 
 
@@ -1069,76 +1011,12 @@ def obfuscate_model(obf_config, **kwargs):
     obf_net = nn.GraphCell(obf_graph)
     if obf_random_seed != 0:
         append_y_tensor = Tensor(np.ones((1, 1)).astype(np.int32))
-        model_inputs += [append_y_tensor]
+        model_inputs += [append_y_tensor,]
     export(obf_net, *model_inputs, file_name=saved_path, file_format="MINDIR", **kwargs)
 
 
-def _load_into_param_dict(ckpt_file_name, parameter_dict, specify_prefix, filter_prefix, choice_func, dec_key,
-                          dec_mode, crc_check):
-    """load parameter into parameter_dict"""
-    ckpt_file_name = _check_ckpt_file_name(ckpt_file_name)
-    checkpoint_list = _parse_ckpt_proto(ckpt_file_name, dec_key, dec_mode, crc_check)
-    try:
-        param_data_list = []
-        map_data_list = [[], [], []]
-        map_shape_list = [0, 0, 0]
-        if specify_prefix:
-            logger.warning("For load_checkpoint, this parameter `specity_prefix` will be deprecated, "
-                           "please use `choice_func` instead.")
-        if filter_prefix:
-            logger.warning("For load_checkpoint, this parameter `filter_prefix` will be deprecated, "
-                           "please use `choice_func` instead.")
-        for element_id, element in enumerate(checkpoint_list.value):
-            if element.tag == "random_op":
-                parameter_dict["random_op"] = element.tensor.tensor_content
-                continue
-            if not _whether_load_param(specify_prefix, filter_prefix, element.tag):
-                continue
-            if specify_prefix is None and filter_prefix is None and \
-                    choice_func is not None and not choice_func(element.tag):
-                continue
-            if element.tensor.ByteSize() == 0:
-                _load_map_parameter(checkpoint_list, element, element_id, map_data_list, map_shape_list,
-                                    parameter_dict)
-                if element.tag in parameter_dict:
-                    map_data_list = [[], [], []]
-                    map_shape_list = [0, 0, 0]
-                continue
-            data = element.tensor.tensor_content
-            data_type = element.tensor.tensor_type
-            np_type = tensor_to_np_type.get(data_type)
-            ms_type = tensor_to_ms_type[data_type]
-            if data_type == 'str':
-                str_length = int(len(data) / 4)
-                np_type = np_type + str(str_length)
-            param_data_list.append(data)
-            if (element_id == len(checkpoint_list.value) - 1) or \
-                    (element.tag != checkpoint_list.value[element_id + 1].tag):
-                new_data = b"".join(param_data_list)
-                param_data_list.clear()
-                dims = element.tensor.dims
-                if data_type == 'str':
-                    str_value = np.frombuffer(new_data, np_type)
-                    parameter_dict[element.tag] = str(str_value[0])
-                else:
-                    if dims == [0]:
-                        dims = []
-                    param_data = Tensor_.convert_bytes_to_tensor(new_data, tuple(dims), ms_type)
-                    parameter = Parameter(param_data, name=element.tag)
-                    parameter_dict[element.tag] = parameter
-                    _offload_if_config(parameter)
-
-        logger.info("Loading checkpoint files process is finished.")
-
-    except BaseException as e:
-        logger.critical("Failed to load the checkpoint file '%s'.", ckpt_file_name)
-        raise ValueError(e.__str__() + "\nFor 'load_checkpoint', "
-                                       "failed to load the checkpoint file {}.".format(ckpt_file_name)) from e
-
-
 def load_checkpoint(ckpt_file_name, net=None, strict_load=False, filter_prefix=None,
-                    dec_key=None, dec_mode="AES-GCM", specify_prefix=None, choice_func=None,
-                    crc_check=False):
+                    dec_key=None, dec_mode="AES-GCM", specify_prefix=None, choice_func=None):
     """
     Load checkpoint info from a specified file.
 
@@ -1169,7 +1047,6 @@ def load_checkpoint(ckpt_file_name, net=None, strict_load=False, filter_prefix=N
             and the return value is a bool. If returns ``True`` , the Parameter
             that matches the custom condition will be loaded. If returns ``False`` , the Parameter that
             matches the custom condition will be removed. Default: ``None`` .
-        crc_check (bool) : Whether to perform crc32 validation when loading checkpoint. Default: ``False`` .
 
     Returns:
         Dict, key is parameter name, value is a Parameter or string. When the `append_dict` parameter of
@@ -1214,29 +1091,70 @@ def load_checkpoint(ckpt_file_name, net=None, strict_load=False, filter_prefix=N
         - `Saving and Loading the Model - Saving and Loading the Model Weight
           <https://mindspore.cn/tutorials/en/master/beginner/save_load.html#saving-and-loading-the-model-weight>`_
     """
+    ckpt_file_name = _check_ckpt_file_name(ckpt_file_name)
     specify_prefix = _check_prefix(specify_prefix)
     filter_prefix = _check_prefix(filter_prefix)
     dec_key = Validator.check_isinstance('dec_key', dec_key, (type(None), bytes))
     dec_mode = Validator.check_isinstance('dec_mode', dec_mode, str)
-    crc_check = Validator.check_isinstance('crc_check', crc_check, bool)
     logger.info("Execute the process of loading checkpoint files.")
+    checkpoint_list = _parse_ckpt_proto(ckpt_file_name, dec_key, dec_mode)
 
     parameter_dict = {}
+    try:
+        param_data_list = []
+        map_data_list = [[], [], []]
+        map_shape_list = [0, 0, 0]
+        if specify_prefix:
+            logger.warning("For load_checkpoint, this parameter `specity_prefix` will be deprecated, "
+                           "please use `choice_func` instead.")
+        if filter_prefix:
+            logger.warning("For load_checkpoint, this parameter `filter_prefix` will be deprecated, "
+                           "please use `choice_func` instead.")
+        for element_id, element in enumerate(checkpoint_list.value):
+            if element.tag == "random_op":
+                parameter_dict["random_op"] = element.tensor.tensor_content
+                continue
+            if not _whether_load_param(specify_prefix, filter_prefix, element.tag):
+                continue
+            if specify_prefix is None and filter_prefix is None and \
+                    choice_func is not None and not choice_func(element.tag):
+                continue
+            if element.tensor.ByteSize() == 0:
+                _load_map_parameter(checkpoint_list, element, element_id, map_data_list, map_shape_list, parameter_dict)
+                if element.tag in parameter_dict:
+                    map_data_list = [[], [], []]
+                    map_shape_list = [0, 0, 0]
+                continue
+            data = element.tensor.tensor_content
+            data_type = element.tensor.tensor_type
+            np_type = tensor_to_np_type.get(data_type)
+            ms_type = tensor_to_ms_type[data_type]
+            if data_type == 'str':
+                str_length = int(len(data) / 4)
+                np_type = np_type + str(str_length)
+            param_data_list.append(data)
+            if (element_id == len(checkpoint_list.value) - 1) or \
+                    (element.tag != checkpoint_list.value[element_id + 1].tag):
+                new_data = b"".join(param_data_list)
+                param_data_list.clear()
+                dims = element.tensor.dims
+                if dims == [0] and data_type == 'str':
+                    str_value = np.frombuffer(new_data, np_type)
+                    parameter_dict[element.tag] = str(str_value[0])
+                else:
+                    if dims == [0]:
+                        dims = []
+                    param_data = Tensor_.convert_bytes_to_tensor(new_data, tuple(dims), ms_type)
+                    parameter = Parameter(param_data, name=element.tag)
+                    parameter_dict[element.tag] = parameter
+                    _offload_if_config(parameter)
 
-    if os.getenv("AITURBO") == "1":
-        rank_id = get_rank()
-        import aiturbo
-        ckpt_path = os.path.dirname(ckpt_file_name)
-        ckpt_name = os.path.basename(ckpt_file_name)
-        np_dict = aiturbo.load_ckpt(ckpt_path, ckpt_name, rank_id)
-        for key, value in np_dict.items():
-            if isinstance(value, str):
-                parameter_dict[key] = value
-            else:
-                parameter_dict[key] = Parameter(Tensor(value), name=key)
-    else:
-        _load_into_param_dict(ckpt_file_name, parameter_dict, specify_prefix, filter_prefix, choice_func, dec_key,
-                              dec_mode, crc_check)
+        logger.info("Loading checkpoint files process is finished.")
+
+    except BaseException as e:
+        logger.critical("Failed to load the checkpoint file '%s'.", ckpt_file_name)
+        raise ValueError(e.__str__() + "\nFor 'load_checkpoint', "
+                                       "failed to load the checkpoint file {}.".format(ckpt_file_name)) from e
 
     if not parameter_dict:
         raise ValueError(f"The loaded parameter dict is empty after filter or specify, please check whether "
@@ -1403,28 +1321,17 @@ def _check_prefix(prefix):
     return prefix
 
 
-def _parse_ckpt_proto(ckpt_file_name, dec_key, dec_mode, crc_check):
+def _parse_ckpt_proto(ckpt_file_name, dec_key, dec_mode):
     """Parse checkpoint protobuf."""
     checkpoint_list = Checkpoint()
     try:
         if dec_key is None:
-            with _ckpt_fs.open(ckpt_file_name, *_ckpt_fs.open_args) as f:
+            with open(ckpt_file_name, "rb") as f:
                 pb_content = f.read()
         else:
             pb_content = _decrypt(ckpt_file_name, dec_key, len(dec_key), dec_mode)
             if pb_content is None:
                 raise ValueError("For 'load_checkpoint', failed to decrypt the checkpoint file.")
-        if crc_check and pb_content[-17:-10] == b"crc_num":
-            logger.warning("For 'load_checkpoint', the ckpt file do not contain the crc code, please check the file.")
-        if pb_content[-17:-10] == b"crc_num":
-            crc_num_bytes = pb_content[-10:]
-            pb_content = pb_content[:-17]
-            if crc_check:
-                crc_num = int.from_bytes(crc_num_bytes, byteorder='big')
-                cal_crc_num = binascii.crc32(pb_content, 0)
-                if cal_crc_num != crc_num:
-                    raise ValueError("For 'load_checkpoint', the crc check is failed, "
-                                     "please check whether the ckpt file is damaged.")
         checkpoint_list.ParseFromString(pb_content)
     except BaseException as e:
         if _is_cipher_file(ckpt_file_name):
@@ -1457,33 +1364,13 @@ def _whether_load_param(specify_prefix, filter_prefix, param_name):
 
 def _init_parameter_data_in_parallel_mode(net, parameter_dict):
     """In parallel mode, only init the paraemters in ckpt."""
-    is_train_phase = net.phase.startswith('train')
     for _, param in net.parameters_and_names():
-        if param.name in parameter_dict and param.from_ckpt and not is_train_phase:
-            param.shape = tuple(parameter_dict[param.name].shape)
-            continue
         if param.name in parameter_dict and param.has_init:
             logger.warning("{} is not init while load ckpt.".format(param.name))
             new_tensor = param.init_data()
             param._update_tensor_data(new_tensor)
 
 
-def _check_load_param_into_net(net, parameter_dict):
-    """check load_param_into_net"""
-    if not isinstance(net, nn.Cell):
-        logger.critical("Failed to combine the net and the parameters.")
-        msg = ("For 'load_param_into_net', the argument 'net' should be a Cell, but got {}.".format(type(net)))
-        raise TypeError(msg)
-    if not isinstance(parameter_dict, dict):
-        logger.critical("Failed to combine the net and the parameters.")
-        msg = ("For 'load_param_into_net', the argument 'parameter_dict' should be a dict, "
-               "but got {}.".format(type(parameter_dict)))
-        raise TypeError(msg)
-    if "random_op" in parameter_dict.keys():
-        net._add_attr("random_op_snapshot", parameter_dict["random_op"])
-        parameter_dict.pop("random_op")
-
-
 def load_param_into_net(net, parameter_dict, strict_load=False):
     """
     Load parameters into network, return parameter list that are not loaded in the network.
@@ -1520,7 +1407,18 @@ def load_param_into_net(net, parameter_dict, strict_load=False):
         - `Saving and Loading the Model - Saving and Loading the Model Weight
           <https://mindspore.cn/tutorials/en/master/beginner/save_load.html#saving-and-loading-the-model-weight>`_
     """
-    _check_load_param_into_net(net, parameter_dict)
+    if not isinstance(net, nn.Cell):
+        logger.critical("Failed to combine the net and the parameters.")
+        msg = ("For 'load_param_into_net', the argument 'net' should be a Cell, but got {}.".format(type(net)))
+        raise TypeError(msg)
+    if not isinstance(parameter_dict, dict):
+        logger.critical("Failed to combine the net and the parameters.")
+        msg = ("For 'load_param_into_net', the argument 'parameter_dict' should be a dict, "
+               "but got {}.".format(type(parameter_dict)))
+        raise TypeError(msg)
+    if "random_op" in parameter_dict.keys():
+        net._add_attr("random_op_snapshot", parameter_dict["random_op"])
+        parameter_dict.pop("random_op")
     for key, value in parameter_dict.items():
         if not isinstance(key, str) or not isinstance(value, (Parameter, str, list)):
             logger.critical("Load parameters into net failed.")
@@ -1530,8 +1428,6 @@ def load_param_into_net(net, parameter_dict, strict_load=False):
 
     strict_load = Validator.check_bool(strict_load)
     logger.info("Execute the process of loading parameters into net.")
-    for _, param in net.parameters_and_names():
-        param.from_ckpt = True
     if not _is_in_auto_parallel_mode():
         net.init_parameters_data()
     else:
@@ -1546,7 +1442,7 @@ def load_param_into_net(net, parameter_dict, strict_load=False):
             # Add has attr protection when load server checkpoint file on worker.
             if not hasattr(parameter_dict[param.name], "data"):
                 continue
-            new_param = parameter_dict[param.name]
+            new_param = copy.deepcopy(parameter_dict[param.name])
             _update_param(param, new_param, strict_load)
             ckpt_not_load.remove(param.name)
         else:
@@ -1562,14 +1458,6 @@ def load_param_into_net(net, parameter_dict, strict_load=False):
                        "'parameter_dict', please check whether the network structure is consistent "
                        "when training and loading checkpoint.".format(len(param_not_load)))
         logger.warning("{} are not loaded.".format(param_not_load))
-    if os.getenv("AITURBO") == "1" and net.parameter_layout_dict is not None:
-        param_layout = net.parameter_layout_dict
-        param_redundancy = get_parameter_redundancy(param_layout)
-        remove_param_redundancy_dict = remove_param_redundancy(param_redundancy)
-        target_parameter_name_set = set(parameter_dict.keys())
-        for rank_id, param_name_set in remove_param_redundancy_dict:
-            if param_name_set == target_parameter_name_set:
-                parameter_broadcast(net, param_layout, rank_id)
     return param_not_load, ckpt_not_load
 
 
@@ -2321,45 +2209,6 @@ def _save_dataset_to_mindir(model, dataset):
             model.preprocessor.op[-1].offload = op['offload'] if 'offload' in op.keys() else False
 
 
-def check_checkpoint(ckpt_file_name):
-    """
-    Check whether the checkpoint is valid.
-
-    Args:
-        ckpt_file_name (str): Checkpoint file name.
-
-    Returns:
-        bool, whether the checkpoint is valid.
-
-    Examples:
-        >>> import mindspore as ms
-        >>> ckpt_file_name = "./checkpoint/LeNet5-1_32.ckpt"
-        >>> check_result = ms.check_checkpoint(ckpt_file_name)
-        >>> print(check_result)
-        True
-    """
-    if not ckpt_file_name.endswith('.ckpt'):
-        return False
-    checkpoint_list = Checkpoint()
-    with _ckpt_fs.open(ckpt_file_name, *_ckpt_fs.open_args) as f:
-        pb_content = f.read()
-        if pb_content[-17:-10] == b"crc_num":
-            crc_num_bytes = pb_content[-10:]
-            pb_content = pb_content[:-17]
-            crc_num = int.from_bytes(crc_num_bytes, byteorder='big')
-            cal_crc_num = binascii.crc32(pb_content, 0)
-            if cal_crc_num != crc_num:
-                logger.warning("For 'check_checkpoint', the ckpt crc check is failed.")
-                return False
-        try:
-            checkpoint_list.ParseFromString(pb_content)
-        except google.protobuf.message.DecodeError as e:
-            logger.warning("For 'check_checkpoint', the ckpt parse is failed.")
-            logger.warning(e)
-            return False
-        return True
-
-
 def parse_print(print_file_name):
     """
     Parse data file generated by :class:`mindspore.ops.Print`.

mindspore/train/summary/_summary_adapter.py

@@ -190,7 +190,7 @@ def _nptype_to_prototype(np_value):
     if proto is None:
         raise TypeError("Transform numpy type failed in Summary, expect numpy type is one of ['np.bool_', 'np.int8', "
                         "'np.int16', 'np.int32', 'np.int64', 'np.uint8', 'np.uint16', 'np.uint32', 'np.uint64', "
-                        "'np.float16', 'np.float_', 'np.float64'].")
+                        "'np.float16', 'np.float', 'np.float64'].")
 
     return proto
 

mindspore/version.py

@@ -1 +1 @@
-__version__ = '2.3.0'
+__version__ = '2.3.0rc2'

{mindspore-2.3.0.dist-info → mindspore-2.3.0rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mindspore
-Version: 2.3.0
+Version: 2.3.0rc2
 Summary: MindSpore is a new open source deep learning training/inference framework that could be used for mobile, edge and cloud scenarios.
 Home-page: https://www.mindspore.cn
 Download-URL: https://github.com/mindspore-ai/mindspore/tags
@@ -26,7 +26,7 @@ Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
-Requires-Dist: numpy (<2.0.0,>=1.20.0)
+Requires-Dist: numpy (>=1.17.0)
 Requires-Dist: protobuf (>=3.13.0)
 Requires-Dist: asttokens (>=2.0.4)
 Requires-Dist: pillow (>=6.2.0)