mindspore 2.4.0__cp311-cp311-manylinux1_x86_64.whl → 2.4.10__cp311-cp311-manylinux1_x86_64.whl

mindspore/train/_utils.py

@@ -16,6 +16,8 @@
 from __future__ import absolute_import
 
 import os
+import threading
+from datetime import datetime
 import json
 from collections.abc import Iterable
 
@@ -76,7 +78,14 @@ def _exec_datagraph(exec_dataset, dataset_size, phase='dataset', create_data_inf
         queue_name = str("")
 
     use_pipeline_parallel = (context.get_auto_parallel_context("pipeline_stages") > 1)
-    if use_pipeline_parallel:
+
+    # temp env to disable dynamic feature of sink size 1
+    dynamic_sink1_env = os.getenv("MS_DEV_DYNAMIC_SINK1", None)
+    dynamic_sink1 = True
+    if dynamic_sink1_env and dynamic_sink1_env.strip() in ['False', 'false']:
+        dynamic_sink1 = False
+
+    if use_pipeline_parallel or not dynamic_sink1:
         create_data_info_queue = False
 
     exec_dataset = exec_dataset.device_que(send_epoch_end=send_epoch_end,
@@ -303,10 +312,68 @@ def parse_strategy_ckpt(file_name):
 
         for ele in param.parallel_layouts.tensor_map[0].ListFields()[0][1]:
             tensor_map.append(ele)
-        layout_dict[param.param_name] = [dev_matrix, tensor_map]
+        layout_dict[param.param_name] = [dev_matrix, tensor_map, param.parallel_layouts.opt_weight_shard_step,
+                                         param.parallel_layouts.opt_weight_shard_size]
     return layout_dict
 
 
+def _get_strategy_opt_shard(param_redundancy_dict, parameter_layout_opt_shard):
+    """Strategy ckpt append opt shard."""
+    for key, value in parameter_layout_opt_shard.items():
+        if value[1] not in (-1, 0):
+            opt_para_num = value[1]
+            param_redundancy_ranks = param_redundancy_dict.get(key)
+            res = []
+            for param_ranks in param_redundancy_ranks:
+                if len(param_ranks) % opt_para_num == 0:
+                    for i in range(0, opt_para_num):
+                        res.append(param_ranks[i::opt_para_num])
+            param_redundancy_dict[key] = tuple(res)
+
+
+def _get_layout_opt_shard(layout_obj, param_redundancy_dict):
+    """Layout ckpt append opt shard."""
+    for key, value in layout_obj.items():
+        if value[5]:
+            world_groups = ("hccl_world_group", "nccl_world_group", "mccl_world_group")
+            if value[5] in world_groups:
+                opt_para_num = get_group_size()
+            elif "-" in value[5]:
+                opt_para_str = value[5].split("-")[0]
+                opt_para_num = int(opt_para_str)
+            else:
+                raise ValueError(f"For get_parameter_redundancy, the format of the parallel communication domain for "
+                                 f"the optimizer is incorrect.")
+            param_redundancy_ranks = param_redundancy_dict.get(key)
+            res = []
+            for param_ranks in param_redundancy_ranks:
+                if len(param_ranks) % opt_para_num == 0:
+                    for i in range(0, opt_para_num):
+                        res.append(param_ranks[i::opt_para_num])
+            param_redundancy_dict[key] = tuple(res)
+
+
+def _get_parameter_redundancy_without_opt_shard(parameter_layout, param_redundancy_dict, initial_rank):
+    """Get parameter redundancy without opt shard."""
+    for key, (slices, deploy_loc, *_) in parameter_layout.items():
+        redundancy_matrix = np.zeros(shape=slices + [len(slices)], dtype=np.int8)
+        for i in deploy_loc:
+            internal_slice = tuple(slice(None) for _ in range(i))
+            for j in range(slices[-i - 1]):
+                if i == -1:
+                    continue
+                else:
+                    redundancy_matrix[(..., j) + internal_slice + (i,)] = j
+        locate_list = redundancy_matrix.reshape((-1, len(slices))).tolist()
+        redundancy_dict = {}
+        for index, locate in enumerate(locate_list):
+            redundancy_dict.setdefault(tuple(locate), []).append(index + initial_rank)
+        redundancy_list = []
+        for _, indices in sorted(redundancy_dict.items()):
+            redundancy_list.append(tuple(indices))
+        param_redundancy_dict[key] = tuple(redundancy_list)
+
+
 def get_parameter_redundancy(layout_obj, initial_rank=0):
     """
     Get parameter redundancy map.
@@ -327,7 +394,12 @@ def get_parameter_redundancy(layout_obj, initial_rank=0):
          'param4': ((0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15))}
     """
     if isinstance(layout_obj, str):
-        parameter_layout = parse_strategy_ckpt(layout_obj)
+        parameter_layout_total = parse_strategy_ckpt(layout_obj)
+        parameter_layout = {}
+        parameter_layout_opt_shard = {}
+        for key, value in parameter_layout_total.items():
+            parameter_layout[key] = value[0:2]
+            parameter_layout_opt_shard[key] = value[2:]
     elif isinstance(layout_obj, Cell):
         from mindspore.communication.management import get_process_group_ranks
         groups_ranks = (tuple(get_process_group_ranks()),)
@@ -339,37 +411,14 @@ def get_parameter_redundancy(layout_obj, initial_rank=0):
             parameter_layout[k] = v[:2]
 
     param_redundancy_dict = {}
-    for key, (slices, deploy_loc, *_) in parameter_layout.items():
-        redundancy_matrix = np.zeros(shape=slices + [len(slices)], dtype=np.int8)
-        for i in deploy_loc:
-            internal_slice = tuple(slice(None) for _ in range(i))
-            for j in range(slices[-i - 1]):
-                if i == -1:
-                    continue
-                else:
-                    redundancy_matrix[(..., j) + internal_slice + (i,)] = j
-        locate_list = redundancy_matrix.reshape((-1, len(slices))).tolist()
-        redundancy_dict = {}
-        for index, locate in enumerate(locate_list):
-            redundancy_dict.setdefault(tuple(locate), []).append(index + initial_rank)
-        redundancy_list = []
-        for _, indices in sorted(redundancy_dict.items()):
-            redundancy_list.append(tuple(indices))
-        param_redundancy_dict[key] = tuple(redundancy_list)
+
+    _get_parameter_redundancy_without_opt_shard(parameter_layout, param_redundancy_dict, initial_rank)
+
     if isinstance(layout_obj, str):
-        return param_redundancy_dict
+        _get_strategy_opt_shard(param_redundancy_dict, parameter_layout_opt_shard)
+    else:
+        _get_layout_opt_shard(layout_obj, param_redundancy_dict)
 
-    for key, value in layout_obj.items():
-        if value[5]:
-            world_groups = ("hccl_world_group", "nccl_world_group", "mccl_world_group")
-            opt_para_num = int(value[5][0]) if value[5] not in world_groups else get_group_size()
-            param_redundancy_ranks = param_redundancy_dict.get(key)
-            res = []
-            for param_ranks in param_redundancy_ranks:
-                if len(param_ranks) % opt_para_num == 0:
-                    for i in range(0, opt_para_num):
-                        res.append(param_ranks[i::opt_para_num])
-            param_redundancy_dict[key] = tuple(res)
     return param_redundancy_dict
 
 
@@ -463,3 +512,14 @@ def parse_hccl_file(hccl_file_path):
             rankid_dict[int(device["rank_id"])] = device["device_ip"]
 
     return rankid_dict
+
+
+def vlog_print(level, module, file, line, message):
+    '''Read environment variable VLOG_v and print to log'''
+    if os.environ.get("VLOG_v") == level:
+        now = datetime.now()
+        formatted_time = now.strftime("%Y-%m-%d-%H:%M:%S.%f")[:-3] + f".{now.microsecond // 1000}"
+        path = 'mindspore' + file.split("mindspore")[-1]
+        pid = os.getpid()
+        thread_id = threading.get_ident()
+        print(f"[V{level}] {module}({pid},{thread_id},python):{formatted_time} [{path}:{line}] {message}", flush=True)

mindspore/train/callback/_checkpoint.py

@@ -44,6 +44,15 @@ SAVE_DIR = _cur_dir
 _info_list = ["epoch_num", "step_num"]
 
 
+def _wait_async_save_ckpt(async_save=False):
+    """Waiting for asynchronous saving of ckpt to complete."""
+    if async_save:
+        thread_list = threading.enumerate()
+        for thread in thread_list:
+            if thread.getName() == "asyn_save_ckpt":
+                thread.join()
+
+
 def _get_dp_tp_from_redundancy(redundancy_tuple):
     """From redundancy get dp and tp"""
     dp = []
@@ -568,6 +577,7 @@ class ModelCheckpoint(Callback):
                                  "string that does not contain '/', but got {}.".format(self._prefix))
         if self._directory_func:
             self._directory = self._directory_func(cb_params)
+            _make_directory(self._directory)
         collect_host_info("Callback", "ModelCheckpoint", "step_end", start_time=get_clock_syscnt(), level=1)
         # In disaster recovery scenario, the training process may be rolled back to the last step where
         # the ckpt was successfully saved, so the _last_triggered_step should be updated.
@@ -575,7 +585,6 @@ class ModelCheckpoint(Callback):
             self._last_triggered_step = cb_params.last_save_ckpt_step
             cb_params.last_save_ckpt_step = None
 
-        _make_directory(self._directory)
         # save graph (only once)
         if not self._graph_saved:
             graph_file_name = os.path.join(self._directory, self._prefix + '-graph.meta')
@@ -583,10 +592,6 @@ class ModelCheckpoint(Callback):
                 os.remove(graph_file_name)
             _save_graph(cb_params.train_network, graph_file_name)
             self._graph_saved = True
-        thread_list = threading.enumerate()
-        for thread in thread_list:
-            if thread.getName() == "asyn_save_ckpt":
-                thread.join()
         self._save_ckpt(cb_params)
 
     def end(self, run_context):
@@ -602,10 +607,7 @@ class ModelCheckpoint(Callback):
 
         self._save_ckpt(cb_params, _to_save_last_ckpt)
 
-        thread_list = threading.enumerate()
-        for thread in thread_list:
-            if thread.getName() == "asyn_save_ckpt":
-                thread.join()
+        _wait_async_save_ckpt(self._config.async_save)
 
         destroy_allgather_cell()
 
@@ -643,6 +645,7 @@ class ModelCheckpoint(Callback):
         step_num_in_epoch = int((cb_params.cur_step_num - 1) % cb_params.batch_num + 1)
 
         if save_ckpt:
+            _wait_async_save_ckpt(self._config.async_save)
             if self._prefix_func:
                 cur_ckpoint_file = self._prefix + f".{self._config.format}"
             else:

mindspore/train/callback/_on_request_exit.py

@@ -240,7 +240,18 @@ class OnRequestExit(Callback):
                 if param.name == "graceful_exit" and param.asnumpy() == True:  # pylint: disable=C0121
                     logger.warning("Graceful exit is triggered, stop training.")
                     if self.save_ckpt:
-                        save_checkpoint(net, self.train_name, integrated_save=self.integrated_save)
+                        append_dict = {"epoch_num": call_params.cur_epoch_num,
+                                       "step_num": call_params.cur_step_num,
+                                       "batch_num": call_params.batch_num}
+                        if call_params.loss_scale_mananger is not None:
+                            append_dict["loss_scale"] = call_params.loss_scale_mananger.get_loss_scale()
+                        if call_params.optimizer is not None:
+                            global_step = int(call_params.optimizer.global_step.data)
+                        else:
+                            global_step = int(call_params.network.optimizer.global_step.data)
+                        append_dict["global_step"] = global_step
+                        save_checkpoint(net, self.train_name, integrated_save=self.integrated_save,
+                                        append_dict=append_dict)
                     if self.save_mindir:
                         inputs = call_params.train_dataset_element
                         export(net, *inputs, file_name=self.train_name, file_format='MINDIR')

mindspore/train/callback/_tft_register.py

@@ -21,6 +21,7 @@ from mindspore import _checkparam as Validator
 from mindspore.train.callback._callback import Callback
 from mindspore import context
 from mindspore.common.parameter import Parameter
+from mindspore.common.tensor import Tensor
 from mindspore.communication import get_rank, get_group_size
 from mindspore import log as logger
 from mindspore.train.serialization import _get_cur_rank_dp
@@ -29,6 +30,9 @@ from mindspore._c_expression import clean_tdt_channel
 from mindspore._c_expression import send_recv
 from mindspore._c_expression import CollectiveManager
 from mindspore._c_expression import _get_uce_process_strategy, _get_uce_mem_info
+from mindspore._c_expression import Tensor as Tensor_
+import mindspore
+import mindspore.common.dtype as mstype
 
 def _get_ckpt_dir(step, ckpt_save_path, is_tmp_file):
     """ Common func to generate ckpt dir name."""
@@ -39,6 +43,9 @@ def _get_ckpt_dir(step, ckpt_save_path, is_tmp_file):
 def _save_checkpoint_on_failure(step, save_info, args, cb_ctx):
     """ Callback used for TFT save ckpt function when errors occur."""
     logger.info("Enter _save_checkpoint_on_failure function")
+    if not cb_ctx._is_params_consistent():    # pylint: disable=W0212
+        raise RuntimeError("Can't save parameters, because they are left in inconsistent state!")
+
     ckpt_save_path = cb_ctx.ckpt_save_path
     cb_params = args
     cur_rank = get_rank()
@@ -83,8 +90,6 @@ def _tft_exit_cb(ctx):
     _tft_sem_post()
     os._exit(1)   # pylint: disable=W0212
 
-
-
 def _tft_repair_callback(step, need_rebuild, error_ranks, repair_info, args, cb_ctx):
     """ Callback used for TFT repair function."""
     logger.info("Enter _tft_repair_callback repair type: {}".format(repair_info["repair_type"]))
@@ -104,7 +109,7 @@ or repair_info["repair_type"] == cb_ctx.tft.RepairType.RT_UCE_LOWLEVEL.value):
     logger.info("Finish _tft_repair_callback")
 
 
-def _tft_clean_callback(is_uce_error, ctx):
+def _tft_clean_callback(is_uce_error, args, ctx):
     """ Callback used for TFT clean function."""
     logger.info("Enter _tft_clean_callback")
     ret = 0
@@ -125,10 +130,13 @@ def _tft_clean_callback(is_uce_error, ctx):
     return ret
 
 
-def _tft_stop_callback(cb_ctx):
+def _tft_stop_callback(args, cb_ctx):
     """ Callback used for TFT stop function."""
     logger.info("Enter _tft_stop_callback device_id: {}".format(cb_ctx.device_id))
     _stop_device(cb_ctx.device_id)
+    if (not cb_ctx.is_uce_rank) and (not cb_ctx._is_params_consistent()):    # pylint: disable=W0212
+        raise RuntimeError("Can't stop device, because training parameters are left in inconsistent state!")
+    cb_ctx.is_uce_rank = False
     logger.info("Finish _tft_stop_callback")
 
 
@@ -254,15 +262,29 @@ class TFTRegister(Callback):
         # let it raise errors if not install mindio_tft package
         from mindio_ttp import framework_ttp as tft
         self.tft = tft
+        self.is_uce_rank = False
         self.global_step = 0
         Validator.check_non_negative_int(ctrl_port)
         self.has_init_replica = False
         self._controller_ip = ctrl_ip
         self._controller_rank_id = ctrl_rank_id
         self._controller_port = ctrl_port
+        self.cb_params = None
         self.device_id = context.get_context("device_id")
         self._init_tft()
         self.ckpt_save_path = ckpt_save_path
+        self.assign = mindspore.ops.Assign()
+        self.g_one = Parameter(Tensor([1], dtype=mstype.int32))
+        self.s1 = mindspore.hal.Stream()
+
+    def _is_params_consistent(self):
+        for key, param in self.cb_params.train_network.parameters_and_names():
+            if "tft_g_one_flag" in key:
+                with mindspore.hal.StreamCtx(self.s1):
+                    tft_g_one_flag = Tensor(Tensor_.move_to(param, "CPU", False))
+                self.s1.synchronize()
+                return int(tft_g_one_flag) == 1
+        return False
 
     def _set_tft_optimizer_replica(self, run_context):
         """ set Mindio TFT optimizer replica info, used internal. """
@@ -280,7 +302,7 @@ class TFTRegister(Callback):
         replica_info = [
             {
                 "type": 1,
-                "rank_list": dp,
+                "rank_list": list(dp),
                 "replica_cnt": len(dp),
                 "replica_shift": 0
             }
@@ -301,13 +323,12 @@ class TFTRegister(Callback):
         cur_rank = get_rank()
         enable_local_copy = False
         enable_arf = False
-        enable_zit = False
         enable_tls = False
         tls_key_dir = ""
 
         if cur_rank == self._controller_rank_id:
             logger.info(f"Begin to start tft controller on rank_id:{cur_rank}")
-            self.tft.tft_init_controller(cur_rank, world_size, enable_local_copy, enable_arf, enable_zit)
+            self.tft.tft_init_controller(cur_rank, world_size, enable_local_copy, enable_arf)
             self.tft.tft_start_controller(self._controller_ip, self._controller_port, enable_tls, tls_key_dir)
             logger.info("Finish start tft controller.")
 
@@ -328,12 +349,14 @@ class TFTRegister(Callback):
             self.has_init_replica = True
             self._set_tft_optimizer_replica(run_context)
         cb_params = run_context.original_args()
+        logger.info("START Set optimizer finish step status to TFT. step: {}".format(cb_params.cur_step_num))
+        self.tft.tft_end_updating_os(cb_params.cur_step_num)
         if cb_params.optimizer is not None:
             self.global_step = int(cb_params.optimizer.global_step.data)
+            self.assign(cb_params.optimizer.tft_g_one_flag, self.g_one)
         else:
             self.global_step = int(cb_params.network.optimizer.global_step.data)
-        logger.info("START Set optimizer finish step status to TFT. step: {}".format(cb_params.cur_step_num))
-        self.tft.tft_end_updating_os(cb_params.cur_step_num)
+            self.assign(cb_params.network.optimizer.tft_g_one_flag, self.g_one)
         logger.info("END Set optimizer finish step status to TFT.")
 
 
@@ -344,6 +367,7 @@ class TFTRegister(Callback):
             raise ValueError("TFT feature doesn't support sink_size > 1.")
         logger.info("Set set args to TFT.")
         self.tft.tft_set_step_args(cb_params)
+        self.cb_params = cb_params
 
     def end(self, run_context):
         cur_rank = get_rank()

mindspore/train/dataset_helper.py

@@ -15,6 +15,7 @@
 """Dataset help for minddata dataset"""
 from __future__ import absolute_import
 
+import os
 import math
 import copy
 
@@ -264,7 +265,14 @@ def connect_network_with_dataset(network, dataset_helper):
     queue_name = dataset.__transfer_dataset__.queue_name
     # In pipeline parallel, some stages have no GetNext, should not get in.
     use_pipeline_parallel = (context.get_auto_parallel_context("pipeline_stages") > 1)
-    if _dynamic_sink_scenario(dataset, dataset_iter, is_dynamic) and not use_pipeline_parallel:
+
+    # temp env to disable dynamic feature of sink size 1
+    dynamic_sink1_env = os.getenv("MS_DEV_DYNAMIC_SINK1", None)
+    dynamic_sink1 = True
+    if dynamic_sink1_env and dynamic_sink1_env.strip() in ['False', 'false']:
+        dynamic_sink1 = False
+
+    if _dynamic_sink_scenario(dataset, dataset_iter, is_dynamic) and not use_pipeline_parallel and dynamic_sink1:
         dataset_types, dataset_shapes = dataset_helper.get_data_info()
         # Need to do full_batch for shapes which also do in the _DatasetIterMSLoopSink
         if _need_to_full():
@@ -306,7 +314,7 @@ def connect_network_with_dataset(network, dataset_helper):
             aux.__shape_type__ = str(dataset_types) + str(dataset_shapes)
 
     if _dynamic_sink_data(dataset, dataset_iter) and _dynamic_sink_exception_scenario(dataset_iter, is_dynamic) and \
-        not use_pipeline_parallel:
+        not use_pipeline_parallel and dynamic_sink1:
         dataset_helper.get_data_info()
     network.add_flags(sink_mode=True)
     return network

mindspore/train/model.py

@@ -46,6 +46,7 @@ from mindspore.parallel._utils import _get_parallel_mode, _get_device_num, _get_
 from mindspore.parallel._ps_context import _is_role_worker, _is_role_pserver, _is_ps_mode, \
     _cache_enable, _enable_distributed_mindrt
 from mindspore.train.metrics import Loss
+from mindspore.train._utils import vlog_print
 from mindspore import nn
 from mindspore.boost import AutoBoost
 from mindspore.context import ParallelMode
@@ -148,6 +149,7 @@ def _handle_tft(func):
                     e_str = str(e)
                     logger.info("uce wrapper caught RuntimeError e_str:{}".format(e_str))
                     if "UCEError" in e_str:
+                        obj.is_uce_rank = True
                         logger.info("uce wrapper report UCEError")
                         tft.tft_report_error(tft.ReportState.RS_UCE.value)
                     elif "ForceStopError" in e_str:
@@ -654,10 +656,12 @@ class Model:
             dataset.__loop_size__ = 1
 
         if dataset_helper is None:
+            vlog_print("1", "ME", __file__, sys._getframe().f_lineno, "Begin to create DatasetHelper.")
             logger.info("Begin to create DatasetHelper.")
             dataset_helper = DatasetHelper(dataset, dataset_sink_mode, sink_size, epoch_num)
 
         if dataset_sink_mode:
+            vlog_print("1", "ME", __file__, sys._getframe().f_lineno, "Begin to connect network with dataset.")
             logger.info("Begin to connect network with dataset.")
             network = connect_network_with_dataset(network, dataset_helper)
 
@@ -779,6 +783,7 @@ class Model:
         if not train_dataset and not valid_dataset:
             raise ValueError("The argument 'train_dataset' and 'valid_dataset' can not both be None or empty.")
 
+        vlog_print("1", "ME", __file__, sys._getframe().f_lineno, "Begin to check device number in model.build().")
         logger.info("Begin to check device number in model.build() procedure.")
         _device_number_check(self._parallel_mode, self._device_number)
 
@@ -787,17 +792,21 @@ class Model:
                 raise TypeError("The type of 'train_dataset' must be `Dataset`, "
                                 "but got {}.".format(type(train_dataset)))
 
+            vlog_print("1", "ME", __file__, sys._getframe().f_lineno,
+                       "Begin to check parameter broadcast in model.build().")
             logger.info("Begin to check parameter broadcast in model.build() procedure.")
             _parameter_broadcast_check(self._parallel_mode, self._parameter_broadcast)
             if self._parameter_broadcast:
                 self._train_network.set_broadcast_flag()
 
+            vlog_print("1", "ME", __file__, sys._getframe().f_lineno, "Begin to exec preprocess in model.build().")
             logger.info("Begin to exec preprocess in model.build() procedure.")
             train_dataset.__no_send__ = True
             train_dataset_helper, train_network = self._exec_preprocess(is_train=True,
                                                                         dataset=train_dataset,
                                                                         dataset_sink_mode=True,
                                                                         sink_size=sink_size)
+            vlog_print("1", "ME", __file__, sys._getframe().f_lineno, "Begin to warmup dataset in model.build().")
             logger.info("Begin to warmup dataset in model.build() procedure.")
             self._warmup_dataset(epoch, train_dataset, sink_size)
 
@@ -805,13 +814,19 @@ class Model:
             delattr(train_dataset, "__no_send__")
 
             # Waiting for the dataset warmup ready
+            vlog_print("1", "ME", __file__, sys._getframe().f_lineno,
+                       "Begin waiting for dataset warmup in model.build().")
             logger.info("Begin waiting for dataset warmup in model.build() procedure.")
             self._waiting_for_dataset_warmup_ready(train_dataset)
+            vlog_print("1", "ME", __file__, sys._getframe().f_lineno,
+                       "The dataset warmup was successful in model.build().")
             logger.info("The dataset warmup was successful in model.build() procedure.")
 
             if context.get_auto_parallel_context("pipeline_stages") > 1 and valid_dataset:
                 train_network.add_flags_recursive(is_first_iteration=True)
             for inputs in train_dataset_helper:
+                vlog_print("1", "ME", __file__, sys._getframe().f_lineno,
+                           "Begin to compile train network in model.build().")
                 logger.info("Begin to compile train network in model.build() procedure.")
                 train_network.compile(*inputs)
                 self._train_network.parameter_layout_dict = train_network.parameter_layout_dict
@@ -832,6 +847,8 @@ class Model:
             if context.get_auto_parallel_context("pipeline_stages") > 1:
                 eval_network.add_flags_recursive(is_first_iteration=False)
             for inputs in valid_dataset_helper:
+                vlog_print("1", "ME", __file__, sys._getframe().f_lineno,
+                           "Begin to compile eval network in model.build().")
                 logger.info("Begin to compile eval network in model.build() procedure.")
                 eval_network.compile(*inputs)
                 break
@@ -905,6 +922,7 @@ class Model:
             epoch = 1
         cb_params.last_save_ckpt_step = None
         cb_params.latest_ckpt_file = None
+        cb_params.loss_scale_mananger = self._loss_scale_manager
 
         # build callback list
         with _CallbackManager(callbacks) as list_callback:
@@ -1567,8 +1585,11 @@ class Model:
         if hasattr(self._train_network, '_is_check_and_refresh') and not self._train_network._is_check_and_refresh:
             self._train_network.check_names_and_refresh_name()
             self._train_network._is_check_and_refresh = True
+        vlog_print("1", "ME", __file__, sys._getframe().f_lineno, "Begin to init dataset in model.build().")
         logger.info("Begin to init dataset in model.build() procedure.")
         self._init(train_dataset, valid_dataset, sink_size, epoch)
+        vlog_print("1", "ME", __file__, sys._getframe().f_lineno,
+                   "The model.build() which contains dataset warmup and network compile is success.")
         logger.info("The model.build() which contains dataset warmup and network compile is success.")
 
     def _eval_in_fit(self, valid_dataset, callbacks=None, dataset_sink_mode=True, cb_params=None):

mindspore/train/serialization.py

@@ -64,7 +64,7 @@ from mindspore.parallel._cell_wrapper import get_allgather_cell, _single_paramet
 from mindspore.parallel._tensor import _load_tensor, _get_tensor_strategy, _get_tensor_slice_index
 from mindspore.parallel._tensor import _reshape_param_data, _reshape_param_data_with_weight
 from mindspore.parallel._utils import _infer_rank_list, _remove_repeated_slices, _is_in_auto_parallel_mode, \
-    _get_device_num, _is_parallel_mode
+    _get_device_num
 from mindspore.parallel._auto_parallel_context import _get_auto_parallel_context
 from mindspore.parallel._parallel_serialization import _convert_to_list, _convert_to_layout, _build_searched_strategy, \
     _restore_group_info_list, _get_param_list_when_first_dim_sharded
@@ -81,6 +81,7 @@ from safetensors.numpy import save_file
 from safetensors import safe_open
 from ..ops.operations._opaque_predicate_registry import add_opaque_predicate, clean_funcs
 
+
 tensor_to_ms_type = {"Int8": mstype.int8, "UInt8": mstype.uint8, "Int16": mstype.int16, "UInt16": mstype.uint16,
                      "Int32": mstype.int32, "UInt32": mstype.uint32, "Int64": mstype.int64, "UInt64": mstype.uint64,
                      "Float16": mstype.float16, "Float32": mstype.float32, "Float64": mstype.float64,
@@ -328,6 +329,8 @@ def _exec_save(ckpt_file_name, data_list, enc_key=None, enc_mode="AES-GCM", map_
             file_name_list = list(os.path.splitext(ckpt_file_name))
             file_name_list[1] = file_name_list[1].replace(f".{format}", ".tmp")
             tmp_name = ''.join(file_name_list)
+            if _ckpt_fs.backend == "mindio":
+                tmp_name = ckpt_file_name
             if os.path.exists(ckpt_file_name):
                 os.chmod(ckpt_file_name, stat.S_IWUSR)
                 os.remove(ckpt_file_name)
@@ -383,7 +386,7 @@ def _exec_save(ckpt_file_name, data_list, enc_key=None, enc_mode="AES-GCM", map_
             if not os.path.exists(tmp_name):
                 logger.warning(f"Rename failed, can't find {tmp_name}, it is possible that multiple processes have "
                                f"simultaneously modified a file.")
-            else:
+            elif _ckpt_fs.backend != "mindio":
                 os.rename(tmp_name, ckpt_file_name)
             os.chmod(ckpt_file_name, stat.S_IRUSR)
     except BaseException as e:
@@ -1569,6 +1572,9 @@ def _parse_ckpt_proto(ckpt_file_name, dec_key, dec_mode, crc_check):
                     raise ValueError("For 'load_checkpoint', the crc check is failed, "
                                      "please check whether the ckpt file is damaged.")
         checkpoint_list.ParseFromString(pb_content)
+    except google.protobuf.message.DecodeError as e:
+        raise ValueError(f"Failed to read the checkpoint file {ckpt_file_name}. "
+                         f"The file may be corrupted, and the content cannot be parsed.") from e
     except BaseException as e:
         if _is_cipher_file(ckpt_file_name):
             err_info = "Failed to read the checkpoint file {}. The file may be encrypted or tempered with, " \
@@ -1598,19 +1604,6 @@ def _whether_load_param(specify_prefix, filter_prefix, param_name):
     return whether_load
 
 
-def _init_parameter_data_in_parallel_mode(net, parameter_dict):
-    """In parallel mode, only init the paraemters in ckpt."""
-    is_train_phase = net.phase.startswith('train')
-    for _, param in net.parameters_and_names():
-        if param.name in parameter_dict and param.from_ckpt and not is_train_phase:
-            param.shape = tuple(parameter_dict[param.name].shape)
-            continue
-        if param.name in parameter_dict and param.has_init:
-            logger.warning("{} is not init while load ckpt.".format(param.name))
-            new_tensor = param.init_data()
-            param._update_tensor_data(new_tensor)
-
-
 def _check_load_param_into_net(net, parameter_dict):
     """check load_param_into_net"""
     if not isinstance(net, nn.Cell):
@@ -1682,10 +1675,6 @@ def load_param_into_net(net, parameter_dict, strict_load=False, remove_redundanc
     logger.info("Execute the process of loading parameters into net.")
     for _, param in net.parameters_and_names():
         param.from_ckpt = True
-    if not (_is_in_auto_parallel_mode() or _is_parallel_mode()):
-        net.init_parameters_data()
-    else:
-        _init_parameter_data_in_parallel_mode(net, parameter_dict)
     param_not_load = []
     ckpt_not_load = list(parameter_dict.keys())
     for _, param in net.parameters_and_names():
@@ -1698,6 +1687,8 @@ def load_param_into_net(net, parameter_dict, strict_load=False, remove_redundanc
                 continue
             new_param = parameter_dict[param.name]
             _update_param(param, new_param, strict_load)
+            if hasattr(param, "init_param") and not param.init_param:
+                param.init_param = True
             ckpt_not_load.remove(param.name)
         else:
             param_not_load.append(param.name)
@@ -1822,6 +1813,8 @@ def _load_dismatch_prefix_params(net, parameter_dict, param_not_load, strict_loa
                 if param.name in param_not_load and new_param_name in parameter_dict:
                     new_param = parameter_dict[new_param_name]
                     _update_param(param, new_param, strict_load)
+                    if hasattr(param, "init_param") and not param.init_param:
+                        param.init_param = True
                     param_not_load.remove(param.name)
 
 

mindspore/version.py

@@ -1 +1 @@
-__version__ = '2.4.0'
+__version__ = '2.4.10'

{mindspore-2.4.0.dist-info → mindspore-2.4.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mindspore
-Version: 2.4.0
+Version: 2.4.10
 Summary: MindSpore is a new open source deep learning training/inference framework that could be used for mobile, edge and cloud scenarios.
 Home-page: https://www.mindspore.cn
 Download-URL: https://github.com/mindspore-ai/mindspore/tags
@@ -118,7 +118,7 @@ MindSpore offers build options across multiple backends:
 
 | Hardware Platform | Operating System | Status |
 | :---------------- | :--------------- | :----- |
-| Ascend910 | Ubuntu-x86 | ✔️ |
+| Ascend | Ubuntu-x86 | ✔️ |
 |  | Ubuntu-aarch64 | ✔️ |
 |  | EulerOS-aarch64 | ✔️ |
 |  | CentOS-x86 | ✔️ |
@@ -316,15 +316,17 @@ Project stable branches will be in one of the following states:
 | Development | 3 months      | Features are under development.                  |
 | Maintained  | 6 - 12 months | All bugfixes are appropriate. Releases produced. |
 | Unmaintained| 0 - 3 months  | All bugfixes are appropriate. No Maintainers and No Releases produced.                                                 |
-| End Of Life (EOL) |  N/A |  Branch no longer accepting changes.    |
+| End Of Life (EOL) |  N/A |  Version no longer accepting changes.    |
 
 ## Maintenance status
 
-| **Branch** | **Status**   | **Initial Release Date** | **Next Phase**                         | **EOL Date**|
+| **Version** | **Status**   | **Initial Release Date** | **Next Phase**                         | **EOL Date**|
 |------------|--------------|--------------------------|----------------------------------------|-------------|
-| **r2.2**   | Maintained   | 2023-10-18               | Unmaintained <br> 2024-10-18 estimated |             |
-| **r2.1**   | Maintained   | 2023-07-29               | Unmaintained <br> 2024-07-29 estimated |             |
-| **r2.0**   | Maintained   | 2023-06-15               | Unmaintained <br> 2024-06-15 estimated |             |
+| **r2.4**   | Maintained   | 2024-10-30               | Unmaintained <br> 2025-10-30 estimated | 2025-10-30  |
+| **r2.3**   | Maintained   | 2024-07-15               | Unmaintained <br> 2025-07-15 estimated | 2025-07-15  |
+| **r2.2**   | End Of Life  | 2023-10-18               |                                        | 2024-10-18  |
+| **r2.1**   | End Of Life  | 2023-07-29               |                                        | 2024-07-29  |
+| **r2.0**   | End Of Life  | 2023-06-15               |                                        | 2024-06-15  |
 | **r1.10**  | End Of Life  | 2023-02-02               |                                        | 2024-02-02  |
 | **r1.9**   | End Of Life  | 2022-10-26               |                                        | 2023-10-26  |
 | **r1.8**   | End Of Life  | 2022-07-29               |                                        | 2023-07-29  |