mindspore 2.3.0__cp39-none-any.whl → 2.3.0rc2__cp39-none-any.whl

mindspore/train/model.py

@@ -18,7 +18,6 @@ from __future__ import absolute_import
 from collections.abc import Iterable
 from functools import wraps
 
-import sys
 import os
 import math
 import copy
@@ -35,10 +34,8 @@ from mindspore.common.tensor import Tensor
 from mindspore.train.metrics import get_metrics, get_metric_fn
 from mindspore._checkparam import check_input_data, check_output_data
 from mindspore import _checkparam as Validator
-from mindspore.train.callback import _InternalCallbackParam, RunContext, _CallbackManager, Callback, TimeMonitor,\
-    FlopsUtilizationCollector, MindIOTTPAdapter
+from mindspore.train.callback import _InternalCallbackParam, RunContext, _CallbackManager, Callback, TimeMonitor
 from mindspore.train.callback import __all__ as internal_cb_names
-from mindspore.train.callback._cluster_monitor import ClusterMonitor
 from mindspore import context
 from mindspore.parallel._utils import _get_parallel_mode, _get_device_num, _get_parameter_broadcast, \
     _device_number_check, _parameter_broadcast_check, _parallel_predict_check, \
@@ -51,7 +48,7 @@ from mindspore.boost import AutoBoost
 from mindspore.context import ParallelMode
 from mindspore.parallel._recovery_context import _set_recovery_context, _get_recovery_context
 from mindspore.train.dataset_helper import DatasetHelper, connect_network_with_dataset
-from mindspore.common.api import _pynative_executor, ARG_SPECIFIED, TOTAL_ARG_LEN
+from mindspore.common.api import _pynative_executor
 from mindspore.dataset.core.config import get_debug_mode
 from mindspore.dataset.engine.datasets import _set_training_dataset, _reset_training_dataset
 from mindspore.train import amp
@@ -78,7 +75,6 @@ class _FrameworkProfilerCallback(Callback):
     """
     Profiler callback of framework for training.
     """
-
     def step_begin(self, run_context):
         _framework_profiler_step_start()
 
@@ -120,141 +116,6 @@ def _save_final_ckpt(func):
     return wrapper
 
 
-def _append_ccae(callbacks):
-    """Add cluster monitoring when CCAE is enabled."""
-    perf_config = os.getenv("PERF_DUMP_CONFIG")
-    if perf_config is None:
-        return callbacks
-    pairs = perf_config.split(',')
-    perf_config_dict = {}
-    for pair in pairs:
-        key, value = pair.split(':')
-        if value.lower() == 'true':
-            perf_config_dict[key] = True
-        elif value.lower() == 'false':
-            perf_config_dict[key] = False
-        elif value.isdigit():
-            perf_config_dict[key] = int(value)
-        else:
-            perf_config_dict[key] = value
-    if perf_config_dict.get("enable", False):
-        if callbacks is None:
-            callbacks = ClusterMonitor()
-        elif isinstance(callbacks, list):
-            callbacks.append(ClusterMonitor())
-        else:
-            callbacks = [callbacks, ClusterMonitor()]
-    return callbacks
-
-
-def _get_arg_infos(inputs):
-    """Get compile argument information from inputs.
-
-    Args:
-        inputs (Union[list, tuple, dict]): Argument got from cell which is set by `set_inputs`.
-
-    Raises:
-        RuntimeError: inputs is not a list, tuple or dict.
-        RuntimeError: inputs is a dict without necessary keys and values.
-
-    Returns:
-        _type_: _description_
-    """
-    if isinstance(inputs, (list, tuple)):
-        arg_specified = [[idx, arg] for idx, arg in enumerate(inputs)]
-        arg_len = len(inputs)
-    elif isinstance(inputs, dict):
-        arg_specified = inputs.get(ARG_SPECIFIED, None)
-        arg_len = inputs.get(TOTAL_ARG_LEN, None)
-        if arg_specified is None or arg_len is None:
-            raise RuntimeError(
-                "The incremental inputs should be processed(with \"%s\" and \"%s\"), but got %s." %
-                (ARG_SPECIFIED, TOTAL_ARG_LEN, str(inputs)))
-    else:
-        raise RuntimeError("inputs should be a list/tuple or a dict, but got %s!" % str(inputs))
-
-    return arg_len, arg_specified
-
-
-def _merge_inputs(inputs1, inputs2):
-    """Merge two processed inputs to a new inputs for latter setting cell's inputs."""
-    is_fullmode1 = isinstance(inputs1, (list, tuple))
-    is_fullmode2 = isinstance(inputs2, (list, tuple))
-
-    if is_fullmode1 and is_fullmode2:
-        return [*inputs1, *inputs2]
-
-    arg_len1, arg_specified1 = _get_arg_infos(inputs1)
-    arg_len2, arg_specified2 = _get_arg_infos(inputs2)
-
-    res_arg_len = arg_len1 + arg_len2
-    res_arg_specified = []
-    res_arg_specified.extend(arg_specified1)
-    # The second inputs should add offset before merging.
-    for idx, arg in arg_specified2:
-        res_arg_specified.append([idx + arg_len1, arg])
-
-    return {ARG_SPECIFIED: res_arg_specified, TOTAL_ARG_LEN: res_arg_len}
-
-
-def _process_loss_inputs(loss_inputs):
-    """Process loss's inputs whose first input should be dropped for train or eval.
-
-    Args:
-        loss_inputs (Union[list, tuple, dict]): Arguments save by `set_inputs` or `jit`.
-
-    Raises:
-        RuntimeError: inputs is not a list, tuple or dict.
-        RuntimeError: inputs is a dict without necessary keys and values.
-
-    Returns:
-        list, tuple or dict: Arguments for latter setting.
-    """
-    # For train or eval, the first input of loss is the inner-tensor, so drop it.
-    res = None
-    if isinstance(loss_inputs, (list, tuple)):
-        res = [*loss_inputs]
-        res.pop(0)
-    elif isinstance(loss_inputs, dict):
-        loss_arg_specified = loss_inputs.get(ARG_SPECIFIED, None)
-        loss_arg_len = loss_inputs.get(TOTAL_ARG_LEN, None)
-        if loss_arg_specified is None or loss_arg_len is None:
-            raise RuntimeError(
-                "The loss incremental inputs should be processed(with \"%s\" and \"%s\"), but got %s." %
-                (ARG_SPECIFIED, TOTAL_ARG_LEN, str(loss_inputs)))
-        res_loss_arg_specified = []
-        for idx, arg in loss_arg_specified:
-            if idx == 0:
-                continue
-            res_loss_arg_specified.append([idx, arg])
-        res = {ARG_SPECIFIED: res_loss_arg_specified, TOTAL_ARG_LEN: loss_arg_len - 1}
-    else:
-        raise RuntimeError("loss_inputs should be a list/tuple or a dict, but got %s!" % str(loss_inputs))
-
-    return res
-
-
-def _set_with_processed_inputs(network, inputs):
-    """Save set inputs for computation graph with processed inputs.
-
-    Args:
-        network (nn.Cell): Target cell.
-        inputs (Union[list, tuple, dict]): Inputs argument got from other cell.
-
-    Raises:
-        RuntimeError: network is not a nn.Cell.
-        RuntimeError: inputs is not a list, tuple or dict.
-    """
-    Validator.check_value_type('network', network, nn.Cell)
-    if isinstance(inputs, (list, tuple)):
-        network.set_inputs(*inputs)
-    elif isinstance(inputs, dict):
-        network.set_inputs(**inputs)
-    else:
-        raise RuntimeError(
-            "Reset inputs from a process inputs, should be a list/tuple or a dict, but got %s!" % str(inputs))
-
-
 class Model:
     """
     High-Level API for training or inference.
@@ -380,6 +241,7 @@ class Model:
         self._lite_infer = True  # if backend lite infer fails, set False
         self._mindspore_lite_model_group_id = id(self) & 0xFFFF
 
+
     def _check_for_graph_cell(self, kwargs):
         """Check for graph cell"""
         if not isinstance(self._network, nn.GraphCell):
@@ -450,10 +312,13 @@ class Model:
             raise ValueError("The argument 'optimizer' can not be None when set 'loss_scale_manager'.")
 
         net_inputs = network.get_inputs()
+        loss_inputs = [None]
         if self._loss_fn:
-            if self._loss_fn.get_inputs() and net_inputs:
-                loss_inputs = _process_loss_inputs(self._loss_fn.get_inputs())
-                net_inputs = _merge_inputs(net_inputs, loss_inputs)
+            if self._loss_fn.get_inputs():
+                loss_inputs = [*self._loss_fn.get_inputs()]
+            loss_inputs.pop(0)
+            if net_inputs:
+                net_inputs = [*net_inputs, *loss_inputs]
         if self._optimizer:
             amp_config = {}
             if self._loss_scale_manager_set:
@@ -471,7 +336,7 @@ class Model:
         # If need to check if loss_fn is not None, but optimizer is None
 
         if net_inputs is not None:
-            _set_with_processed_inputs(network, net_inputs)
+            network.set_inputs(*net_inputs)
         return network
 
     def _build_eval_network(self, metrics, eval_network, eval_indexes):
@@ -497,13 +362,17 @@ class Model:
                                  f" optional, and then you can set `eval_network` or `loss_fn`. For the latter case,"
                                  f" framework will automatically build an evaluation network with `network` and"
                                  f" `loss_fn`.")
+
             net_inputs = self._network.get_inputs()
-            if self._loss_fn.get_inputs() and net_inputs:
-                loss_inputs = _process_loss_inputs(self._loss_fn.get_inputs())
-                net_inputs = _merge_inputs(net_inputs, loss_inputs)
+            loss_inputs = [None]
+            if self._loss_fn.get_inputs():
+                loss_inputs = [*self._loss_fn.get_inputs()]
+            loss_inputs.pop(0)
+            if net_inputs:
+                net_inputs = [*net_inputs, *loss_inputs]
             self._eval_network = nn.WithEvalCell(self._network, self._loss_fn, self._amp_level in ["O2", "O3", "auto"])
             if net_inputs is not None:
-                _set_with_processed_inputs(self._eval_network, net_inputs)
+                self._eval_network.set_inputs(*net_inputs)
             self._eval_indexes = [0, 1, 2]
 
     def _build_predict_network(self):
@@ -576,6 +445,7 @@ class Model:
         if _get_recovery_context("enable_recovery") and is_train:
             _set_training_dataset(dataset_helper)
 
+
         network.set_train(is_train)
         network.phase = phase
         self._backbone_is_train = is_train
@@ -591,40 +461,6 @@ class Model:
             self._backbone_is_train = is_train
         return network
 
-    def _check_need_ckpt(self, callbacks):
-        """Check callback list contain ckpt"""
-        need_ckpt = False
-        save_ckpt_steps = 1
-        last_triggered_step = 0
-        for cb in callbacks:
-            if isinstance(cb, ModelCheckpoint):
-                need_ckpt = True
-                cfg_size = cb._get_save_checkpoint_steps
-                save_ckpt_steps = save_ckpt_steps if (cfg_size is None or cfg_size >= sys.maxsize) else cfg_size
-                last_triggered_step = cb._get_last_trigger_step
-                break
-        return need_ckpt, save_ckpt_steps, last_triggered_step
-
-    def _store_training_step_info(self, cb_params):
-        """
-        cache train step info
-        :param cb_params: callback params
-        :return: none
-        """
-        if os.environ.get("MS_ENABLE_CKPT_D2H_ASYNC") != "1":
-            return
-        if (context.get_context("mode") == context.GRAPH_MODE) and (context.get_context("device_target") == "Ascend"):
-            cb_params.need_ckpt, cb_params.save_checkpoint_steps, \
-            cb_params.last_triggered_step = self._check_need_ckpt(cb_params.list_callback)
-            logger.info(f"need_ckpt:{cb_params.need_ckpt},"
-                        f"save_checkpoint_steps:{cb_params.save_checkpoint_steps},"
-                        f"cur_step_num:{cb_params.cur_step_num},"
-                        f"last_triggered_step:{cb_params.last_triggered_step}")
-            context.set_context(ascend_config={"need_ckpt": cb_params.need_ckpt,
-                                               "save_checkpoint_steps": cb_params.save_checkpoint_steps,
-                                               "cur_step_num": cb_params.cur_step_num,
-                                               "last_triggered_step": cb_params.last_triggered_step})
-
     def _warmup_dataset(self, epoch, train_dataset, sink_size=-1):
         """
         Trigger dataset pipeline running before graph compiling.
@@ -650,22 +486,6 @@ class Model:
         train_dataset._dataset_helper = dataset_helper
         train_dataset._warmup_epoch = epoch
 
-    def _waiting_for_dataset_warmup_ready(self, train_dataset):
-        """
-        Wait for the dataset to warmup until there is a batch of data available for training on the device side.
-
-        Args:
-            train_dataset (Dataset): A training dataset iterator. If `train_dataset` is defined, training graphs will be
-                                     initialized. Default: ``None``.
-        """
-        mbuf_size = train_dataset.__transfer_dataset__.get_mbuf_queue_size()
-        while mbuf_size == 0:
-            time.sleep(10)
-            mbuf_size = train_dataset.__transfer_dataset__.get_mbuf_queue_size()
-            if mbuf_size != 0:
-                break
-            logger.warning(f"Waiting for the dataset warmup, current device queue size: {mbuf_size}")
-
     def _init(self, train_dataset=None, valid_dataset=None, sink_size=-1, epoch=1):
         """
         Initialize compute graphs and data graphs with the sink mode.
@@ -704,12 +524,17 @@ class Model:
                                                                         dataset_sink_mode=True,
                                                                         sink_size=sink_size)
             self._warmup_dataset(epoch, train_dataset, sink_size)
-
             # Since dataset pipeline has been triggered, delete flag
             delattr(train_dataset, "__no_send__")
-
-            # Waiting for the dataset warmup ready
-            self._waiting_for_dataset_warmup_ready(train_dataset)
+            if train_dataset.get_init_step() > 0:
+                mbuf_size = train_dataset.__transfer_dataset__.get_mbuf_queue_size()
+                while mbuf_size == 0:
+                    time.sleep(10)
+                    mbuf_size = train_dataset.__transfer_dataset__.get_mbuf_queue_size()
+                    if mbuf_size != 0:
+                        break
+                    logger.warning(f"Failover mode, waiting for dataset recover to specify step, "
+                                   f"current device queue size: {mbuf_size}")
 
             if context.get_auto_parallel_context("pipeline_stages") > 1 and valid_dataset:
                 train_network.add_flags_recursive(is_first_iteration=True)
@@ -787,10 +612,6 @@ class Model:
         cb_params.list_callback = self._transform_callbacks(callbacks)
         valid_infos = (valid_dataset, valid_frequency, valid_dataset_sink_mode)
         cb_params.list_callback.insert(0, _FrameworkProfilerCallback())
-        if os.environ.get("ENABLE_FLOPS_UTILIZATION_COLLECTOR") == "1" and \
-            FlopsUtilizationCollector not in cb_params.list_callback:
-            cb_params.list_callback.insert(0, FlopsUtilizationCollector(
-                cb_params.batch_num, full_flops=False))
         if context.get_context("mode") == context.PYNATIVE_MODE:
             cb_params.list_callback.insert(0, _StepSync())
         callbacks = cb_params.list_callback
@@ -849,7 +670,6 @@ class Model:
             dataset_sink_num = math.ceil(epoch * sink_size / dataset_size)
             train_dataset.__total_batch__ = epoch * sink_size
 
-        cb_params.sink_size = sink_size
         cb_params.cur_step_num = 0
         cb_params.dataset_sink_mode = True
 
@@ -895,7 +715,6 @@ class Model:
                 else:
                     cb_params.cur_step_num += 1
                 self._current_step_num = int((cb_params.cur_step_num - 1) % cb_params.batch_num + 1)
-                self._store_training_step_info(cb_params)
                 cb_params.train_dataset_element = inputs
                 list_callback.on_train_step_begin(run_context)
                 train_network = self._check_network_mode(train_network, True)
@@ -1150,31 +969,6 @@ class Model:
 
         list_callback.on_train_end(run_context)
 
-    def _wrapper_train(self, callbacks):
-        """
-        This method used to wrap train function with ttp wrapper which will do event notify when
-        exceptions throw.
-
-        Args:
-            callbacks (function): Callbacks passed by train method.
-        """
-
-        if not callbacks:
-            return self._train
-        cbs = callbacks if isinstance(callbacks, list) else [callbacks]
-        obj = None
-        _train_wrapper = None
-        for item in cbs:
-            if isinstance(item, MindIOTTPAdapter):
-                obj = item
-
-        if (obj is not None) and (obj.enable is True):
-            logger.info("MindIO TTP is enable, so we wrapper ttp exception handdler for self train method.")
-            _train_wrapper = obj.wrapper_ttp_persist(self._train)
-
-        return self._train if not _train_wrapper else _train_wrapper
-
-
     def train(self, epoch, train_dataset, callbacks=None, dataset_sink_mode=False, sink_size=-1, initial_epoch=0):
         """
         Training API.
@@ -1282,17 +1076,15 @@ class Model:
 
         _device_number_check(self._parallel_mode, self._device_number)
 
-        callbacks = _append_ccae(callbacks)
-        _train_wrapper = None
         if callbacks:
             self._check_methods_for_custom_callbacks(callbacks, "train")
-        _train_wrapper = self._wrapper_train(callbacks)
-        _train_wrapper(epoch,
-                       train_dataset,
-                       callbacks=callbacks,
-                       dataset_sink_mode=dataset_sink_mode,
-                       sink_size=sink_size,
-                       initial_epoch=initial_epoch)
+
+        self._train(epoch,
+                    train_dataset,
+                    callbacks=callbacks,
+                    dataset_sink_mode=dataset_sink_mode,
+                    sink_size=sink_size,
+                    initial_epoch=initial_epoch)
 
         # When it's distributed training and using MindRT,
         # the node id should be reset to start from 0.
@@ -1320,7 +1112,7 @@ class Model:
             callbacks = [callbacks]
         for cb in callbacks:
             cb_name = cb.__class__.__name__
-            if cb_name not in internal_cb_names:
+            if  cb_name not in internal_cb_names:
                 cb_methods_names = set(cb.__class__.__dict__.keys())
                 invalid_methods_names = cb_methods_names & old_version_methods_names
                 if invalid_methods_names:
@@ -1683,10 +1475,6 @@ class Model:
         cb_params.mode = "eval"
         cb_params.cur_step_num = 0
         cb_params.list_callback = self._transform_callbacks(callbacks)
-        if os.environ.get("ENABLE_FLOPS_UTILIZATION_COLLECTOR") == "1" and \
-            FlopsUtilizationCollector not in cb_params.list_callback:
-            cb_params.list_callback.insert(0, FlopsUtilizationCollector(
-                cb_params.batch_num, full_flops=False))
         cb_params.network = self._network
 
         self._clear_metrics()
@@ -1933,25 +1721,8 @@ class Model:
                 self._lite_infer = False
                 logger.warning(f"Lite inference failed, {e.__str__()}, fallback to original inference!")
 
-        def _check_input_data():
-            """Input data check."""
-            for item in predict_data:
-                if item is None:
-                    continue
-                if isinstance(item, Tensor):
-                    if item.size == 0:
-                        msg = "The input data can not be empty."
-                        logger.critical(msg)
-                        raise ValueError(msg)
-                    continue
-                if not isinstance(item, (int, float, str)):
-                    data_class_str = "Tensor, None, int, float, str"
-                    raise TypeError(f'The types of input data must be in the Union({data_class_str}, ' \
-                                    f'tuple[{data_class_str}], list[{data_class_str}], dict[{data_class_str}]), ' \
-                                    f'but got type {item if item is None else type(item).__name__}.')
-
         self._check_network_mode(self._predict_network, False)
-        _check_input_data()
+        check_input_data(*predict_data, data_class=(int, float, str, None, Tensor))
         _parallel_predict_check()
         result = self._predict_network(*predict_data)
 
@@ -2063,6 +1834,7 @@ class Model:
         train_dataset.__model_hash__ = hash(self)
         return train_network.parameter_layout_dict
 
+
     def infer_predict_layout(self, *predict_data, skip_backend_compile=False):
         """
         Generate parameter layout for the predict network in 'AUTO_PARALLEL' or 'SEMI_AUTO_PARALLEL' mode.