mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/train/callback/_on_request_exit.py

@@ -16,12 +16,19 @@
 
 from __future__ import absolute_import
 import os
+import json
 import signal
-
-from mindspore import log
+import threading
+from mindspore.common import dtype as mstype
+from mindspore import context
+from mindspore import log as logger
+from mindspore.common.tensor import Tensor
+from mindspore.train._utils import _make_directory
 from mindspore import _checkparam as Validator
 from mindspore.train.serialization import load_checkpoint, save_checkpoint, export
 from mindspore.train.callback._callback import Callback
+from mindspore.parallel._utils import _get_parallel_mode
+from mindspore.context import ParallelMode
 
 
 class OnRequestExit(Callback):
@@ -29,7 +36,8 @@ class OnRequestExit(Callback):
     Respond to the user's closing request, exit the training or eval process, and save the checkpoint and mindir.
 
     Register OnRequestExit Callback before training, when the user want to exit the training process
-    and save the training data, could send the registered exit signal 'sig' to the training process.
+    and save the training data, could send the registered exit signal 'sig' to the training process or modify the
+    'GracefulExit' that a key in the json file specified by the 'config_file' to '1'.
     After the training process executes the current step, saves the current training status,
     including checkpoint and mindir, and then exit the training process.
 
@@ -38,9 +46,12 @@ class OnRequestExit(Callback):
         save_mindir (bool): Whether save the mindir before the training process exit. Default: ``True`` .
         file_name (str): The saved checkpoint and mindir file name,
             the checkpoint file add suffix '.ckpt', the mindir file add suffix '.mindir'. Default: ``'Net'`` .
-        directory (str): The directory save checkpoint and mindir. Default: ``'./'`` .
+        directory (str): The path to save files. It will generate a 'rank_{id}' path by rank_id
+            to save checkpoint and mindir. Default: ``'./'`` .
         sig (int): The user registered exit signal, it must be a captureable and negligible signal.
             When the process receives the signal, exits the training or eval process. Default: ``signal.SIGTERM`` .
+        config_file (str): A json config file used to exit training process gracefully. Key: ``{"GracefulExit": 1}`` .
+            Default: ``None`` .
 
     Raises:
         ValueError: If the 'save_ckpt' is not a bool.
@@ -55,32 +66,40 @@ class OnRequestExit(Callback):
         >>> import mindspore as ms
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
         >>> optim = nn.Momentum(net.trainable_params(), 0.01, 0.9)
         >>> model = Model(net, loss_fn=loss, optimizer=optim)
         >>> # Create the dataset taking MNIST as an example. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/mnist.py
         >>> dataset = create_dataset()
         >>> on_request_exit = ms.train.OnRequestExit(file_name='LeNet5')
         >>> model.train(10, dataset, callbacks=on_request_exit)
     """
 
-    def __init__(self, save_ckpt=True, save_mindir=True, file_name='Net', directory='./', sig=signal.SIGTERM):
+    def __init__(self, save_ckpt=True, save_mindir=True, file_name='Net', directory='./', config_file=None,
+                 sig=signal.SIGTERM):
         super(OnRequestExit, self).__init__()
         self.save_ckpt = Validator.check_isinstance('save_ckpt', save_ckpt, bool)
         self.save_mindir = Validator.check_isinstance('save_mindir', save_mindir, bool)
-        if self.save_ckpt or self.save_mindir:
-            file_name = Validator.check_isinstance('file_name', file_name, str)
-            directory = Validator.check_isinstance('directory', directory, str)
-            os.makedirs(os.path.abspath(directory), exist_ok=True)
-            self.train_file_path = os.path.abspath(os.path.join(directory, f"{file_name}_train"))
-            self.eval_file_path = os.path.abspath(os.path.join(directory, f"{file_name}_eval"))
         self.sig = Validator.check_isinstance('sig', sig, int)
         if hasattr(signal, "SIGKILL") and self.sig == signal.SIGKILL:
             raise ValueError("Not support send exit request by signal SIGKILL.")
-        self.exit = False
+        self.exit = False  # used signal to exit the training process
+        self.lock = threading.Lock()
+        self.save_path = directory
+        self.key = "GracefulExit"
+        self.remote_config_file = config_file  # used config file to save checkpoint and exit training process
+        self.use_graceful = os.environ.get("MS_ENABLE_GRACEFUL_EXIT") == "1"
+        self.is_distributed = _get_parallel_mode() != ParallelMode.STAND_ALONE
+        self.integrated_save = True
+        if self.is_distributed:
+            self.integrated_save = _get_parallel_mode() == ParallelMode.AUTO_PARALLEL
+        self.stop_train = False
+        self.need_do_step_end = False
+        if self.save_ckpt or self.save_mindir:
+            self.train_name, self.eval_name = self._get_save_path(file_name)
 
     def on_train_begin(self, run_context):
         """
@@ -91,22 +110,31 @@ class OnRequestExit(Callback):
                 For more details, please refer to :class:`mindspore.train.RunContext`.
         """
         signal.signal(self.sig, self._handle_signal)
-        if self.save_ckpt and os.path.isfile(f"{self.train_file_path}.ckpt"):
+        if self.save_ckpt and os.path.isfile(f"{self.train_name}.ckpt"):
             cb_params = run_context.original_args()
             train_net = cb_params.train_network
-            load_checkpoint(f"{self.train_file_path}.ckpt", net=train_net)
+            load_checkpoint(f"{self.train_name}.ckpt", net=train_net)
+
+    def on_train_step_begin(self, run_context):
+        """
+        Check whether received the exit signal or
+        whether the value of 'GracefulExit' in 'config_file' was changed to '1'.
+
+        Args:
+            run_context (RunContext): Context information of the model.
+                For more details, please refer to :class:`mindspore.train.RunContext`.
+        """
+        self._do_step_begin(run_context)
 
     def on_train_step_end(self, run_context):
         """
-        When the train step end, if received the exit signal, set the 'run_context' attribute '_stop_requested' to True.
-        Then exit the training process after this step training.
+        Save checkpoint file or mindir file according to config, and exit the training process.
 
         Args:
             run_context (RunContext): Include some information of the model.
                 For more details, please refer to :class:`mindspore.train.RunContext`.
         """
-        if self.exit:
-            run_context.request_stop()
+        self._do_step_end(run_context)
 
     def on_train_epoch_end(self, run_context):
         """
@@ -118,8 +146,7 @@ class OnRequestExit(Callback):
             run_context (RunContext): Include some information of the model.
                 For more details, please refer to :class:`mindspore.train.RunContext`.
         """
-        if self.exit:
-            run_context.request_stop()
+        self._do_step_end(run_context)
 
     def on_train_end(self, run_context):
         """
@@ -135,10 +162,10 @@ class OnRequestExit(Callback):
         cb_params = run_context.original_args()
         train_net = cb_params.train_network
         if self.save_ckpt:
-            save_checkpoint(train_net, ckpt_file_name=self.train_file_path)
+            save_checkpoint(train_net, ckpt_file_name=self.train_name)
         if self.save_mindir:
             inputs = cb_params.train_dataset_element
-            export(train_net, *inputs, file_name=self.train_file_path, file_format='MINDIR')
+            export(train_net, *inputs, file_name=self.train_name, file_format='MINDIR')
 
     def on_eval_begin(self, run_context):
         """
@@ -153,15 +180,15 @@ class OnRequestExit(Callback):
             return
         cb_params = run_context.original_args()
         eval_net = cb_params.eval_network
-        if os.path.isfile(f"{self.eval_file_path}.ckpt"):
-            load_checkpoint(f"{self.eval_file_path}.ckpt", net=eval_net)
-        elif os.path.isfile(f"{self.train_file_path}.ckpt"):
-            load_checkpoint(f"{self.train_file_path}.ckpt", net=eval_net)
+        if os.path.isfile(f"{self.eval_name}.ckpt"):
+            load_checkpoint(f"{self.eval_name}.ckpt", net=eval_net)
+        elif os.path.isfile(f"{self.train_name}.ckpt"):
+            load_checkpoint(f"{self.train_name}.ckpt", net=eval_net)
 
     def on_eval_step_end(self, run_context):
         """
-        When the eval step end, if received the exit signal, set the 'run_context' attribute '_stop_requested' to True.
-        Then exit the eval process after this step eval.
+        When the eval step end, if received the exit signal, set attribute '_stop_requested' of the
+        'run_context' to True. Then exit the eval process after this step eval.
 
         Args:
             run_context (RunContext): Include some information of the model.
@@ -184,12 +211,88 @@ class OnRequestExit(Callback):
         cb_params = run_context.original_args()
         eval_net = cb_params.eval_network
         if self.save_ckpt:
-            save_checkpoint(eval_net, ckpt_file_name=self.eval_file_path)
+            save_checkpoint(eval_net, ckpt_file_name=self.eval_name)
         if self.save_mindir:
             inputs = cb_params.eval_dataset_element
-            export(eval_net, *inputs, file_name=self.eval_file_path, file_format='MINDIR')
+            export(eval_net, *inputs, file_name=self.eval_name, file_format='MINDIR')
 
     def _handle_signal(self, signum, frame):
         """Handle the received signal"""
-        log.debug(f"signum: {signum}, frame: {frame}")
+        logger.debug(f"signum: {signum}, frame: {frame}")
         self.exit = True
+
+    def _do_step_end(self, run_context):
+        """
+        Save the checkpoint or mindir, and then exit training process.
+
+        Args:
+            run_context (RunContext): Include some information of the model.
+                For more details, please refer to :class:`mindspore.train.RunContext`.
+        """
+        with self.lock:
+            # save once
+            if self.stop_train or not self.need_do_step_end:
+                return
+            logger.info("Gracefully exiting training process on step end.")
+            call_params = run_context.original_args()
+            net = call_params.train_network
+            for _, param in net.parameters_and_names():
+                if param.name == "graceful_exit" and param.asnumpy() == True:  # pylint: disable=C0121
+                    logger.warning("Graceful exit is triggered, stop training.")
+                    if self.save_ckpt:
+                        save_checkpoint(net, self.train_name, integrated_save=self.integrated_save)
+                    if self.save_mindir:
+                        inputs = call_params.train_dataset_element
+                        export(net, *inputs, file_name=self.train_name, file_format='MINDIR')
+                    run_context.request_stop()
+                    self.stop_train = True
+
+    def _do_step_begin(self, run_context):
+        """
+        Check training process exit configuration at the step begin.
+
+        Args:
+            run_context (RunContext): Include some information of the model.
+                For more details, please refer to :class:`mindspore.train.RunContext`.
+        """
+        with self.lock:
+            # no env
+            if not self.use_graceful:
+                return
+            if self._check_config_info() or self.exit:
+                call_params = run_context.original_args()
+                net = call_params.train_network
+                for _, param in net.parameters_and_names():
+                    if not self.is_distributed and param.name == "graceful_exit":
+                        param.set_data(Tensor(True, mstype.bool_))
+                        self.need_do_step_end = True
+                        break
+                    if param.name == "graceful_init":
+                        param.set_data(Tensor([1], mstype.int32))
+                        self.need_do_step_end = True
+                        break
+
+    def _check_config_info(self):
+        """check json config info"""
+        if self.remote_config_file is not None and os.path.exists(self.remote_config_file):
+            with open(self.remote_config_file, "r") as f:
+                try:
+                    config_info = json.load(f)
+                except json.JSONDecodeError as e:
+                    logger.warning(f"Parse json file failed: {e}, please check json file: {self.remote_config_file}")
+                    return False
+                if self.key in config_info and config_info[self.key] == 1:
+                    return True
+        return False
+
+    def _get_save_path(self, file_name):
+        """path to save checkpoint files or mindir files"""
+        device_id = context.get_context("device_id")
+        if self.save_path is None:
+            tmp = os.path.join(os.getcwd(), r"rank_" + str(device_id))
+            path_ = _make_directory(tmp)
+            return os.path.join(path_, f"{file_name}_train"), os.path.join(path_, f"{file_name}_eval")
+
+        save_path = os.path.join(self.save_path, r"rank_" + str(device_id))
+        save_path = _make_directory(save_path)
+        return os.path.join(save_path, f"{file_name}_train"), os.path.join(save_path, f"{file_name}_eval")

mindspore/train/callback/_reduce_lr_on_plateau.py

@@ -84,13 +84,13 @@ class ReduceLROnPlateau(Callback):
         >>> from mindspore import nn
         >>> from mindspore.train import Model, ReduceLROnPlateau
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
         >>> optim = nn.Momentum(net.trainable_params(), 0.01, 0.9)
         >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics={"acc"})
         >>> # Create the dataset taking MNIST as an example. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/mnist.py
         >>> dataset = create_dataset()
         >>> cb = ReduceLROnPlateau(monitor="acc", patience=3, verbose=True)
         >>> model.fit(10, dataset, callbacks=cb)

mindspore/train/callback/_summary_collector.py

@@ -41,7 +41,7 @@ from mindspore.nn.optim.optimizer import Optimizer
 from mindspore.nn.loss.loss import LossBase
 from mindspore.train._utils import check_value_type, _make_directory
 from mindspore._c_expression import security
-from mindspore._c_expression import _collect_host_info
+from mindspore._c_expression import collect_host_info, get_clock_syscnt
 
 HYPER_CONFIG_ENV_NAME = "MINDINSIGHT_HYPER_CONFIG"
 HYPER_CONFIG_LEN_LIMIT = 100000
@@ -105,11 +105,11 @@ class SummaryCollector(Callback):
               training computational graph is collected. Default: ``True`` .
             - collect_train_lineage (bool): Whether to collect lineage data for the training phase,
               this field will be displayed on the `lineage page \
-              <https://www.mindspore.cn/mindinsight/docs/en/r2.2/lineage_and_scalars_comparison.html>`_
+              <https://www.mindspore.cn/mindinsight/docs/en/master/lineage_and_scalars_comparison.html>`_
               of MindInsight. Default: ``True`` .
             - collect_eval_lineage (bool): Whether to collect lineage data for the evaluation phase,
               this field will be displayed on the `lineage page
-              <https://www.mindspore.cn/mindinsight/docs/en/r2.2/lineage_and_scalars_comparison.html>`_
+              <https://www.mindspore.cn/mindinsight/docs/en/master/lineage_and_scalars_comparison.html>`_
               of MindInsight. Default: ``True`` .
             - collect_input_data (bool): Whether to collect dataset for each training.
               Currently only image data is supported.
@@ -134,7 +134,7 @@ class SummaryCollector(Callback):
                 Optional: epoch/step.
               - create_landscape (dict): Select how to create loss landscape.
                 Training process loss landscape(train) and training result loss landscape(result).
-                Default: {"train": True, "result": True}. Optional: ``True`` / ``False`` .
+                Default: ``{"train": True, "result": True}``. Optional: ``True`` / ``False`` .
               - num_samples (int): The size of the dataset used to create the loss landscape.
                 For example, in image dataset, You can set num_samples is 128,
                 which means that 128 images are used to create loss landscape.
@@ -150,7 +150,7 @@ class SummaryCollector(Callback):
             False: it means that after specified data is set, only the specified data is collected,
             and the others are not collected. Default: ``True`` .
         custom_lineage_data (Union[dict, None]): Allows you to customize the data and present it on the MingInsight
-            `lineage page <https://www.mindspore.cn/mindinsight/docs/en/r2.2/lineage_and_scalars_comparison.html>`_ .
+            `lineage page <https://www.mindspore.cn/mindinsight/docs/en/master/lineage_and_scalars_comparison.html>`_ .
             In the custom data, the type of the key supports str, and the type of value supports str, int
             and float. Default: ``None`` , it means there is no custom data.
         collect_tensor_freq (Optional[int]): The same semantics as the `collect_freq`, but controls TensorSummary only.
@@ -190,10 +190,10 @@ class SummaryCollector(Callback):
         ...     ms.set_context(mode=ms.GRAPH_MODE, device_target="Ascend")
         ...     mnist_dataset_dir = '/path/to/mnist_dataset_directory'
         ...     # Create the dataset taking MNIST as an example. Refer to
-        ...     # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        ...     # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/mnist.py
         ...     ds_train = create_dataset()
         ...     # Define the network structure of LeNet5. Refer to
-        ...     # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        ...     # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         ...     network = LeNet5(10)
         ...     net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
         ...     net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9)
@@ -472,7 +472,7 @@ class SummaryCollector(Callback):
 
     def begin(self, run_context):
         cb_params = run_context.original_args()
-        _collect_host_info("Callback", "SummaryCollector", "begin", level=1)
+        collect_host_info("Callback", "SummaryCollector", "begin", start_time=get_clock_syscnt(), level=1)
         self._check_callbacks(cb_params)
 
         if cb_params.mode not in ModeEnum.to_list():
@@ -484,7 +484,7 @@ class SummaryCollector(Callback):
 
     def step_end(self, run_context):
         cb_params = run_context.original_args()
-        _collect_host_info("Callback", "SummaryCollector", "step_end", level=1)
+        collect_host_info("Callback", "SummaryCollector", "step_end", start_time=get_clock_syscnt(), level=1)
         if cb_params.mode != ModeEnum.TRAIN.value:
             return
 
@@ -559,7 +559,7 @@ class SummaryCollector(Callback):
 
     def epoch_end(self, run_context):
         cb_params = run_context.original_args()
-        _collect_host_info("Callback", "SummaryCollector", "epoch_end", level=1)
+        collect_host_info("Callback", "SummaryCollector", "epoch_end", start_time=get_clock_syscnt(), level=1)
         self._collect_tensor_data(cb_params)
         collect_landscape = self._collect_specified_data.get('collect_landscape')
         if collect_landscape is not None:
@@ -576,7 +576,7 @@ class SummaryCollector(Callback):
 
     def end(self, run_context):
         cb_params = run_context.original_args()
-        _collect_host_info("Callback", "SummaryCollector", "end", level=1)
+        collect_host_info("Callback", "SummaryCollector", "end", start_time=get_clock_syscnt(), level=1)
         if cb_params.mode == ModeEnum.TRAIN.value:
             self._collect_train_lineage(cb_params)
         else:
@@ -1156,7 +1156,7 @@ class SummaryCollector(Callback):
         except TypeError as exc:
             logger.warning("Summary cannot collect the type of metrics, currently support type: dict, list, tuple, "
                            "str, int, float, bool and None. %s.", str(exc))
-        _ = self._parse_dataset(cb_params, eval_lineage)
+        self._parse_dataset(cb_params, eval_lineage)
 
         eval_lineage_message = self._package_eval_lineage_message(eval_lineage)
         self._record.add_value(PluginEnum.EVAL_LINEAGE.value, 'eval_lineage', eval_lineage_message)