mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/train/callback/_cluster_monitor.py

@@ -0,0 +1,201 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""ClusterMonitor Callback class."""
+from __future__ import absolute_import
+
+import os
+import stat
+import glob
+import time
+from threading import RLock
+
+from mindspore.train.callback._callback import Callback
+from mindspore.communication.management import get_rank, get_local_rank
+from mindspore import log as logger
+from mindspore.parallel._auto_parallel_context import _get_auto_parallel_context
+from mindspore.parallel._utils import _get_device_num
+from mindspore.train._utils import get_parameter_redundancy
+
+_perf_mutex = RLock()
+
+
+def _get_dp_tp_from_redundancy(redundancy_tuple):
+    """From redundancy get dp and tp"""
+    dp = []
+    tp = []
+    for dp_value in redundancy_tuple:
+        dp.append(list(dp_value))
+    for i in range(len(redundancy_tuple[0])):
+        tp.append([v[i] for v in redundancy_tuple])
+    return dp, tp
+
+
+def _get_dp_tp_from_layout(parameter_layout_dict, initial_rank=0):
+    """From layout dict get dp and tp"""
+    tp = []
+    dp = []
+    parameter_redundancy_dict = get_parameter_redundancy(parameter_layout_dict, initial_rank)
+    value_len = 0
+    for _, value in parameter_redundancy_dict.items():
+        if len(value) > value_len:
+            value_len = len(value)
+            dp, tp = _get_dp_tp_from_redundancy(value)
+    return dp, tp
+
+
+def _check_perf_config(perf_config):
+    """Check if the format of perf_config is correct."""
+    enabled = perf_config.get("enable", None)
+    if enabled is None or not isinstance(enabled, bool):
+        raise TypeError(f"For cluster monitor, enabled should be bool, but got {type(enabled)}.")
+    enable_step_time = perf_config.get("steptime", None)
+    if enable_step_time is None or not isinstance(enable_step_time, bool):
+        raise TypeError(f"For cluster monitor, enable_step_time should be bool, but got {type(enable_step_time)}.")
+    enabled_dtp_group = perf_config.get("dtpGroup", None)
+    if enabled_dtp_group is None or not isinstance(enabled_dtp_group, bool):
+        raise TypeError(f"For cluster monitor, enabled_dtp_group should be bool, but got {type(enabled_dtp_group)}.")
+
+
+def _parse_perf_config():
+    """parse perf config"""
+    perf_config = os.getenv("PERF_DUMP_CONFIG")
+    perf_config_dict = {}
+    if perf_config is None:
+        return perf_config_dict
+    pairs = perf_config.split(',')
+    for pair in pairs:
+        key, value = pair.split(':')
+        if value.lower() == 'true':
+            perf_config_dict[key] = True
+        elif value.lower() == 'false':
+            perf_config_dict[key] = False
+        elif value.isdigit():
+            perf_config_dict[key] = int(value)
+        else:
+            perf_config_dict[key] = value
+    _check_perf_config(perf_config_dict)
+    return perf_config_dict
+
+
+def _remove_pre_log():
+    """Remove the previously saved log files."""
+    directory = os.getenv("PERF_DUMP_PATH")
+    device_id = get_local_rank()
+    pattern = os.path.join(directory, f"perf_ms_*_{device_id}.log")
+    files_to_delete = glob.glob(pattern)
+    for file_path in files_to_delete:
+        if os.path.islink(file_path):
+            continue
+        try:
+            os.remove(file_path)
+        except OSError as e:
+            logger.warning(f"When CCAE is opening, {file_path} need to be removed, but failed to remove.")
+            raise e
+
+
+class ClusterMonitor(Callback):
+    """
+    Monitor the cluster in train process.
+    """
+
+    def __init__(self):
+        super(ClusterMonitor, self).__init__()
+        self.perf_config = _parse_perf_config()
+        self.enabled = self.perf_config.get("enable")
+        self.enable_step_time = self.perf_config.get("steptime")
+        self.enabled_dtp_group = self.perf_config.get("dtpGroup")
+        self.data_time_start = None
+        self.data_time_end = None
+        self.frame_work = "MindSpore"
+        self.ms_sched_host = os.getenv("MS_SCHED_HOST", "127.0.0.1")
+        self.ms_sched_port = os.getenv("MS_SCHED_PORT", "8888")
+        self.uuid_value = self.ms_sched_host + "_" + self.ms_sched_port
+        self.global_rank = get_rank()
+        self.process_id = os.getpid()
+        self.device_id = get_local_rank()
+        self.log_name = "perf_ms" + "_" + str(self.process_id) + "_" + str(self.device_id) + ".log"
+        self.log_path = os.getenv("PERF_DUMP_PATH")
+        if not self.log_path.endswith(os.path.sep):
+            self.log_path += os.path.sep
+        self.full_path = self.log_path + self.log_name
+
+        self.write_dp_tp_flag = True
+        self.initial_rank = 0
+
+    def begin(self, run_context):
+        _remove_pre_log()
+        pp_num = _get_auto_parallel_context("pipeline_stages")
+        device_num = _get_device_num()
+
+        original_list = list(range(device_num))
+        chunk_size = device_num // pp_num
+        split_pp_lists = []
+        for i in range(0, device_num, chunk_size):
+            end_index = i + chunk_size if i + chunk_size <= device_num else device_num
+            split_pp_lists.append(original_list[i:end_index])
+
+        self.initial_rank = (self.global_rank // chunk_size) * chunk_size
+        with _perf_mutex:
+            dir_path = os.path.dirname(self.full_path)
+            if not os.path.exists(dir_path):
+                os.makedirs(dir_path, mode=0o700)
+            if os.path.exists(self.full_path):
+                os.chmod(self.full_path, stat.S_IWUSR)
+                os.remove(self.full_path)
+            with open(self.full_path, 'w') as file:
+                log_message = f'UUID:{self.uuid_value}\nFRAMEWORK:{self.frame_work}\nGLOBAL RANKID:{self.global_rank}\n'
+                file.write(log_message)
+                for _, split_pp_list in enumerate(split_pp_lists):
+                    file.write(f'PP:{split_pp_list}\n')
+            os.chmod(self.full_path, stat.S_IRUSR)
+
+    def step_begin(self, run_context):
+        """
+        Record time at the beginning of step.
+
+        Args:
+            run_context (RunContext): Context of the process running. For more details,
+                    please refer to :class:`mindspore.train.RunContext`.
+        """
+        self.data_time_start = time.time()
+
+    def step_end(self, run_context):
+        """
+        Record time at the end of step.
+
+        Args:
+            run_context (RunContext): Context of the process running. For more details,
+                    please refer to :class:`mindspore.train.RunContext`.
+        """
+        self.data_time_end = time.time()
+        if self.enabled and self.enabled_dtp_group and self.write_dp_tp_flag:
+            cb_params = run_context.original_args()
+            param_layout_dict = cb_params.train_network.parameter_layout_dict
+            dp, tp = _get_dp_tp_from_layout(param_layout_dict, self.initial_rank)
+            with _perf_mutex:
+                os.chmod(self.full_path, stat.S_IWUSR)
+                with open(self.full_path, 'a') as file:
+                    for dp_value in dp:
+                        file.write(f'dp:{dp_value}\n')
+                    for tp_value in tp:
+                        file.write(f'tp:{tp_value}\n')
+                os.chmod(self.full_path, stat.S_IRUSR)
+            self.write_dp_tp_flag = False
+        if self.enabled and self.enable_step_time:
+            with _perf_mutex:
+                os.chmod(self.full_path, stat.S_IWUSR)
+                with open(self.full_path, 'a') as file:
+                    file.write(f"STEPTIME:{int(self.data_time_start * 1000)},{int(self.data_time_end * 1000)}\n")
+                os.chmod(self.full_path, stat.S_IRUSR)

mindspore/train/callback/_early_stop.py

@@ -85,13 +85,13 @@ class EarlyStopping(Callback):
         >>> from mindspore import nn
         >>> from mindspore.train import Model, EarlyStopping
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
         >>> optim = nn.Momentum(net.trainable_params(), 0.01, 0.9)
         >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics={"acc"})
         >>> # Create the dataset taking MNIST as an example. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/mnist.py
         >>> dataset = create_dataset()
         >>> cb = EarlyStopping(monitor="acc", patience=3, verbose=True)
         >>> model.fit(10, dataset, callbacks=cb)

mindspore/train/callback/_flops_collector.py

@@ -0,0 +1,239 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""FLops Utilization collector Callback class."""
+from __future__ import absolute_import
+
+import time
+import os
+import stat
+import hashlib
+
+from math import floor
+from mindspore import _checkparam as Validator
+from mindspore import log as logger
+from mindspore.train.callback._callback import Callback
+from mindspore.common.api import flops_collection
+
+from mindspore.communication.management import (create_group, get_group_size,
+                                                get_rank)
+from mindspore.parallel._auto_parallel_context import auto_parallel_context
+from mindspore.ops import operations as P
+from mindspore.common import Tensor
+from mindspore import context
+import mindspore.nn as nn
+
+
+class AllReduceNet(nn.Cell):
+    """
+    Used to accumulate flops in pipeline parallel.
+    """
+    def __init__(self, group_name):
+        super(AllReduceNet, self).__init__()
+        self.allreduce_sum = P.AllReduce(op=P.ReduceOp.SUM, group=group_name)
+        self.add_flags(skip_auto_parallel_compile=True)
+
+    def construct(self, x):
+        return self.allreduce_sum(x)
+
+
+class FlopsUtilizationCollector(Callback):
+    """
+    The FlopsUtilizationCollector interface counts the model utilization information MFU
+    and the hardware utilization information HFU.
+    Currently, the API counts only the forward and backward flops of MatMul,
+    BatchMatMul, FlashAttentionScore, and Conv2D operators.
+    Only used in graph mode with static shape.
+
+    Args:
+        data_size (int): How many steps are the intervals between print information each time.
+        computility (int): The peak flops of each compute card. Default: ``1`` .
+        full_flops(bool): Whether to count the full model flops. If set full_flops to False,
+            FlopsUtilizationCollector would count the shard model flops in each device. Default: ``True`` .
+
+    Raises:
+        TypeError: If data_size is not positive int.
+        TypeError: If full_flops is not bool.
+        AssertionError: If the training mode is not a static graph or not a static shape.
+
+    Examples:
+        >>> import numpy as np
+        >>> import mindspore.dataset as ds
+        >>> from mindspore import nn
+        >>> from mindspore.train import Model, FlopsUtilizationCollector
+        >>> from mindspore import context
+        >>> context.set_context(mode=context.GRAPH_MODE)
+        >>> data = {"x": np.float32(np.random.rand(64, 10)), "y": np.random.randint(0, 5, (64,))}
+        >>> train_dataset = ds.NumpySlicesDataset(data=data).batch(32)
+        >>> net = nn.Dense(10, 5)
+        >>> crit = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
+        >>> opt = nn.Momentum(net.trainable_params(), 0.01, 0.9)
+        >>> flops_callback = FlopsUtilizationCollector(train_dataset.get_dataset_size(), computility=10e6)
+        >>> model = Model(network=net, optimizer=opt, loss_fn=crit, metrics={"recall"})
+        >>> model.train(2, train_dataset, callbacks=[flops_callback])
+        Full model flops is 6400, Full hardware flops is 6400, Shard model flops is 6400, Shard hardware flops is 6400
+        Train per step time: 135.572 ms, mfu:0.47% hfu:0.47%
+        Train per step time: 1.317 ms, mfu:48.59% hfu:48.59%
+    """
+    def __init__(self, data_size, computility=1, full_flops=True):
+        super(FlopsUtilizationCollector, self).__init__()
+        self.step_time = time.time()
+        self.computility = computility
+        self.full_mfu = 0.0
+        self.full_hfu = 0.0
+        self.shard_mfu = 0.0
+        self.shard_hfu = 0.0
+        self.full_model_flops = 0.0
+        self.shard_model_flops = 0.0
+        self.full_hardware_flops = 0.0
+        self.shard_hardware_flops = 0.0
+        self.mfu_calculated = False
+        self.data_size = data_size
+        self.time_step_path = ''
+        self.verbose = full_flops
+        self.ma = os.environ.get("ENABLE_FLOPS_UTILIZATION_COLLECTOR") == "1"
+        Validator.check_bool(full_flops, "verbose")
+        Validator.check_positive_int(data_size, "data_size")
+
+    def step_begin(self, run_context):
+        """
+        Record time at the beginning of step.
+
+
+        Args:
+            run_context (RunContext): Context of the process running. For more details,
+                    please refer to :class:`mindspore.train.RunContext`.
+        """
+        self.step_time = time.time()
+
+    def _get_pipeline_group(self):
+        """
+        Calculate the communication group between all pipeline stages
+        """
+        rank = get_rank()
+        stage_nums = auto_parallel_context().get_pipeline_stages()
+        device_nums = get_group_size()
+        per_stage_device_nums = device_nums // stage_nums
+        local_stage_rank_id = rank % per_stage_device_nums
+        group = range(0, stage_nums)
+        rank_list = [local_stage_rank_id + x *
+                     per_stage_device_nums for x in group]
+        rank_str_list = [str(local_stage_rank_id + x *
+                             per_stage_device_nums) for x in group]
+        rank_list_str = "-".join(rank_str_list)
+        return rank_list, rank_list_str
+
+    def step_end(self, run_context):
+        """
+        Print mfu and hfu time at the end of step.
+
+        Args:
+           run_context (RunContext): Context of the process running. For more details,
+                   please refer to :class:`mindspore.train.RunContext`.
+        """
+        if context.get_context("mode") != context.GRAPH_MODE:
+            logger.warning("FlopsUtilizationCollector now only support graph mode.")
+            return
+
+        step_seconds = (time.time() - self.step_time) * 1000
+        if not self.mfu_calculated:
+            cb_params = run_context.original_args()
+            if cb_params.mode == 'train':
+                network = cb_params.train_network
+            elif cb_params.mode == 'eval':
+                network = cb_params.eval_network
+            else:
+                logger.warning('FlopsUtilizationCollector only support train and eval mode!')
+                return
+            full_model_flops, full_hardware_flops, shard_model_flops, \
+                shard_hardware_flops, is_dynamic_shape = flops_collection(network.current_phase)
+            if is_dynamic_shape:
+                logger.warning("FlopsUtilizationCollector now do not support dynamic shape.")
+                return
+            self.full_mfu = full_model_flops / self.computility
+            self.full_hfu = full_hardware_flops / self.computility
+
+            self.shard_mfu = shard_model_flops / self.computility
+            self.shard_hfu = shard_hardware_flops / self.computility
+            self.full_model_flops = full_model_flops
+            self.full_hardware_flops = full_hardware_flops
+            self.shard_model_flops = shard_model_flops
+            self.shard_hardware_flops = shard_hardware_flops
+            self.mfu_calculated = True
+            shard_mf_dir = os.path.realpath(os.getenv('MA_LOG_DIR', './'))
+            if self.ma:
+                flops_path = os.path.join(shard_mf_dir, "flops.txt")
+                self.time_step_path = os.path.join(shard_mf_dir, "time_step.txt")
+                if auto_parallel_context().get_parallel_mode() != "stand_alone":
+                    flops_path = os.path.join(
+                        shard_mf_dir, "flops_rank_" + str(get_rank())) + ".txt"
+                    self.time_step_path = os.path.join(
+                        shard_mf_dir, "time_step_rank_" + str(get_rank())) + ".txt"
+                time_stamp = time.time()
+                model_flops_log = "flops{{type=\"model_flops\"}} {} {}\n".\
+                    format(shard_model_flops, int(round(time_stamp * 1000)))
+                hardware_flops_log = "flops{{type=\"hardware_flops\"}} {} {}\n".\
+                    format(shard_hardware_flops, int(round(time_stamp * 1000)))
+                flags = os.O_WRONLY | os.O_CREAT
+                modes = stat.S_IWUSR | stat.S_IRUSR
+                with os.fdopen(os.open(flops_path, flags, modes), 'w') as f:
+                    f.write(model_flops_log)
+                    f.write(hardware_flops_log)
+            if self.verbose:
+                flops_log = f"Full model flops is {full_model_flops}, Full hardware flops is {full_hardware_flops}, " \
+                            f"Shard model flops is {shard_model_flops}, Shard hardware flops is {shard_hardware_flops}."
+                print(flops_log, flush=True)
+                if auto_parallel_context().get_pipeline_stages() > 1:
+                    pipeline_group_list, pipeline_group_name = self._get_pipeline_group()
+                    auto_parallel_context().set_pipeline_stages(1)
+                    hashed = hashlib.md5(
+                        pipeline_group_name.encode()).hexdigest()[:48]
+                    pipeline_group_name = str(hashed)
+                    create_group(pipeline_group_name, pipeline_group_list)
+                    self.full_mfu = AllReduceNet(pipeline_group_name)(
+                        Tensor([self.full_mfu])).asnumpy()[0]
+                    self.full_hfu = AllReduceNet(pipeline_group_name)(
+                        Tensor([self.full_hfu])).asnumpy()[0]
+                if auto_parallel_context().get_parallel_mode() != "stand_alone":
+                    self.full_mfu = self.full_mfu / get_group_size()
+                    self.full_hfu = self.full_hfu / get_group_size()
+
+        step_size = self.data_size
+        cb_params = run_context.original_args()
+        if hasattr(cb_params, "batch_num"):
+            batch_num = cb_params.batch_num
+            if isinstance(batch_num, int) and batch_num > 0:
+                step_size = cb_params.batch_num
+        Validator.check_positive_int(step_size)
+        if cb_params.dataset_sink_mode:
+            step_seconds = step_seconds / step_size
+        time_stamp = time.time()
+        train_log = "time_monitor{{type=\"per_step_time\"}} {} {}".format(step_seconds, int(round(time_stamp * 1000)))
+        if self.ma:
+            flags = os.O_WRONLY | os.O_CREAT
+            modes = stat.S_IWUSR | stat.S_IRUSR
+            with os.fdopen(os.open(self.time_step_path, flags, modes), 'w') as f:
+                f.write(train_log + '\n')
+        train_log = "{} per step time: {:5.3f} ms".format(
+            cb_params.mode.title(), step_seconds)
+        if self.verbose:
+            mfu = 1000 * self.full_mfu / step_seconds
+            hfu = 1000 * self.full_hfu / step_seconds
+
+            def floored_percentage(index, val, digits):
+                val *= 10 ** (digits + 2)
+                return index + '{1:.{0}f}%'.format(digits, floor(val) / 10 ** digits)
+            train_log += floored_percentage(' mfu:', mfu, 2)
+            train_log += floored_percentage(' hfu:', hfu, 2)
+            print(train_log, flush=True)

mindspore/train/callback/_landscape.py

@@ -66,7 +66,7 @@ def nptype_to_prototype(np_value):
         np.uint32: 'DT_UINT32',
         np.uint64: 'DT_UINT64',
         np.float16: 'DT_FLOAT16',
-        np.float: 'DT_FLOAT64',
+        np.float_: 'DT_FLOAT64',
         np.float32: 'DT_FLOAT32',
         np.float64: 'DT_FLOAT64',
         None: 'DT_UNDEFINED'
@@ -170,7 +170,7 @@ class SummaryLandscape:
     It can create landscape in PCA direction or random direction by calculating loss.
 
     Note:
-        1. SummaryLandscape only supports Linux systems.
+        SummaryLandscape only supports Linux systems.
 
     Args:
         summary_dir (str): The path of summary is used to save the model weight,
@@ -186,10 +186,10 @@ class SummaryLandscape:
         ...     # If the device_target is Ascend, set the device_target to "Ascend"
         ...     ms.set_context(mode=ms.GRAPH_MODE, device_target="GPU")
         ...     # Create the dataset taking MNIST as an example. Refer to
-        ...     # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        ...     # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/mnist.py
         ...     ds_train = create_dataset()
         ...     # Define the network structure of LeNet5. Refer to
-        ...     # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        ...     # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         ...     network = LeNet5()
         ...     net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
         ...     net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9)
@@ -209,13 +209,13 @@ class SummaryLandscape:
         ...     # Simple usage for visualization landscape:
         ...     def callback_fn():
         ...         # Define the network structure of LeNet5. Refer to
-        ...         # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        ...         # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         ...         network = LeNet5()
         ...         net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
         ...         metrics = {"Loss": Loss()}
         ...         model = Model(network, net_loss, metrics=metrics)
         ...         # Create the dataset taking MNIST as an example. Refer to
-        ...         # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        ...         # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/mnist.py
         ...         ds_eval = create_dataset()
         ...         return model, network, ds_eval, metrics
         ...
@@ -253,7 +253,13 @@ class SummaryLandscape:
                 self._epoch_group[i].append(j)
 
     def clean_ckpt(self):
-        """Clean the checkpoint."""
+        """
+        Clean the checkpoint.
+
+        Tutorial Examples:
+            - `Training Optimization Process Visualization
+              <https://www.mindspore.cn/mindinsight/docs/en/master/landscape.html>`_
+        """
         shutil.rmtree(self._ckpt_dir, ignore_errors=True)
 
     def gen_landscapes_with_multi_process(self, callback_fn, collect_landscape=None,
@@ -280,7 +286,7 @@ class SummaryLandscape:
                   Default: ``40`` . Optional values: between 3 and 256.
                 - create_landscape (dict): Select how to create loss landscape.
                   Training process loss landscape(train) and training result loss landscape(result).
-                  Default: {"train": True, "result": True}. Optional: ``True`` / ``False`` .
+                  Default: ``{"train": True, "result": True}``. Optional: ``True`` / ``False`` .
                 - num_samples (int): The size of the dataset used to create the loss landscape.
                   For example, in image dataset, You can set num_samples is 2048,
                   which means that 2048 images are used to create loss landscape.
@@ -815,7 +821,7 @@ class SummaryLandscape:
             #device_id should be between 0 and 7.
             if i < 0 or i > 7:
                 raise ValueError(f'For "{self.__class__.__name__}.gen_landscapes_with_multi_process", the parameter '
-                                 f'"device_ids" should be between 0 and 7,but got {i}.')
+                                 f'"device_ids" should be between 0 and 7, but got {i}.')
 
     def _check_collect_landscape_data(self, collect_landscape):
         """Check collect landscape data type and value."""

mindspore/train/callback/_loss_monitor.py

@@ -19,7 +19,7 @@ import numpy as np
 
 from mindspore import _checkparam as Validator
 from mindspore.train.callback._callback import Callback, _handle_loss
-from mindspore._c_expression import _collect_host_info
+from mindspore._c_expression import collect_host_info, get_clock_syscnt
 
 
 class LossMonitor(Callback):
@@ -43,13 +43,13 @@ class LossMonitor(Callback):
         >>> from mindspore.train import Model, LossMonitor
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
         >>> optim = nn.Momentum(net.trainable_params(), 0.01, 0.9)
         >>> model = Model(net, loss_fn=loss, optimizer=optim)
         >>> # Create the dataset taking MNIST as an example. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/mnist.py
         >>> dataset = create_dataset()
         >>> loss_monitor = LossMonitor()
         >>> model.train(10, dataset, callbacks=loss_monitor)
@@ -70,7 +70,7 @@ class LossMonitor(Callback):
                     please refer to :class:`mindspore.train.RunContext`.
         """
         cb_params = run_context.original_args()
-        _collect_host_info("Callback", "LossMonitor", "step_end", level=1)
+        collect_host_info("Callback", "LossMonitor", "step_end", start_time=get_clock_syscnt(), level=1)
         cur_epoch_num = cb_params.get("cur_epoch_num", 1)
         loss = _handle_loss(cb_params.net_outputs)
 
@@ -101,7 +101,7 @@ class LossMonitor(Callback):
                     please refer to :class:`mindspore.train.RunContext`.
         """
         cb_params = run_context.original_args()
-        _collect_host_info("Callback", "LossMonitor", "train_epoch_end", level=1)
+        collect_host_info("Callback", "LossMonitor", "train_epoch_end", start_time=get_clock_syscnt(), level=1)
         metrics = cb_params.get("metrics")
         if metrics:
             print("Eval result: epoch %d, metrics: %s" % (cb_params.cur_epoch_num, metrics))