mindspore 2.3.0__cp39-none-any.whl → 2.3.0rc2__cp39-none-any.whl

mindspore/profiler/parser/ascend_integrate_generator.py

@@ -1,42 +0,0 @@
-# Copyright 2024 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""PROF csv data analyze module"""
-import os
-
-from mindspore.profiler.parser.ascend_analysis.file_manager import FileManager
-
-
-class AscendIntegrateGenerator:
-    """Generate ms profiler output csv"""
-
-    def __init__(self, mindstudio_profiler_output: str, ascend_ms_output_path: str):
-        self._mindstudio_profiler_output = mindstudio_profiler_output
-        self._ascend_ms_output_path = ascend_ms_output_path
-
-    def parse(self):
-        """Generate ms profiler output csv"""
-        self._generate_kernel_details()
-
-    def _generate_kernel_details(self):
-        """Generate kernel_details.csv"""
-        header_map = {
-            "Op Name": "Name", "OP Type": "Type", "Task Type": "Accelerator Core",
-            "Task Start Time(us)": "Start Time(us)", "Task Duration(us)": "Duration(us)",
-            "Task Wait Time(us)": "Wait Time(us)",
-        }
-        op_summary_file_list = FileManager.get_csv_file_list_by_start_name(self._mindstudio_profiler_output,
-                                                                           "op_summary")
-        kernel_details_file = os.path.join(self._ascend_ms_output_path, "kernel_details.csv")
-        FileManager.combine_csv_file(op_summary_file_list, kernel_details_file, header_map)

mindspore/profiler/parser/ascend_memory_generator.py

@@ -1,185 +0,0 @@
-# Copyright 2024 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""memory data analyze module"""
-import os
-
-from decimal import Decimal
-
-from mindspore.profiler.parser.ascend_analysis.file_manager import FileManager
-
-
-class AscendMemoryGenerator:
-    """Parsing the memory data of the ascend device"""
-    GE_MEMORY_RECORD_HEADERS = ["Device id", "Component", "Timestamp(us)", "Total Allocated(KB)",
-                                "Total Reserved(KB)", "Device"]
-    MS_MEMORY_RECORD_HEADERS = ["Timestamp(ns)", "Total Allocated(Byte)", "Total Reserved(Byte)", "Total Active(Byte)"]
-    TARGET_MEMORY_RECORD_HEADERS = ["Component", "Timestamp(us)", "Total Allocated(MB)", "Total Reserved(MB)",
-                                    "Total Active(MB)", "Device Type"]
-
-    def __init__(self, source_path: str, rank_id: int, mindstudio_profiler_output: str, ascend_ms_output_path: str):
-        self._source_path = source_path
-        self._rank_id = rank_id
-        self._mindstudio_profiler_output = mindstudio_profiler_output
-        self._ascend_ms_output_path = ascend_ms_output_path
-        self._ge_memory_record = []
-        self._ms_memory_record = []
-
-    def parse(self):
-        """Parse all memory data"""
-        self.copy_npu_module_mem_csv()
-        self.parse_memory_record()
-
-    def copy_npu_module_mem_csv(self):
-        """Generate npu_module_mem.csv"""
-        npu_module_mem_file_list = FileManager.get_csv_file_list_by_start_name(self._mindstudio_profiler_output,
-                                                                               "npu_module_mem")
-        target_file_path = os.path.join(self._ascend_ms_output_path, "npu_module_mem.csv")
-        FileManager.combine_csv_file(npu_module_mem_file_list, target_file_path)
-
-    def parse_memory_record(self):
-        """Generate memory_record.csv"""
-        self._parse_ge_memory_record()
-        self._parse_ms_memory_record()
-        combined_memory_data = self._combine_ge_ms_memory_record()
-        target_file_path = os.path.join(self._ascend_ms_output_path, "memory_record.csv")
-        FileManager.create_csv_file(target_file_path, combined_memory_data, self.TARGET_MEMORY_RECORD_HEADERS)
-
-    def _parse_ge_memory_record(self):
-        """Parse ge memory record data"""
-        memory_record_file_list = FileManager.get_csv_file_list_by_start_name(self._mindstudio_profiler_output,
-                                                                              "memory_record")
-        for file in memory_record_file_list:
-            data = FileManager.read_csv_file(file)
-            if len(data) > 1:
-                self._ge_memory_record.extend(data[1:])
-
-    def _parse_ms_memory_record(self):
-        """Parse mindspore memory record data"""
-        memory_record_file = os.path.join(self._source_path, f"cpu_ms_memory_record_{self._rank_id}.txt")
-        data = FileManager.read_txt_file(memory_record_file)
-        if len(data) > 1:
-            self._ms_memory_record.extend(data[1:])
-
-    def _get_app_reserved_memory(self) -> list:
-        """Get the reserved memory of the application from npu_mem.csv"""
-        npu_module_mem_file_list = FileManager.get_csv_file_list_by_start_name(self._mindstudio_profiler_output,
-                                                                               "npu_mem")
-        app_mems = []
-        for file in npu_module_mem_file_list:
-            md_mems = FileManager.read_csv_file(file)
-            for mem in md_mems:
-                if mem[1] == "APP":
-                    app_mems.append(MemoryRecordBean([mem[1], mem[-1].rstrip('\t'), 0.0, float(mem[4]), 0.0,
-                                                      f"NPU:{self._rank_id}"]).row)
-
-        return app_mems
-
-    def _combine_ge_ms_memory_record(self) -> list:
-        """Combine ge and mindspore memory record data"""
-        memory_records = []
-        for ge_memory in self._ge_memory_record:
-            memory_record = dict(zip(self.GE_MEMORY_RECORD_HEADERS, ge_memory))
-            memory_records.append(
-                MemoryRecordBean([memory_record.get("Component", "GE"), memory_record.get("Timestamp(us)"),
-                                  memory_record.get("Total Allocated(KB)", 0),
-                                  memory_record.get("Total Reserved(KB)", 0),
-                                  0, memory_record.get("Device")]))
-        for ms_memory in self._ms_memory_record:
-            memory_record = dict(zip(self.MS_MEMORY_RECORD_HEADERS, ms_memory))
-            memory_records.append(
-                MemoryRecordBean(["MindSpore", Decimal(memory_record.get("Timestamp(ns)", 0)) / 1000,
-                                  float(memory_record.get("Total Allocated(Byte)", 0)) / 1024,
-                                  float(memory_record.get("Total Reserved(Byte)", 0)) / 1024,
-                                  float(memory_record.get("Total Active(Byte)", 0)) / 1024,
-                                  f"NPU:{self._rank_id}"]))
-        memory_records.sort(key=lambda x: x.time_us)
-        last_ge_memory, last_ms_memory = MemoryRecordBean([0] * 6), MemoryRecordBean([0] * 6)
-        result_data = []
-        for memory_record in memory_records:
-            result_data.append(memory_record.row)
-            last_memory = last_ms_memory if memory_record.is_ge_component() else last_ge_memory
-            combined_mem = MemoryRecordBean(["MindSpore+GE", memory_record.time_us,
-                                             memory_record.total_allocated_kb + last_memory.total_allocated_kb,
-                                             memory_record.total_reserved_kb + last_memory.total_reserved_kb,
-                                             memory_record.total_active_kb + last_memory.total_active_kb,
-                                             f"NPU:{self._rank_id}"])
-            result_data.append(combined_mem.row)
-            if memory_record.is_ge_component():
-                last_ge_memory = memory_record
-            else:
-                last_ms_memory = memory_record
-        return result_data + self._get_app_reserved_memory()
-
-
-class MemoryRecordBean:
-    """Memory Record Struct"""
-    KB_TO_MB = 1000
-    HEADERS = ["Component", "Timestamp(us)", "Total Allocated(KB)", "Total Reserved(KB)",
-               "Total Active(KB)", "Device Type"]
-
-    def __init__(self, data: list):
-        self._data = dict(zip(self.HEADERS, data))
-
-    @property
-    def row(self) -> list:
-        return [self.component, self.time_us_str, self.total_allocated_mb, self.total_reserved_mb,
-                self.total_active_mb, self.device_type]
-
-    @property
-    def component(self) -> str:
-        return self._data.get("Component", "")
-
-    @property
-    def time_us_str(self) -> str:
-        ts_us = self._data.get("Timestamp(us)", 0)
-        return str(ts_us)
-
-    @property
-    def total_allocated_mb(self) -> float:
-        return float(self._data.get("Total Allocated(KB)", 0)) / self.KB_TO_MB
-
-    @property
-    def total_reserved_mb(self) -> float:
-        return float(self._data.get("Total Reserved(KB)", 0)) / self.KB_TO_MB
-
-    @property
-    def total_active_mb(self) -> float:
-        return float(self._data.get("Total Active(KB)", 0)) / self.KB_TO_MB
-
-    @property
-    def device_type(self) -> float:
-        return self._data.get("Device Type", "")
-
-    @property
-    def total_allocated_kb(self) -> float:
-        return float(self._data.get("Total Allocated(KB)", 0))
-
-    @property
-    def total_reserved_kb(self) -> float:
-        return float(self._data.get("Total Reserved(KB)", 0))
-
-    @property
-    def total_active_kb(self) -> float:
-        return float(self._data.get("Total Active(KB)", 0))
-
-    @property
-    def time_us(self) -> Decimal:
-        return Decimal(self._data.get("Timestamp(us)", 0))
-
-    def is_ge_component(self):
-        """
-        Determine if it is GE
-        """
-        return self.component == "GE"

mindspore/train/callback/_cluster_monitor.py

@@ -1,201 +0,0 @@
-# Copyright 2024 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""ClusterMonitor Callback class."""
-from __future__ import absolute_import
-
-import os
-import stat
-import glob
-import time
-from threading import RLock
-
-from mindspore.train.callback._callback import Callback
-from mindspore.communication.management import get_rank, get_local_rank
-from mindspore import log as logger
-from mindspore.parallel._auto_parallel_context import _get_auto_parallel_context
-from mindspore.parallel._utils import _get_device_num
-from mindspore.train._utils import get_parameter_redundancy
-
-_perf_mutex = RLock()
-
-
-def _get_dp_tp_from_redundancy(redundancy_tuple):
-    """From redundancy get dp and tp"""
-    dp = []
-    tp = []
-    for dp_value in redundancy_tuple:
-        dp.append(list(dp_value))
-    for i in range(len(redundancy_tuple[0])):
-        tp.append([v[i] for v in redundancy_tuple])
-    return dp, tp
-
-
-def _get_dp_tp_from_layout(parameter_layout_dict, initial_rank=0):
-    """From layout dict get dp and tp"""
-    tp = []
-    dp = []
-    parameter_redundancy_dict = get_parameter_redundancy(parameter_layout_dict, initial_rank)
-    value_len = 0
-    for _, value in parameter_redundancy_dict.items():
-        if len(value) > value_len:
-            value_len = len(value)
-            dp, tp = _get_dp_tp_from_redundancy(value)
-    return dp, tp
-
-
-def _check_perf_config(perf_config):
-    """Check if the format of perf_config is correct."""
-    enabled = perf_config.get("enable", None)
-    if enabled is None or not isinstance(enabled, bool):
-        raise TypeError(f"For cluster monitor, enabled should be bool, but got {type(enabled)}.")
-    enable_step_time = perf_config.get("steptime", None)
-    if enable_step_time is None or not isinstance(enable_step_time, bool):
-        raise TypeError(f"For cluster monitor, enable_step_time should be bool, but got {type(enable_step_time)}.")
-    enabled_dtp_group = perf_config.get("dtpGroup", None)
-    if enabled_dtp_group is None or not isinstance(enabled_dtp_group, bool):
-        raise TypeError(f"For cluster monitor, enabled_dtp_group should be bool, but got {type(enabled_dtp_group)}.")
-
-
-def _parse_perf_config():
-    """parse perf config"""
-    perf_config = os.getenv("PERF_DUMP_CONFIG")
-    perf_config_dict = {}
-    if perf_config is None:
-        return perf_config_dict
-    pairs = perf_config.split(',')
-    for pair in pairs:
-        key, value = pair.split(':')
-        if value.lower() == 'true':
-            perf_config_dict[key] = True
-        elif value.lower() == 'false':
-            perf_config_dict[key] = False
-        elif value.isdigit():
-            perf_config_dict[key] = int(value)
-        else:
-            perf_config_dict[key] = value
-    _check_perf_config(perf_config_dict)
-    return perf_config_dict
-
-
-def _remove_pre_log():
-    """Remove the previously saved log files."""
-    directory = os.getenv("PERF_DUMP_PATH")
-    device_id = get_local_rank()
-    pattern = os.path.join(directory, f"perf_ms_*_{device_id}.log")
-    files_to_delete = glob.glob(pattern)
-    for file_path in files_to_delete:
-        if os.path.islink(file_path):
-            continue
-        try:
-            os.remove(file_path)
-        except OSError as e:
-            logger.warning(f"When CCAE is opening, {file_path} need to be removed, but failed to remove.")
-            raise e
-
-
-class ClusterMonitor(Callback):
-    """
-    Monitor the cluster in train process.
-    """
-
-    def __init__(self):
-        super(ClusterMonitor, self).__init__()
-        self.perf_config = _parse_perf_config()
-        self.enabled = self.perf_config.get("enable")
-        self.enable_step_time = self.perf_config.get("steptime")
-        self.enabled_dtp_group = self.perf_config.get("dtpGroup")
-        self.data_time_start = None
-        self.data_time_end = None
-        self.frame_work = "MindSpore"
-        self.ms_sched_host = os.getenv("MS_SCHED_HOST", "127.0.0.1")
-        self.ms_sched_port = os.getenv("MS_SCHED_PORT", "8888")
-        self.uuid_value = self.ms_sched_host + "_" + self.ms_sched_port
-        self.global_rank = get_rank()
-        self.process_id = os.getpid()
-        self.device_id = get_local_rank()
-        self.log_name = "perf_ms" + "_" + str(self.process_id) + "_" + str(self.device_id) + ".log"
-        self.log_path = os.getenv("PERF_DUMP_PATH")
-        if not self.log_path.endswith(os.path.sep):
-            self.log_path += os.path.sep
-        self.full_path = self.log_path + self.log_name
-
-        self.write_dp_tp_flag = True
-        self.initial_rank = 0
-
-    def begin(self, run_context):
-        _remove_pre_log()
-        pp_num = _get_auto_parallel_context("pipeline_stages")
-        device_num = _get_device_num()
-
-        original_list = list(range(device_num))
-        chunk_size = device_num // pp_num
-        split_pp_lists = []
-        for i in range(0, device_num, chunk_size):
-            end_index = i + chunk_size if i + chunk_size <= device_num else device_num
-            split_pp_lists.append(original_list[i:end_index])
-
-        self.initial_rank = (self.global_rank // chunk_size) * chunk_size
-        with _perf_mutex:
-            dir_path = os.path.dirname(self.full_path)
-            if not os.path.exists(dir_path):
-                os.makedirs(dir_path)
-            if os.path.exists(self.full_path):
-                os.chmod(self.full_path, stat.S_IWUSR)
-                os.remove(self.full_path)
-            with open(self.full_path, 'w') as file:
-                log_message = f'UUID:{self.uuid_value}\nFRAMEWORK:{self.frame_work}\nGLOBAL RANKID:{self.global_rank}\n'
-                file.write(log_message)
-                for _, split_pp_list in enumerate(split_pp_lists):
-                    file.write(f'PP:{split_pp_list}\n')
-            os.chmod(self.full_path, stat.S_IRUSR)
-
-    def step_begin(self, run_context):
-        """
-        Record time at the beginning of step.
-
-        Args:
-            run_context (RunContext): Context of the process running. For more details,
-                    please refer to :class:`mindspore.train.RunContext`.
-        """
-        self.data_time_start = time.time()
-
-    def step_end(self, run_context):
-        """
-        Record time at the end of step.
-
-        Args:
-            run_context (RunContext): Context of the process running. For more details,
-                    please refer to :class:`mindspore.train.RunContext`.
-        """
-        self.data_time_end = time.time()
-        if self.enabled and self.enabled_dtp_group and self.write_dp_tp_flag:
-            cb_params = run_context.original_args()
-            param_layout_dict = cb_params.train_network.parameter_layout_dict
-            dp, tp = _get_dp_tp_from_layout(param_layout_dict, self.initial_rank)
-            with _perf_mutex:
-                os.chmod(self.full_path, stat.S_IWUSR)
-                with open(self.full_path, 'a') as file:
-                    for dp_value in dp:
-                        file.write(f'dp:{dp_value}\n')
-                    for tp_value in tp:
-                        file.write(f'tp:{tp_value}\n')
-                os.chmod(self.full_path, stat.S_IRUSR)
-            self.write_dp_tp_flag = False
-        if self.enabled and self.enable_step_time:
-            with _perf_mutex:
-                os.chmod(self.full_path, stat.S_IWUSR)
-                with open(self.full_path, 'a') as file:
-                    file.write(f"STEPTIME:{int(self.data_time_start * 1000)},{int(self.data_time_end * 1000)}\n")
-                os.chmod(self.full_path, stat.S_IRUSR)

mindspore/train/callback/_flops_collector.py

@@ -1,238 +0,0 @@
-# Copyright 2024 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""FLops Utilization collector Callback class."""
-from __future__ import absolute_import
-
-import time
-import os
-import stat
-import hashlib
-
-from math import floor
-from mindspore import _checkparam as Validator
-from mindspore import log as logger
-from mindspore.train.callback._callback import Callback
-from mindspore.common.api import flops_collection
-
-from mindspore.communication.management import (create_group, get_group_size,
-                                                get_rank)
-from mindspore.parallel._auto_parallel_context import auto_parallel_context
-from mindspore.ops import operations as P
-from mindspore.common import Tensor
-from mindspore import context
-import mindspore.nn as nn
-
-
-class AllReduceNet(nn.Cell):
-    """
-    Used to accumulate flops in pipeline parallel.
-    """
-    def __init__(self, group_name):
-        super(AllReduceNet, self).__init__()
-        self.allreduce_sum = P.AllReduce(op=P.ReduceOp.SUM, group=group_name)
-        self.add_flags(skip_auto_parallel_compile=True)
-
-    def construct(self, x):
-        return self.allreduce_sum(x)
-
-
-class FlopsUtilizationCollector(Callback):
-    """
-    The FlopsUtilizationCollector interface counts the model utilization information MFU
-    and the hardware utilization information HFU.
-    Currently, the API counts only the forward and backward flops of MatMul,
-    BatchMatMul, FlashAttentionScore, and Conv2D operators.
-    Only used in graph mode with static shape.
-
-    Args:
-        data_size (int): How many steps are the intervals between print information each time.
-        computility (int): The peak flops of each compute card. Default: ``1`` .
-        full_flops(bool): Whether to count the full model flops. If set full_flops to False,
-            FlopsUtilizationCollector would count the shard model flops in each device. Default: ``True`` .
-
-    Raises:
-        TypeError: If data_size is not positive int.
-        TypeError: If full_flops is not bool.
-
-    Examples:
-        >>> import numpy as np
-        >>> import mindspore.dataset as ds
-        >>> from mindspore import nn
-        >>> from mindspore.train import Model, FlopsUtilizationCollector
-        >>> from mindspore import context
-        >>> context.set_context(mode=context.GRAPH_MODE)
-        >>> data = {"x": np.float32(np.random.rand(64, 10)), "y": np.random.randint(0, 5, (64,))}
-        >>> train_dataset = ds.NumpySlicesDataset(data=data).batch(32)
-        >>> net = nn.Dense(10, 5)
-        >>> crit = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
-        >>> opt = nn.Momentum(net.trainable_params(), 0.01, 0.9)
-        >>> flops_callback = FlopsUtilizationCollector(train_dataset.get_dataset_size(), computility=10e6)
-        >>> model = Model(network=net, optimizer=opt, loss_fn=crit, metrics={"recall"})
-        >>> model.train(2, train_dataset, callbacks=[flops_callback])
-        Full model flops is 6400, Full hardware flops is 6400, Shard model flops is 6400, Shard hardware flops is 6400
-        Train per step time: 135.572 ms, mfu:0.47% hfu:0.47%
-        Train per step time: 1.317 ms, mfu:48.59% hfu:48.59%
-    """
-    def __init__(self, data_size, computility=1, full_flops=True):
-        super(FlopsUtilizationCollector, self).__init__()
-        self.step_time = time.time()
-        self.computility = computility
-        self.full_mfu = 0.0
-        self.full_hfu = 0.0
-        self.shard_mfu = 0.0
-        self.shard_hfu = 0.0
-        self.full_model_flops = 0.0
-        self.shard_model_flops = 0.0
-        self.full_hardware_flops = 0.0
-        self.shard_hardware_flops = 0.0
-        self.mfu_calculated = False
-        self.data_size = data_size
-        self.time_step_path = ''
-        self.verbose = full_flops
-        self.ma = os.environ.get("ENABLE_FLOPS_UTILIZATION_COLLECTOR") == "1"
-        Validator.check_bool(full_flops, "verbose")
-        Validator.check_positive_int(data_size, "data_size")
-
-    def step_begin(self, run_context):
-        """
-        Record time at the beginning of step.
-
-
-        Args:
-            run_context (RunContext): Context of the process running. For more details,
-                    please refer to :class:`mindspore.train.RunContext`.
-        """
-        self.step_time = time.time()
-
-    def _get_pipeline_group(self):
-        """
-        Calculate the communication group between all pipeline stages
-        """
-        rank = get_rank()
-        stage_nums = auto_parallel_context().get_pipeline_stages()
-        device_nums = get_group_size()
-        per_stage_device_nums = device_nums // stage_nums
-        local_stage_rank_id = rank % per_stage_device_nums
-        group = range(0, stage_nums)
-        rank_list = [local_stage_rank_id + x *
-                     per_stage_device_nums for x in group]
-        rank_str_list = [str(local_stage_rank_id + x *
-                             per_stage_device_nums) for x in group]
-        rank_list_str = "-".join(rank_str_list)
-        return rank_list, rank_list_str
-
-    def step_end(self, run_context):
-        """
-        Print mfu and hfu time at the end of step.
-
-        Args:
-           run_context (RunContext): Context of the process running. For more details,
-                   please refer to :class:`mindspore.train.RunContext`.
-        """
-        if context.get_context("mode") != context.GRAPH_MODE:
-            logger.warning("FlopsUtilizationCollector now only support graph mode.")
-            return
-
-        step_seconds = (time.time() - self.step_time) * 1000
-        if not self.mfu_calculated:
-            cb_params = run_context.original_args()
-            if cb_params.mode == 'train':
-                network = cb_params.train_network
-            elif cb_params.mode == 'eval':
-                network = cb_params.eval_network
-            else:
-                logger.warning('FlopsUtilizationCollector only support train and eval mode!')
-                return
-            full_model_flops, full_hardware_flops, shard_model_flops, \
-                shard_hardware_flops, is_dynamic_shape = flops_collection(network.current_phase)
-            if is_dynamic_shape:
-                logger.warning("FlopsUtilizationCollector now do not support dynamic shape.")
-                return
-            self.full_mfu = full_model_flops / self.computility
-            self.full_hfu = full_hardware_flops / self.computility
-
-            self.shard_mfu = shard_model_flops / self.computility
-            self.shard_hfu = shard_hardware_flops / self.computility
-            self.full_model_flops = full_model_flops
-            self.full_hardware_flops = full_hardware_flops
-            self.shard_model_flops = shard_model_flops
-            self.shard_hardware_flops = shard_hardware_flops
-            self.mfu_calculated = True
-            shard_mf_dir = os.path.realpath(os.getenv('MA_LOG_DIR', './'))
-            if self.ma:
-                flops_path = os.path.join(shard_mf_dir, "flops.txt")
-                self.time_step_path = os.path.join(shard_mf_dir, "time_step.txt")
-                if auto_parallel_context().get_parallel_mode() != "stand_alone":
-                    flops_path = os.path.join(
-                        shard_mf_dir, "flops_rank_" + str(get_rank())) + ".txt"
-                    self.time_step_path = os.path.join(
-                        shard_mf_dir, "time_step_rank_" + str(get_rank())) + ".txt"
-                time_stamp = time.time()
-                model_flops_log = "flops{{type=\"model_flops\"}} {} {}\n".\
-                    format(shard_model_flops, int(round(time_stamp * 1000)))
-                hardware_flops_log = "flops{{type=\"hardware_flops\"}} {} {}\n".\
-                    format(shard_hardware_flops, int(round(time_stamp * 1000)))
-                flags = os.O_WRONLY | os.O_CREAT
-                modes = stat.S_IWUSR | stat.S_IRUSR
-                with os.fdopen(os.open(flops_path, flags, modes), 'w') as f:
-                    f.write(model_flops_log)
-                    f.write(hardware_flops_log)
-            if self.verbose:
-                flops_log = f"Full model flops is {full_model_flops}, Full hardware flops is {full_hardware_flops}, " \
-                            f"Shard model flops is {shard_model_flops}, Shard hardware flops is {shard_hardware_flops}."
-                print(flops_log, flush=True)
-                if auto_parallel_context().get_pipeline_stages() > 1:
-                    pipeline_group_list, pipeline_group_name = self._get_pipeline_group()
-                    auto_parallel_context().set_pipeline_stages(1)
-                    hashed = hashlib.md5(
-                        pipeline_group_name.encode()).hexdigest()[:48]
-                    pipeline_group_name = str(hashed)
-                    create_group(pipeline_group_name, pipeline_group_list)
-                    self.full_mfu = AllReduceNet(pipeline_group_name)(
-                        Tensor([self.full_mfu])).asnumpy()[0]
-                    self.full_hfu = AllReduceNet(pipeline_group_name)(
-                        Tensor([self.full_hfu])).asnumpy()[0]
-                if auto_parallel_context().get_parallel_mode() != "stand_alone":
-                    self.full_mfu = self.full_mfu / get_group_size()
-                    self.full_hfu = self.full_hfu / get_group_size()
-
-        step_size = self.data_size
-        cb_params = run_context.original_args()
-        if hasattr(cb_params, "batch_num"):
-            batch_num = cb_params.batch_num
-            if isinstance(batch_num, int) and batch_num > 0:
-                step_size = cb_params.batch_num
-        Validator.check_positive_int(step_size)
-        if cb_params.dataset_sink_mode:
-            step_seconds = step_seconds / step_size
-        time_stamp = time.time()
-        train_log = "time_monitor{{type=\"per_step_time\"}} {} {}".format(step_seconds, int(round(time_stamp * 1000)))
-        if self.ma:
-            flags = os.O_WRONLY | os.O_CREAT
-            modes = stat.S_IWUSR | stat.S_IRUSR
-            with os.fdopen(os.open(self.time_step_path, flags, modes), 'w') as f:
-                f.write(train_log + '\n')
-        train_log = "{} per step time: {:5.3f} ms".format(
-            cb_params.mode.title(), step_seconds)
-        if self.verbose:
-            mfu = 1000 * self.full_mfu / step_seconds
-            hfu = 1000 * self.full_hfu / step_seconds
-
-            def floored_percentage(index, val, digits):
-                val *= 10 ** (digits + 2)
-                return index + '{1:.{0}f}%'.format(digits, floor(val) / 10 ** digits)
-            train_log += floored_percentage(' mfu:', mfu, 2)
-            train_log += floored_percentage(' hfu:', hfu, 2)
-            print(train_log, flush=True)