mindspore 2.2.11__cp38-none-any.whl → 2.2.14__cp38-none-any.whl

mindspore/profiler/parser/framework_parser.py

@@ -768,7 +768,7 @@ class DynamicFrameWorkParser:
         rank_id (int): The rank ID.
     """
 
-    def __init__(self, output_path, rank_id):
+    def __init__(self, output_path, rank_id, pretty=False):
         """Initialization of parsing framework data."""
         self._output_path = output_path
         self._all_op_exe_time = defaultdict(list)
@@ -779,6 +779,12 @@ class DynamicFrameWorkParser:
         self._exe_time_and_shape_detail = defaultdict(dict)
         self._dynamic_shape_info = defaultdict(list)
         self._step = 0
+        self._pretty = pretty
+
+    @property
+    def indent(self):
+        indent = 1 if self._pretty else None
+        return indent
 
     def write_dynamic_shape_data(self, df_op_summary):
         """Analyze dynamic shape data and write to dynamic shape file."""
@@ -804,7 +810,7 @@ class DynamicFrameWorkParser:
         self._dynamic_shape_info['op_type'] = self._op_info.get("op_type")
         dynamic_shape_file_path = os.path.join(self._output_path, output_dynamic_shape_file_name)
         with os.fdopen(os.open(dynamic_shape_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as fp:
-            json.dump(self._dynamic_shape_info, fp)
+            json.dump(self._dynamic_shape_info, fp, indent=self.indent)
         os.chmod(dynamic_shape_file_path, stat.S_IREAD | stat.S_IWRITE)
 
     def _analyse_op_execute_time(self, op_summary):

mindspore/profiler/parser/memory_usage_parser.py

@@ -40,7 +40,7 @@ GIGABYTES = 1024 * 1024 * 1024
 class MemoryUsageParser:
     """MemoryUsageParser to parse memory raw data."""
 
-    def __init__(self, profiling_dir, device_id):
+    def __init__(self, profiling_dir, device_id, pretty=False):
         self._profiling_dir = profiling_dir
         self._device_id = device_id
         self._proto_file_path = 'memory_usage_{}.pb'
@@ -57,6 +57,12 @@ class MemoryUsageParser:
         }
         self._framework = {}
         self._points = {}
+        self._pretty = pretty
+
+    @property
+    def indent(self):
+        indent = 1 if self._pretty else None
+        return indent
 
     @staticmethod
     def _process_framework_info(aicore_detail_data):
@@ -164,7 +170,7 @@ class MemoryUsageParser:
 
         try:
             with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as json_file:
-                json.dump(content, json_file)
+                json.dump(content, json_file, indent=self.indent)
             os.chmod(file_path, stat.S_IREAD | stat.S_IWRITE)
         except (IOError, OSError) as err:
             logger.critical('Fail to write memory file.\n%s', err)

mindspore/profiler/parser/minddata_analyzer.py

@@ -41,7 +41,7 @@ class MinddataProfilingAnalyzer:
         ProfilerFileNotFoundException: If any of the MindData profiling input files do not exist.
     """
 
-    def __init__(self, source_dir, device_id, output_path='./'):
+    def __init__(self, source_dir, device_id, output_path='./', pretty=False):
         # Validate and save input parameters
         self._device_id = device_id
         self._source_dir = self._validate_directory(source_dir, 'Source directory')
@@ -55,6 +55,12 @@ class MinddataProfilingAnalyzer:
 
         # Save output filename
         self._save_path = self._get_save_path(output_path)
+        self._pretty = pretty
+
+    @property
+    def indent(self):
+        indent = 1 if self._pretty else None
+        return indent
 
     @property
     def save_path(self):
@@ -624,7 +630,7 @@ class MinddataProfilingAnalyzer:
 
         # Save summary output dictionary to JSON output file (format#1)
         with os.fdopen(os.open(self._save_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as save_file:
-            json.dump(summary_dict, save_file)
+            json.dump(summary_dict, save_file, indent=self.indent)
 
         os.chmod(self._save_path, stat.S_IREAD | stat.S_IWRITE)
 

mindspore/profiler/parser/minddata_parser.py

@@ -110,7 +110,7 @@ class MinddataParser:
                         input_path=os.path.join(source_path, "data"), file_name='DATA_PREPROCESS.dev.AICPUMI')
                     if not minddata_aicpu_source_path:
                         return
-        minddata_aicpu_output_path = os.path.join(output_path, "minddata_aicpu_" + device_id + ".txt")
+        minddata_aicpu_output_path = os.path.join(output_path, "minddata_aicpu_" + str(device_id) + ".txt")
         minddata_aicpu_data = MinddataParser.parse_minddata_aicpu_data(minddata_aicpu_source_path)
         if minddata_aicpu_data:
             fwrite_format(minddata_aicpu_output_path, " ".join(col_names), is_start=True)

mindspore/profiler/parser/msadvisor_analyzer.py

@@ -29,10 +29,11 @@ class Msadvisor:
     """
     The interface to call MSAdvisor(CANN) by command line.
     """
-    def __init__(self, job_id, rank_id, output_path):
+    def __init__(self, job_id, rank_id, output_path, pretty=False):
         self._job_id, self._device_id = job_id.split("/")
         self._rank_id = rank_id
         self._output_path = output_path
+        self._pretty = pretty
 
     def call_msadvisor(self):
         """
@@ -75,6 +76,7 @@ class Msadvisor:
         """
         Execute the MSAdvisor parser, generate timeline file and call MSAdvisor by command line.
         """
-        reformater = MsadvisorParser(self._job_id, self._device_id, self._rank_id, self._output_path)
+        reformater = MsadvisorParser(self._job_id, self._device_id,
+                                     self._rank_id, self._output_path, pretty=self._pretty)
         reformater.parse()
         self.call_msadvisor()

mindspore/profiler/parser/msadvisor_parser.py

@@ -36,7 +36,7 @@ class MsadvisorParser:
     Data format conversion for MSAdvisor AICPU model.
     """
 
-    def __init__(self, job_id, device_id, rank_id, output_path):
+    def __init__(self, job_id, device_id, rank_id, output_path, pretty=False):
         self._job_id = job_id
         self._device_id = device_id
         self._rank_id = rank_id
@@ -45,6 +45,12 @@ class MsadvisorParser:
         self._aicpu_path = ""
         self._time_start = 0
         self._time_end = 0
+        self._pretty = pretty
+
+    @property
+    def indent(self):
+        indent = 1 if self._pretty else None
+        return indent
 
     @staticmethod
     def check_clear_make_dir(dir_path):
@@ -199,7 +205,7 @@ class MsadvisorParser:
                         break
                     if tid > 1:
                         output_file.write(",")
-                    json.dump(op, output_file)
+                    json.dump(op, output_file, indent=self.indent)
 
     def write_aicpu(self):
         """
@@ -221,7 +227,7 @@ class MsadvisorParser:
                     if op.get("ts") > self._time_end:
                         break
                     output_file.write(",")
-                    json.dump(op, output_file)
+                    json.dump(op, output_file, indent=self.indent)
             output_file.write("]")
 
     def parse(self):

mindspore/profiler/profiling.py

@@ -20,6 +20,8 @@ import json
 import glob
 import subprocess
 import csv
+import socket
+import shutil
 from enum import Enum
 import numpy as np
 
@@ -53,7 +55,9 @@ from mindspore.profiler.parser.ascend_fpbp_generator import AscendFPBPGenerator
 from mindspore.profiler.parser.ascend_op_generator import AscendOPGenerator
 from mindspore.profiler.parser.ascend_steptrace_generator import AscendStepTraceGenerator
 from mindspore.profiler.parser.ascend_flops_generator import AscendFlopsGenerator
+from mindspore.profiler.parser.ascend_cluster_generator import AscendClusterGenerator
 from mindspore.profiler.parser.ascend_hccl_generator import AscendHCCLGenerator, AscendHCCLGeneratorOld
+from mindspore.profiler.parser.ascend_communicate_generator import AscendCommunicationGenerator
 
 INIT_OP_NAME = 'Default/InitDataSetQueue'
 
@@ -460,6 +464,7 @@ class Profiler:
         self._dynamic_status = False
         self._profile_framework = "all"
         self._msprof_enable = os.getenv("PROFILER_SAMPLECONFIG")
+        self._pretty_json = False
         if self._msprof_enable:
             return
         self._start_time = int(time.time() * 1000000)
@@ -467,9 +472,7 @@ class Profiler:
         if kwargs.get("env_enable"):
             self._profiler_init(kwargs)
             return
-        if Profiler._has_initialized:
-            msg = "Do not init twice in the profiler."
-            raise RuntimeError(msg)
+
         Profiler._has_initialized = True
         # get device_id and device_target
         self._get_devid_rankid_and_devtarget()
@@ -527,6 +530,9 @@ class Profiler:
             dev_info = info_dict.get("DeviceInfo", [])
             dev_id = dev_info[0].get("id", -1)
 
+            if int(rank_id) < 0:
+                rank_id = 0
+
             return str(rank_id), str(dev_id)
 
     def op_analyse(self, op_name, device_id=None):
@@ -594,7 +600,7 @@ class Profiler:
                 return message
         return op_info
 
-    def analyse(self, offline_path=None):
+    def analyse(self, offline_path=None, pretty=False):
         """
         Collect and analyze training performance data, support calls during and after training. The example shows above.
 
@@ -602,7 +608,9 @@ class Profiler:
             offline_path (Union[str, None], optional): The data path which need to be analysed with offline mode.
                 Offline mode isused in abnormal exit scenario. This parameter should be set to ``None``
                 for online mode. Default: ``None``.
+            pretty (bool, optional): Whether to pretty json files. Default: ``False``.
         """
+        self._pretty_json = pretty
         self._analyse(offline_path=offline_path)
 
     def _analyse(self, offline_path=None, model_iteration_dict=None):
@@ -643,8 +651,6 @@ class Profiler:
         self._dynamic_status = self._profiler_manager.dynamic_status()
         _environment_check()
 
-        self._cpu_profiler.stop()
-
         cpu_op_file = glob.glob(os.path.join(self._output_path, 'cpu_op_type_info_*'))
         if self._device_target and self._device_target != DeviceTarget.CPU.value and cpu_op_file:
             self._is_heterogeneous = True
@@ -673,7 +679,6 @@ class Profiler:
 
         Raises:
             RuntimeError: If the profiler has already started.
-            RuntimeError: If MD profiling has stopped, repeated start action is not supported.
             RuntimeError: If the `start_profile` parameter is not set or is set to ``True``.
 
         Examples:
@@ -707,13 +712,8 @@ class Profiler:
         if not self._has_started:
             if not self._has_started_twice:
                 self._has_started = True
-                self._has_started_twice = True
-            else:
-                raise RuntimeError("MindSpore Profiling has finished, repeated start and stop actions are not "
-                                   "supported.")
         else:
-            raise RuntimeError("The profiler has already started. Use profiler.start() only when start_profile value "
-                               "is set to False.")
+            raise RuntimeError("The profiler has already started. Do not turn on again in the open state.")
 
         # No need to start anything if parse profiling data offline
         if self._is_offline_parser():
@@ -785,7 +785,8 @@ class Profiler:
         # Stop data collection after all operators are executed.
         _pynative_executor.sync()
 
-        if self._data_process:
+        self._cpu_profiler.stop()
+        if self._data_process and self._md_profiler is not None:
             self._md_profiler.stop()
             self._md_profiler.save(self._output_path)
 
@@ -962,9 +963,6 @@ class Profiler:
         if self._profile_communication:
             hccl_option = {"output": self._output_path, "task_trace": "on"}
             os.environ['PROFILING_OPTIONS'] = json.dumps(hccl_option)
-            if not self.start_profile:
-                raise RuntimeError(f"For '{self.__class__.__name__}', the parameter profile_communication can "
-                                   f"not be True while starting profiler in the process of training.")
 
         self._profile_memory = kwargs.pop("profile_memory", False)
         if not isinstance(self._profile_memory, bool):
@@ -1060,7 +1058,8 @@ class Profiler:
         # Analyze minddata information
         logger.info("Profiling: analyzing the minddata information.")
         try:
-            MinddataProfilingAnalyzer(self._output_path, store_id, self._output_path).analyze()
+            MinddataProfilingAnalyzer(self._output_path, store_id,
+                                      self._output_path, pretty=self._pretty_json).analyze()
         except ProfilerException as err:
             logger.warning(err.message)
         finally:
@@ -1080,7 +1079,7 @@ class Profiler:
 
             step_trace_point_info_path = validate_and_normalize_path(step_trace_point_info_path)
 
-            fpbp_analyse = AscendFPBPGenerator(op_summary, steptrace)
+            fpbp_analyse = AscendFPBPGenerator(op_summary, steptrace, pretty=self._pretty_json)
             points, _ = fpbp_analyse.parse()
             fpbp_analyse.write(step_trace_point_info_path)
         except ProfilerException as err:
@@ -1149,7 +1148,7 @@ class Profiler:
             logger.info("Profiling: analyzing the timeline data")
             timeline_analyser = AscendTimelineGenerator(self._output_path, self._dev_id, self._rank_id, self._rank_size,
                                                         context.get_context('mode'))
-            timeline_analyser.init_timeline(op_summary, steptrace)
+            timeline_analyser.init_timeline(op_summary, steptrace, pretty=self._pretty_json)
             timeline_analyser.write_timeline(self._timeline_size_limit_byte)
             timeline_analyser.write_timeline_summary()
         except (ProfilerIOException, ProfilerFileNotFoundException, RuntimeError) as err:
@@ -1166,7 +1165,7 @@ class Profiler:
             logger.warning("The profile_memory parameter cannot be set on the dynamic shape network.")
         logger.warning(
             "[Profiler]Dynamic Shape network does not support collecting step trace performance data currently.")
-        dynamic_parser = DynamicFrameWorkParser(self._output_path, self._rank_id)
+        dynamic_parser = DynamicFrameWorkParser(self._output_path, self._rank_id, pretty=self._pretty_json)
         dynamic_parser.write_dynamic_shape_data(op_summary)
 
     def _ascend_flops_analyse(self, op_summary):
@@ -1184,7 +1183,7 @@ class Profiler:
             flops_path = validate_and_normalize_path(flops_path)
             flops_summary_path = validate_and_normalize_path(flops_summary_path)
 
-            flops_analyser = AscendFlopsGenerator(op_summary)
+            flops_analyser = AscendFlopsGenerator(op_summary, pretty=self._pretty_json)
             flops_analyser.parse()
             flops_analyser.write(flops_path, flops_summary_path)
 
@@ -1208,6 +1207,73 @@ class Profiler:
         finally:
             pass
 
+    def _ascend_ms_analyze(self, source_path):
+        """Ascend ms generate"""
+        time_stamp = time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))
+        if self._rank_id:
+            ascend_ms_path = f"rank-{self._rank_id}_{time_stamp}_ascend_ms"
+        else:
+            ascend_ms_path = f"{socket.gethostname()}--{os.getpid()}_{time_stamp}_ascend_ms"
+        self._ascend_ms_path = os.path.join(self._output_path, ascend_ms_path)
+        if not os.path.exists(self._ascend_ms_path):
+            os.makedirs(self._ascend_ms_path, exist_ok=True)
+            os.chmod(self._ascend_ms_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+
+        dev_id = self._rank_id if self._device_target == DeviceTarget.ASCEND.value else self._dev_id
+        ascend_profiler_output_path = os.path.join(self._ascend_ms_path, 'ASCEND_PROFILER_OUTPUT')
+        os.makedirs(ascend_profiler_output_path, exist_ok=True)
+
+        source_profiler_info_path = os.path.join(self._output_path, f"profiler_info_{dev_id}.json")
+        target_profiler_info_path = os.path.join(self._ascend_ms_path, f"profiler_info_{dev_id}.json")
+        shutil.copy(source_profiler_info_path, target_profiler_info_path)
+
+        source_timeline_path = os.path.join(self._output_path, f"ascend_timeline_display_{dev_id}.json")
+        target_timeline_path = os.path.join(ascend_profiler_output_path, f"trace_view.json")
+        shutil.copy(source_timeline_path, target_timeline_path)
+
+        self._ascend_graph_cluster_analyse(source_path, ascend_profiler_output_path)
+        self._ascend_graph_communicate_analyse(source_path, ascend_profiler_output_path)
+
+    def _ascend_graph_cluster_analyse(self, source_path, ascend_profiler_output_path):
+        """Analyse step trace time info"""
+
+        try:
+            logger.info("Profiling: analyzing the step trace time profiler info.")
+
+            step_trace_time_path = os.path.join(ascend_profiler_output_path, f'step_trace_time.csv')
+            step_trace_time_path = validate_and_normalize_path(step_trace_time_path)
+
+            cluster_analyse = AscendClusterGenerator(os.path.join(source_path, 'timeline'))
+            cluster_analyse.parse()
+            cluster_analyse.write(step_trace_time_path)
+        except (ProfilerIOException, ProfilerFileNotFoundException, ProfilerRawFileException) as err:
+            logger.warning(err.message)
+        finally:
+            pass
+
+    def _ascend_graph_communicate_analyse(self, source_path, ascend_profiler_output_path):
+        """Analyse communicate info"""
+        if not self._profile_communication:
+            return
+
+        try:
+            logger.info("Profiling: analyzing the communicate and communicate_matrix profiler info.")
+
+            communication_file_path = os.path.join(ascend_profiler_output_path, f'communication.json')
+            communication_file_path = validate_and_normalize_path(communication_file_path)
+
+            communication_matrix_file_path = os.path.join(ascend_profiler_output_path, f"communication_matrix.json")
+            communication_matrix_file_path = validate_and_normalize_path(communication_matrix_file_path)
+
+            analyze_path = os.path.join(os.path.dirname(source_path), 'analyze')
+            communicate_analyser = AscendCommunicationGenerator(analyze_path)
+            communicate_analyser.parse()
+            communicate_analyser.write(communication_file_path, communication_matrix_file_path)
+        except (ProfilerIOException, ProfilerFileNotFoundException, ProfilerRawFileException) as err:
+            logger.warning(err.message)
+        finally:
+            pass
+
     def _ascend_graph_hccl_analyse(self, source_path, steptrace, flag):
         """Analyse hccl profiler info."""
         if not self._profile_communication:
@@ -1237,7 +1303,7 @@ class Profiler:
     def _ascend_graph_msadvisor_analyse(self, job_id):
         """Call MSAdvisor function."""
         logger.info("MSAdvisor starts running.")
-        msadvisor = Msadvisor(job_id, self._rank_id, self._output_path)
+        msadvisor = Msadvisor(job_id, self._rank_id, self._output_path, pretty=self._pretty_json)
         try:
             msadvisor.analyse()
         except FileNotFoundError as err:
@@ -1283,6 +1349,7 @@ class Profiler:
                 self._ascend_dynamic_net_analyse(op_summary)
             self._ascend_flops_analyse(op_summary)
             self._ascend_graph_memory_analyse(points)
+            self._ascend_ms_analyze(source_path)
             self._ascend_graph_hccl_analyse(source_path, steptrace, flag)
             self._ascend_graph_msadvisor_analyse(job_id)
             ProfilerInfo.set_graph_ids(graph_ids)
@@ -1368,11 +1435,15 @@ class Profiler:
 
     def _cpu_analyse(self):
         """Collect and analyse cpu performance data."""
+        if self._has_started:
+            self.stop()
+        else:
+            logger.info("No need to stop profiler because profiler has been stopped or profiler has not been started.")
         if not self._op_time:
             return
         try:
             timeline_generator = CpuTimelineGenerator(self._output_path, self._rank_id, context.get_context("mode"))
-            timeline_generator.init_timeline()
+            timeline_generator.init_timeline(pretty=self._pretty_json)
             timeline_generator.write_timeline(self._timeline_size_limit_byte)
             timeline_generator.write_timeline_summary()
         except (ProfilerIOException, ProfilerFileNotFoundException, RuntimeError) as err:
@@ -1462,7 +1533,7 @@ class Profiler:
         """Analyse memory usage data."""
         integrator = Integrator(self._output_path, self._rank_id)
         aicore_detail_data = integrator.get_aicore_detail_data()
-        memory_parser = MemoryUsageParser(self._output_path, self._rank_id)
+        memory_parser = MemoryUsageParser(self._output_path, self._rank_id, pretty=self._pretty_json)
         memory_parser.init_memory_usage_info(aicore_detail_data, points)
         memory_parser.write_memory_files()
 
@@ -1630,6 +1701,7 @@ class Profiler:
         else:
             output_path = kwargs.pop("output_path")
             self._output_path = validate_and_normalize_path(output_path)
+
         self._output_path = os.path.join(self._output_path, "profiler")
         if not os.path.exists(self._output_path):
             os.makedirs(self._output_path, exist_ok=True)

mindspore/rewrite/api/node.py

@@ -283,7 +283,7 @@ class Node:
             >>> dst_node = stree.get_node("relu_3")
             >>> dst_node.set_arg_by_node(0, src_node, 0)
             >>> print(dst_node.get_args())
-            [fc1]
+            [fc1_var]
         """
         Validator.check_value_type("arg_idx", arg_idx, [int], "Node")
         Validator.check_value_type("src_node", src_node, [Node], "Node")

mindspore/rewrite/api/symbol_tree.py

@@ -168,8 +168,8 @@ class SymbolTree:
             >>> net = LeNet5()
             >>> stree = SymbolTree.create(net)
             >>> print([node.get_name() for node in stree.nodes()])
-            ['input_x', 'Expr', 'conv1', 'relu', 'max_pool2d', 'conv2', 'relu_1', 'max_pool2d_1',
-             'flatten', 'fc1', 'relu_2', 'fc2', 'relu_3', 'fc3', 'return']
+            ['input_x', 'Expr', 'conv1', 'relu', 'max_pool2d', 'conv2', 'relu_1', 'max_pool2d_1', 'attribute_assign',
+             'unaryop_not', 'if_node', 'flatten', 'fc1', 'relu_2', 'fc2', 'relu_3', 'fc3', 'return_1']
         """
         Validator.check_value_type("all_nodes", all_nodes, [bool], "nodes")
         nodes = self._symbol_tree.all_nodes() if all_nodes else self._symbol_tree.nodes()

mindspore/train/callback/_checkpoint.py

@@ -205,7 +205,7 @@ class CheckpointConfig:
         Get the value of steps to save checkpoint.
 
         Returns:
-            Int, steps to save checkpoint.
+            int, steps to save checkpoint.
         """
         return self._save_checkpoint_steps
 
@@ -214,7 +214,7 @@ class CheckpointConfig:
         """Get the value of _save_checkpoint_seconds.
 
         Returns:
-            Int, seconds to save the checkpoint file.
+            int, seconds to save the checkpoint file.
         """
         return self._save_checkpoint_seconds
 
@@ -224,7 +224,7 @@ class CheckpointConfig:
         Get the value of maximum number of checkpoint files can be saved.
 
         Returns:
-            Int, Maximum number of checkpoint files can be saved.
+            int, Maximum number of checkpoint files can be saved.
         """
         return self._keep_checkpoint_max
 
@@ -244,7 +244,7 @@ class CheckpointConfig:
         Get the value of whether to merge and save the split Tensor in the automatic parallel scenario.
 
         Returns:
-            Bool, whether to merge and save the split Tensor in the automatic parallel scenario.
+            bool, whether to merge and save the split Tensor in the automatic parallel scenario.
         """
         return self._integrated_save
 
@@ -254,7 +254,7 @@ class CheckpointConfig:
         Get the value of whether asynchronous execution saves the checkpoint to a file.
 
         Returns:
-            Bool, whether asynchronous execution saves the checkpoint to a file.
+            bool, whether asynchronous execution saves the checkpoint to a file.
         """
         return self._async_save
 
@@ -294,7 +294,7 @@ class CheckpointConfig:
         Get the value of information dict saved to checkpoint file.
 
         Returns:
-            Dict, the information saved to checkpoint file.
+            dict, the information saved to checkpoint file.
         """
         return self._append_dict
 
@@ -304,7 +304,7 @@ class CheckpointConfig:
         Get the value of whether to save map Parameter incrementally.
 
         Returns:
-            Bool, whether to save map Parameter incrementally.
+            bool, whether to save map Parameter incrementally.
         """
         return self._map_param_inc
 
@@ -313,7 +313,7 @@ class CheckpointConfig:
         Get the policy of checkpoint.
 
         Returns:
-            Dict, the information of checkpoint policy.
+            dict, the information of checkpoint policy.
         """
         checkpoint_policy = {'save_checkpoint_steps': self.save_checkpoint_steps,
                              'save_checkpoint_seconds': self.save_checkpoint_seconds,

mindspore/train/callback/_landscape.py

@@ -1,4 +1,4 @@
-# Copyright 2021-2022 Huawei Technologies Co., Ltd
+# Copyright 2021-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -170,8 +170,7 @@ class SummaryLandscape:
     It can create landscape in PCA direction or random direction by calculating loss.
 
     Note:
-        1. When using SummaryLandscape, you need to run the code in `if __name__ == "__main__"` .
-        2. SummaryLandscape only supports Linux systems.
+        1. SummaryLandscape only supports Linux systems.
 
     Args:
         summary_dir (str): The path of summary is used to save the model weight,

mindspore/train/callback/_summary_collector.py

@@ -1,4 +1,4 @@
-# Copyright 2020-2022 Huawei Technologies Co., Ltd
+# Copyright 2020-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -77,12 +77,11 @@ class SummaryCollector(Callback):
     SummaryCollector also enables the summary operator to collect data to summary files.
 
     Note:
-        1. When using SummaryCollector, you need to run the code in `if __name__ == "__main__"` .
-        2. Multiple SummaryCollector instances in callback list are not allowed.
-        3. Not all information is collected at the training phase or at the eval phase.
-        4. SummaryCollector always record the data collected by the summary operator.
-        5. SummaryCollector only supports Linux systems.
-        6. The Summary is not supported when compile source with `-s on` option.
+        1. Multiple SummaryCollector instances in callback list are not allowed.
+        2. Not all information is collected at the training phase or at the eval phase.
+        3. SummaryCollector always record the data collected by the summary operator.
+        4. SummaryCollector only supports Linux systems.
+        5. The Summary is not supported when compile source with `-s on` option.
 
     Args:
         summary_dir (str): The collected data will be persisted to this directory.

mindspore/train/dataset_helper.py

@@ -444,6 +444,11 @@ class DatasetHelper:
         # Generally, it works in dynamic shape scenarios.
         return self.iter.get_data_info()
 
+    # pylint: disable=missing-docstring
+    def get_mbuf_queue_size(self):
+        # In sink mode, it returns the element numbers inside mbuf channel.
+        return self.iter.get_mbuf_queue_size()
+
     # pylint: disable=missing-docstring
     def get_send_info(self, run_context):
         # In sink mode, it returns the send information of dataset at this moment.
@@ -526,6 +531,7 @@ class _DatasetIter:
         self.release = dataset.__transfer_dataset__.release
         self.continue_send = dataset.__transfer_dataset__.continue_send
         self.get_data_info = dataset.__transfer_dataset__.get_data_info
+        self.get_mbuf_queue_size = dataset.__transfer_dataset__.get_mbuf_queue_size
         self.get_send_info = dataset.__transfer_dataset__.get_send_info
         if hasattr(dataset.__transfer_dataset__, "_reset"):
             self._reset = dataset.__transfer_dataset__._reset  # pylint: disable=protected-access