mindspore 2.2.11__cp37-cp37m-manylinux1_x86_64.whl → 2.3.0rc1__cp37-cp37m-manylinux1_x86_64.whl

mindspore/train/summary/_writer_pool.py

@@ -31,7 +31,7 @@ from mindspore.train.summary.writer import LineageWriter, SummaryWriter, ExportW
 
 try:
     from multiprocessing import get_context
-    ctx = get_context('forkserver')
+    ctx = get_context('fork')
 except ValueError:
     import multiprocessing as ctx
 

mindspore/train/summary/summary_record.py

@@ -1,4 +1,4 @@
-# Copyright 2020-2022 Huawei Technologies Co., Ltd
+# Copyright 2020-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,13 +21,13 @@ import re
 import threading
 import time
 from collections import defaultdict
-import numpy as np
 
 from mindspore import log as logger
 from mindspore.nn import Cell
 from mindspore import context
 from mindspore._c_expression import security
 from mindspore._c_expression import Tensor as Tensor_
+from mindspore.common import dtype as mstype
 from mindspore.common.tensor import Tensor
 from mindspore import _checkparam as Validator
 from mindspore.common.api import _cell_graph_executor
@@ -37,7 +37,8 @@ from mindspore.train.summary._summary_adapter import get_event_file_name, packag
 from mindspore.train.summary._writer_pool import WriterPool
 from mindspore.train.summary.enums import PluginEnum
 from mindspore.ops.operations import debug_ops
-import mindspore.ops as ops
+from mindspore.ops.primitive import _run_op
+from mindspore.ops._primitive_cache import _get_cache_prim
 
 # for the moment, this lock is for caution's sake,
 # there are actually no any concurrences happening.
@@ -61,19 +62,12 @@ def _cache_summary_tensor_data(summary):
     with _summary_lock:
         for item in summary:
             SUMMARY_TENSOR_CACHE[item['name']] = item['data']
-    return True
+        return True
 
 
-def _get_summary_tensor_data(end_flag=None, del_end_flag=False):
+def _get_summary_tensor_data():
     """Get summary tensor data."""
     global SUMMARY_TENSOR_CACHE
-    if end_flag:
-        for _ in range(0, 100):
-            if SUMMARY_TENSOR_CACHE.get(end_flag):
-                break
-            time.sleep(0.01)
-        if del_end_flag and SUMMARY_TENSOR_CACHE.get(end_flag):
-            del SUMMARY_TENSOR_CACHE[end_flag]
     with _summary_lock:
         data = SUMMARY_TENSOR_CACHE
         SUMMARY_TENSOR_CACHE = {}
@@ -98,7 +92,7 @@ def _record_summary_tensor_data():
             "data": data[2]
         }
         summary_list.append(summary_value)
-    _ = _cache_summary_tensor_data(summary_list)
+    _cache_summary_tensor_data(summary_list)
     debug_ops.SUMMARY_TENSOR_CACHE = []
 
 
@@ -136,17 +130,16 @@ class SummaryRecord:
     SummaryRecord is used to record the summary data and lineage data.
 
     The API will create a summary file and lineage files lazily in a given directory and writes data to them.
-    It writes the data to files by executing the 'record' method. In addition to recording the data bubbled up from
+    It writes the data to files by executing the `record` method. In addition to recording the data bubbled up from
     the network by defining the summary operators, SummaryRecord also supports to record extra data which
     can be added by calling add_value.
 
     Note:
-        1. When using SummaryRecord, you need to run the code in `if __name__ == "__main__"` .
-        2. Make sure to close the SummaryRecord at the end, otherwise the process will not exit.
+        1. Make sure to close the SummaryRecord at the end, otherwise the process will not exit.
            Please see the Example section below to learn how to close properly in two ways.
-        3. Only one SummaryRecord instance is allowed at a time, otherwise it will cause data writing problems.
-        4. SummaryRecord only supports Linux systems.
-        5. The Summary is not supported when compile source with `-s on` option.
+        2. Only one SummaryRecord instance is allowed at a time, otherwise it will cause data writing problems.
+        3. SummaryRecord only supports Linux systems.
+        4. The Summary is not supported when compile source with `-s on` option.
 
     Args:
         log_dir (str): The log_dir is a directory location to save the summary.
@@ -218,7 +211,7 @@ class SummaryRecord:
         self._num_process = num_process
         self.raise_exception = raise_exception
         self._export_options = export_options
-        self.tensor_summary = ops.TensorSummary()
+        self._op_finished_init = False
 
         try:
             self._initialize()
@@ -288,25 +281,25 @@ class SummaryRecord:
                 LossLandscape]): The value to store.
 
                 - The data type of value should be 'GraphProto' (see `mindspore/ccsrc/anf_ir.proto
-                  <https://gitee.com/mindspore/mindspore/blob/r2.2/mindspore/ccsrc/utils/anf_ir.proto>`_) object
+                  <https://gitee.com/mindspore/mindspore/blob/r2.3.q1/mindspore/ccsrc/utils/anf_ir.proto>`_) object
                   when the plugin is 'graph'.
                 - The data type of value should be 'Tensor' object when the plugin is 'scalar', 'image', 'tensor'
                   or 'histogram'.
                 - The data type of value should be a 'TrainLineage' object when the plugin is 'train_lineage',
                   see `mindspore/ccsrc/lineage.proto
-                  <https://gitee.com/mindspore/mindspore/blob/r2.2/mindspore/ccsrc/utils/lineage.proto>`_.
+                  <https://gitee.com/mindspore/mindspore/blob/r2.3.q1/mindspore/ccsrc/utils/lineage.proto>`_.
                 - The data type of value should be a 'EvaluationLineage' object when the plugin is 'eval_lineage',
                   see `mindspore/ccsrc/lineage.proto
-                  <https://gitee.com/mindspore/mindspore/blob/r2.2/mindspore/ccsrc/utils/lineage.proto>`_.
+                  <https://gitee.com/mindspore/mindspore/blob/r2.3.q1/mindspore/ccsrc/utils/lineage.proto>`_.
                 - The data type of value should be a 'DatasetGraph' object when the plugin is 'dataset_graph',
                   see `mindspore/ccsrc/lineage.proto
-                  <https://gitee.com/mindspore/mindspore/blob/r2.2/mindspore/ccsrc/utils/lineage.proto>`_.
+                  <https://gitee.com/mindspore/mindspore/blob/r2.3.q1/mindspore/ccsrc/utils/lineage.proto>`_.
                 - The data type of value should be a 'UserDefinedInfo' object when the plugin is 'custom_lineage_data',
                   see `mindspore/ccsrc/lineage.proto
-                  <https://gitee.com/mindspore/mindspore/blob/r2.2/mindspore/ccsrc/utils/lineage.proto>`_.
+                  <https://gitee.com/mindspore/mindspore/blob/r2.3.q1/mindspore/ccsrc/utils/lineage.proto>`_.
                 - The data type of value should be a 'LossLandscape' object when the plugin is 'LANDSCAPE',
                   see `mindspore/ccsrc/summary.proto
-                  <https://gitee.com/mindspore/mindspore/blob/r2.2/mindspore/ccsrc/utils/summary.proto>`_.
+                  <https://gitee.com/mindspore/mindspore/blob/r2.3.q1/mindspore/ccsrc/utils/summary.proto>`_.
 
         Raises:
             ValueError: `plugin` is not in the optional value.
@@ -352,6 +345,39 @@ class SummaryRecord:
                              f', expect value is one of [tensor, scalar, image, histogram, train_lineage, '
                              f'eval_lineage, dataset_graph, custom_lineage_data, graph, landscape]')
 
+    def _sync_summary_data(self, step):
+        """
+        In the GE process, the summary receiving process is asynchronous.
+        When the record function is called, some data may not be received and
+        needs to be synchronized.
+        """
+        if context.get_context('device_target') != "Ascend":
+            return
+        if self._op_finished_init is False:
+            _ = Cell()
+            self._op_finished_init = True
+
+        step_end_flag = Tensor([1], dtype=mstype.float32)
+        image_step_end_flag = Tensor([[[[1]]]], dtype=mstype.float32)
+        _ = Cell()
+        name = "_step_end_flag_" + str(step)
+        tags = ("[:Tensor]", "[:Scalar]", "[:Image]", "[:Histogram]")
+        _run_op(_get_cache_prim(debug_ops.TensorSummary)(), "TensorSummary", (name, step_end_flag))
+        _run_op(_get_cache_prim(debug_ops.ScalarSummary)(), "ScalarSummary", (name, step_end_flag))
+        _run_op(_get_cache_prim(debug_ops.HistogramSummary)(), "HistogramSummary", (name, step_end_flag))
+        _run_op(_get_cache_prim(debug_ops.ImageSummary)(), "ImageSummary", (name, image_step_end_flag))
+        global SUMMARY_TENSOR_CACHE
+        for tag in tags:
+            item_name = name + tag
+            while item_name not in SUMMARY_TENSOR_CACHE:
+                time.sleep(0.004)
+
+        with _summary_lock:
+            for tag in tags:
+                item_name = name + tag
+                SUMMARY_TENSOR_CACHE.pop(item_name, None)
+
+
     def record(self, step, train_network=None, plugin_filter=None):
         r"""
         Record the summary.
@@ -368,7 +394,7 @@ class SummaryRecord:
 
         Raises:
             TypeError: `step` is not int, or `train_network` is not `mindspore.nn.Cell
-                <https://www.mindspore.cn/docs/en/r2.2/api_python/nn/mindspore.nn.Cell.html#mindspore-nn-cell>`_ .
+                <https://www.mindspore.cn/docs/en/r2.3.q1/api_python/nn/mindspore.nn.Cell.html#mindspore-nn-cell>`_ .
 
         Examples:
             >>> import mindspore as ms
@@ -400,8 +426,6 @@ class SummaryRecord:
                     return True
 
         if self._mode == 'train':
-            step_end_flag = Tensor((np.ones([1])).astype(np.int32))
-            self.tensor_summary("step_end_flag_" + str(step), step_end_flag)
             self._add_summary_tensor_data(step)
 
         if not plugin_filter:
@@ -454,16 +478,14 @@ class SummaryRecord:
                                         self._num_process,
                                         self.raise_exception,
                                         **filename_dict)
-        _ = _get_summary_tensor_data()
+        _get_summary_tensor_data()
         atexit.register(self.close)
 
     def _add_summary_tensor_data(self, step_index=-1):
         """Add summary tensor data."""
+        self._sync_summary_data(step_index)
         _record_summary_tensor_data()
-        end_flag = None
-        if step_index >= 0:
-            end_flag = "step_end_flag_" + str(step_index) + "[:Tensor]"
-        summary_data = _get_summary_tensor_data(end_flag=end_flag, del_end_flag=True)
+        summary_data = _get_summary_tensor_data()
         if not summary_data:
             logger.debug(f'No summary data bubbled from the network.')
         for name, tensor in summary_data.items():

mindspore/train/train_thor/convert_utils.py

@@ -175,7 +175,7 @@ class ConvertNetUtils:
             >>> from mindspore.nn import thor
             >>>
             >>> # Define the network structure of LeNet5. Refer to
-            >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+            >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
             >>> net = LeNet5()
             >>> temp = Tensor([4e-4, 1e-4, 1e-5, 1e-5], ms.float32)
             >>> opt = thor(net, learning_rate=temp, damping=temp, momentum=0.9, loss_scale=128, frequency=4)
@@ -239,10 +239,10 @@ class ConvertModelUtils:
             >>> from mindspore.nn import thor
             >>>
             >>> # Define the network structure of LeNet5. Refer to
-            >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+            >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
             >>> net = LeNet5()
             >>> # Create the dataset taking MNIST as an example. Refer to
-            >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+            >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/mnist.py
             >>> dataset = create_dataset()
             >>> temp = Tensor([4e-4, 1e-4, 1e-5, 1e-5], ms.float32)
             >>> opt = thor(net, learning_rate=temp, damping=temp, momentum=0.9, loss_scale=128, frequency=4)

mindspore/version.py

@@ -1 +1 @@
-__version__ = '2.2.11'
+__version__ = '2.3.0rc1'

{mindspore-2.2.11.dist-info → mindspore-2.3.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mindspore
-Version: 2.2.11
+Version: 2.3.0rc1
 Summary: MindSpore is a new open source deep learning training/inference framework that could be used for mobile, edge and cloud scenarios.
 Home-page: https://www.mindspore.cn
 Download-URL: https://github.com/mindspore-ai/mindspore/tags
@@ -321,7 +321,7 @@ Project stable branches will be in one of the following states:
 | **r2.2**   | Maintained   | 2023-10-18               | Unmaintained <br> 2024-10-18 estimated |             |
 | **r2.1**   | Maintained   | 2023-07-29               | Unmaintained <br> 2024-07-29 estimated |             |
 | **r2.0**   | Maintained   | 2023-06-15               | Unmaintained <br> 2024-06-15 estimated |             |
-| **r1.10**  | Maintained   | 2023-02-02               | Unmaintained <br> 2024-02-02 estimated |             |
+| **r1.10**  | End Of Life  | 2023-02-02               |                                        | 2024-02-02  |
 | **r1.9**   | End Of Life  | 2022-10-26               |                                        | 2023-10-26  |
 | **r1.8**   | End Of Life  | 2022-07-29               |                                        | 2023-07-29  |
 | **r1.7**   | End Of Life  | 2022-04-29               |                                        | 2023-04-29  |