mindspore 2.1.0__cp37-cp37m-win_amd64.whl → 2.2.11__cp37-cp37m-win_amd64.whl

mindspore/train/summary/_summary_adapter.py

@@ -392,7 +392,7 @@ def _fill_image_summary(tag: str, np_value, summary_image, input_format='NCHW'):
     else:
         if max_value != min_value:
             # Mapping the value to range [0, 255] linearly.
-            scale_factor = 255/(max_value - min_value + 1)
+            scale_factor = 255 / (max_value - min_value + 1)
         shift = min_value
     tensor = tensor.astype(np.float32)
     tensor = ((tensor - shift) * scale_factor).astype(np.uint8)

mindspore/train/summary/_writer_pool.py

@@ -73,12 +73,13 @@ class WriterPool(ctx.Process):
         filedict (dict): The mapping from plugin to filename.
     """
 
-    def __init__(self, base_dir, max_file_size, raise_exception=False, **filedict) -> None:
+    def __init__(self, base_dir, max_file_size, num_process=32, raise_exception=False, **filedict) -> None:
         super().__init__()
         self._base_dir, self._filedict = base_dir, filedict
         self._queue, self._writers_ = ctx.Queue(ctx.cpu_count() * 2), None
         self._max_file_size = max_file_size
         self._raise_exception = raise_exception
+        self._num_process = num_process
         self._training_pid = os.getpid()
         self.start()
 
@@ -97,7 +98,7 @@ class WriterPool(ctx.Process):
         # which causes the main process to fail to exit.
         signal.signal(signal.SIGINT, signal.SIG_IGN)
 
-        with ctx.Pool(min(ctx.cpu_count(), 32)) as pool:
+        with ctx.Pool(min(ctx.cpu_count(), self._num_process)) as pool:
             deq = deque()
             while True:
                 if self._check_heartbeat():

mindspore/train/summary/summary_record.py

@@ -21,6 +21,7 @@ import re
 import threading
 import time
 from collections import defaultdict
+import numpy as np
 
 from mindspore import log as logger
 from mindspore.nn import Cell
@@ -36,6 +37,7 @@ from mindspore.train.summary._summary_adapter import get_event_file_name, packag
 from mindspore.train.summary._writer_pool import WriterPool
 from mindspore.train.summary.enums import PluginEnum
 from mindspore.ops.operations import debug_ops
+import mindspore.ops as ops
 
 # for the moment, this lock is for caution's sake,
 # there are actually no any concurrences happening.
@@ -59,12 +61,19 @@ def _cache_summary_tensor_data(summary):
     with _summary_lock:
         for item in summary:
             SUMMARY_TENSOR_CACHE[item['name']] = item['data']
-        return True
+    return True
 
 
-def _get_summary_tensor_data():
+def _get_summary_tensor_data(end_flag=None, del_end_flag=False):
     """Get summary tensor data."""
     global SUMMARY_TENSOR_CACHE
+    if end_flag:
+        for _ in range(0, 100):
+            if SUMMARY_TENSOR_CACHE.get(end_flag):
+                break
+            time.sleep(0.01)
+        if del_end_flag and SUMMARY_TENSOR_CACHE.get(end_flag):
+            del SUMMARY_TENSOR_CACHE[end_flag]
     with _summary_lock:
         data = SUMMARY_TENSOR_CACHE
         SUMMARY_TENSOR_CACHE = {}
@@ -89,7 +98,7 @@ def _record_summary_tensor_data():
             "data": data[2]
         }
         summary_list.append(summary_value)
-    _cache_summary_tensor_data(summary_list)
+    _ = _cache_summary_tensor_data(summary_list)
     debug_ops.SUMMARY_TENSOR_CACHE = []
 
 
@@ -143,6 +152,8 @@ class SummaryRecord:
         log_dir (str): The log_dir is a directory location to save the summary.
         file_prefix (str): The prefix of file. Default: ``"events"`` .
         file_suffix (str): The suffix of file. Default: ``"_MS"`` .
+        num_process (int): Number of processes saving summary data. The more processes there are, the better the
+            performance, but there may be host memory overflow issues. Default: ``32`` .
         network (Cell): Obtain a pipeline through network for saving graph summary. Default: ``None`` .
         max_file_size (int, optional): The maximum size of each file that can be written to disk (in bytes).
             For example, to write not larger than 4GB, specify `max_file_size=4*1024**3`.
@@ -181,7 +192,7 @@ class SummaryRecord:
 
     count = 0
 
-    def __init__(self, log_dir, file_prefix="events", file_suffix="_MS",
+    def __init__(self, log_dir, file_prefix="events", file_suffix="_MS", num_process=32,
                  network=None, max_file_size=None, raise_exception=False, export_options=None):
         self._check_count()
         with _instance_lock:
@@ -204,8 +215,11 @@ class SummaryRecord:
         self.file_suffix = file_suffix
         self.network = network
         self.max_file_size = max_file_size
+        self._num_process = num_process
         self.raise_exception = raise_exception
         self._export_options = export_options
+        self.tensor_summary = ops.TensorSummary()
+
         try:
             self._initialize()
         except (TypeError, ValueError) as err:
@@ -231,8 +245,8 @@ class SummaryRecord:
             mode (str): The mode to be set, which should be 'train' or 'eval'. When the mode is 'eval',
                 summary_record will not record the data of summary operators.
 
-                - train：the model running phase is train mode.
-                - eval：the model running phase is eval mode，When the mode is 'eval',
+                - train: the model running phase is train mode.
+                - eval: the model running phase is eval mode, When the mode is 'eval',
                   summary_record will not record the data of summary operators.
 
         Raises:
@@ -274,25 +288,25 @@ class SummaryRecord:
                 LossLandscape]): The value to store.
 
                 - The data type of value should be 'GraphProto' (see `mindspore/ccsrc/anf_ir.proto
-                  <https://gitee.com/mindspore/mindspore/blob/r2.1/mindspore/ccsrc/utils/anf_ir.proto>`_) object
+                  <https://gitee.com/mindspore/mindspore/blob/r2.2/mindspore/ccsrc/utils/anf_ir.proto>`_) object
                   when the plugin is 'graph'.
                 - The data type of value should be 'Tensor' object when the plugin is 'scalar', 'image', 'tensor'
                   or 'histogram'.
                 - The data type of value should be a 'TrainLineage' object when the plugin is 'train_lineage',
                   see `mindspore/ccsrc/lineage.proto
-                  <https://gitee.com/mindspore/mindspore/blob/r2.1/mindspore/ccsrc/utils/lineage.proto>`_.
+                  <https://gitee.com/mindspore/mindspore/blob/r2.2/mindspore/ccsrc/utils/lineage.proto>`_.
                 - The data type of value should be a 'EvaluationLineage' object when the plugin is 'eval_lineage',
                   see `mindspore/ccsrc/lineage.proto
-                  <https://gitee.com/mindspore/mindspore/blob/r2.1/mindspore/ccsrc/utils/lineage.proto>`_.
+                  <https://gitee.com/mindspore/mindspore/blob/r2.2/mindspore/ccsrc/utils/lineage.proto>`_.
                 - The data type of value should be a 'DatasetGraph' object when the plugin is 'dataset_graph',
                   see `mindspore/ccsrc/lineage.proto
-                  <https://gitee.com/mindspore/mindspore/blob/r2.1/mindspore/ccsrc/utils/lineage.proto>`_.
+                  <https://gitee.com/mindspore/mindspore/blob/r2.2/mindspore/ccsrc/utils/lineage.proto>`_.
                 - The data type of value should be a 'UserDefinedInfo' object when the plugin is 'custom_lineage_data',
                   see `mindspore/ccsrc/lineage.proto
-                  <https://gitee.com/mindspore/mindspore/blob/r2.1/mindspore/ccsrc/utils/lineage.proto>`_.
+                  <https://gitee.com/mindspore/mindspore/blob/r2.2/mindspore/ccsrc/utils/lineage.proto>`_.
                 - The data type of value should be a 'LossLandscape' object when the plugin is 'LANDSCAPE',
                   see `mindspore/ccsrc/summary.proto
-                  <https://gitee.com/mindspore/mindspore/blob/r2.1/mindspore/ccsrc/utils/summary.proto>`_.
+                  <https://gitee.com/mindspore/mindspore/blob/r2.2/mindspore/ccsrc/utils/summary.proto>`_.
 
         Raises:
             ValueError: `plugin` is not in the optional value.
@@ -354,7 +368,7 @@ class SummaryRecord:
 
         Raises:
             TypeError: `step` is not int, or `train_network` is not `mindspore.nn.Cell
-                <https://www.mindspore.cn/docs/en/r2.1/api_python/nn/mindspore.nn.Cell.html#mindspore-nn-cell>`_ 。
+                <https://www.mindspore.cn/docs/en/r2.2/api_python/nn/mindspore.nn.Cell.html#mindspore-nn-cell>`_ .
 
         Examples:
             >>> import mindspore as ms
@@ -386,7 +400,9 @@ class SummaryRecord:
                     return True
 
         if self._mode == 'train':
-            self._add_summary_tensor_data()
+            step_end_flag = Tensor((np.ones([1])).astype(np.int32))
+            self.tensor_summary("step_end_flag_" + str(step), step_end_flag)
+            self._add_summary_tensor_data(step)
 
         if not plugin_filter:
             self._event_writer.write(self._consume_data_pool(step))
@@ -435,15 +451,19 @@ class SummaryRecord:
                              exporter=export_dir)
         self._event_writer = WriterPool(self.base_log_dir,
                                         self.max_file_size,
+                                        self._num_process,
                                         self.raise_exception,
                                         **filename_dict)
-        _get_summary_tensor_data()
+        _ = _get_summary_tensor_data()
         atexit.register(self.close)
 
-    def _add_summary_tensor_data(self):
+    def _add_summary_tensor_data(self, step_index=-1):
         """Add summary tensor data."""
         _record_summary_tensor_data()
-        summary_data = _get_summary_tensor_data()
+        end_flag = None
+        if step_index >= 0:
+            end_flag = "step_end_flag_" + str(step_index) + "[:Tensor]"
+        summary_data = _get_summary_tensor_data(end_flag=end_flag, del_end_flag=True)
         if not summary_data:
             logger.debug(f'No summary data bubbled from the network.')
         for name, tensor in summary_data.items():

mindspore/train/train_thor/convert_utils.py

@@ -175,7 +175,7 @@ class ConvertNetUtils:
             >>> from mindspore.nn import thor
             >>>
             >>> # Define the network structure of LeNet5. Refer to
-            >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+            >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
             >>> net = LeNet5()
             >>> temp = Tensor([4e-4, 1e-4, 1e-5, 1e-5], ms.float32)
             >>> opt = thor(net, learning_rate=temp, damping=temp, momentum=0.9, loss_scale=128, frequency=4)
@@ -239,10 +239,10 @@ class ConvertModelUtils:
             >>> from mindspore.nn import thor
             >>>
             >>> # Define the network structure of LeNet5. Refer to
-            >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+            >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
             >>> net = LeNet5()
             >>> # Create the dataset taking MNIST as an example. Refer to
-            >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/mnist.py
+            >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
             >>> dataset = create_dataset()
             >>> temp = Tensor([4e-4, 1e-4, 1e-5, 1e-5], ms.float32)
             >>> opt = thor(net, learning_rate=temp, damping=temp, momentum=0.9, loss_scale=128, frequency=4)

mindspore/train/train_thor/dataset_helper.py

@@ -147,7 +147,7 @@ class _DatasetIter:
         if hasattr(self.dataset, '__loop_size__'):
             sink_size = self.dataset.__loop_size__
         else:
-            if context.get_context("enable_ge") or context.get_context("device_target") == "Ascend":
+            if context.get_context("device_target") == "Ascend":
                 if self.sink_size > 0:
                     sink_size = self.sink_size
                 else:

mindspore/version.py

@@ -1 +1 @@
-__version__ = '2.1.0'
+__version__ = '2.2.11'

{mindspore-2.1.0.dist-info → mindspore-2.2.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mindspore
-Version: 2.1.0
+Version: 2.2.11
 Summary: MindSpore is a new open source deep learning training/inference framework that could be used for mobile, edge and cloud scenarios.
 Home-page: https://www.mindspore.cn
 Author: The MindSpore Authors
@@ -40,7 +40,7 @@ Requires-Dist: astunparse (>=1.6.3)
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/mindspore.svg)](https://pypi.org/project/mindspore)
 [![PyPI](https://badge.fury.io/py/mindspore.svg)](https://badge.fury.io/py/mindspore)
-[![Downloads](https://pepy.tech/badge/mindspore)](https://pepy.tech/project/mindspore)
+[![Downloads](https://static.pepy.tech/badge/mindspore)](https://pepy.tech/project/mindspore)
 [![DockerHub](https://img.shields.io/docker/pulls/mindspore/mindspore-cpu.svg)](https://hub.docker.com/r/mindspore/mindspore-cpu)
 [![LICENSE](https://img.shields.io/github/license/mindspore-ai/mindspore.svg?style=flat-square)](https://github.com/mindspore-ai/mindspore/blob/master/LICENSE)
 [![Slack](https://img.shields.io/badge/slack-chat-green.svg?logo=slack)](https://join.slack.com/t/mindspore/shared_invite/zt-dgk65rli-3ex4xvS4wHX7UDmsQmfu8w)
@@ -319,9 +319,12 @@ Project stable branches will be in one of the following states:
 
 | **Branch** | **Status**   | **Initial Release Date** | **Next Phase**                         | **EOL Date**|
 |------------|--------------|--------------------------|----------------------------------------|-------------|
+| **r2.2**   | Maintained   | 2023-10-18               | Unmaintained <br> 2024-10-18 estimated |             |
+| **r2.1**   | Maintained   | 2023-07-29               | Unmaintained <br> 2024-07-29 estimated |             |
+| **r2.0**   | Maintained   | 2023-06-15               | Unmaintained <br> 2024-06-15 estimated |             |
 | **r1.10**  | Maintained   | 2023-02-02               | Unmaintained <br> 2024-02-02 estimated |             |
-| **r1.9**   | Maintained   | 2022-10-26               | Unmaintained <br> 2023-10-26 estimated |             |
-| **r1.8**   | Maintained   | 2022-07-29               | Unmaintained <br> 2023-07-29 estimated |             |
+| **r1.9**   | End Of Life  | 2022-10-26               |                                        | 2023-10-26  |
+| **r1.8**   | End Of Life  | 2022-07-29               |                                        | 2023-07-29  |
 | **r1.7**   | End Of Life  | 2022-04-29               |                                        | 2023-04-29  |
 | **r1.6**   | End Of Life  | 2022-01-29               |                                        | 2023-01-29  |
 | **r1.5**   | End Of Life  | 2021-10-15               |                                        | 2022-10-15  |