mindspore 2.1.0__cp38-cp38-manylinux1_x86_64.whl → 2.2.11__cp38-cp38-manylinux1_x86_64.whl

mindspore/dataset/engine/datasets_user_defined.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2022 Huawei Technologies Co., Ltd
+# Copyright 2019-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -48,6 +48,7 @@ from ..core.config import get_enable_shared_mem, get_prefetch_size, get_multipro
     get_enable_watchdog, get_debug_mode
 from ..core.datatypes import mstypelist_to_detypelist
 from ..core.py_util_helpers import ExceptionHandler
+from ..transforms import transforms
 
 
 def _iter_fn(dataset, num_samples):
@@ -128,13 +129,28 @@ def _fill_worker_indices(workers, indices, idx):
     return idx
 
 
+def _fill_worker_quit_flag(workers, worker_to_quit):
+    """
+    Worker index queue filler, fill worker index queue with QUIT flag.
+    """
+    num_worker = len(workers)
+    for i in range(num_worker):
+        # just put only one QUIT flag to the sub-thread / sub-process
+        if str(i) not in worker_to_quit:
+            try:
+                workers[i].put("QUIT")
+                worker_to_quit[str(i)] = "QUIT"
+            except queue.Full:
+                continue
+
+
 def _convert_row(row):
     """
     Convert Op return value to numpy, or keep as a dict (if already a dict)
     """
 
     # convert single item to np.array
-    prim_type = (int, float, str, bytes, np.ndarray, Tensor)
+    prim_type = (int, float, str, bytes, np.ndarray, Tensor, np.number, np.bool_)
     if isinstance(row, prim_type):
         if isinstance(row, Tensor):  # mindspore.Tensor
             item = row.asnumpy()
@@ -200,7 +216,10 @@ class SamplerFn:
         queue_size = max(2, queue_size)
 
         if multi_process and get_enable_shared_mem():
-            _check_shm_usage(num_worker, queue_size, max_rowsize)
+            # generator dataset use idx_queue and res_queue to transfer data between main and subprocess
+            # idx_queue is used multiprocess.Queue which is not shared memory, so it's size is 0.
+            # res_queue is used shared memory, so it' size is max_rowsize which is defined by user.
+            _check_shm_usage(num_worker, queue_size, 0, max_rowsize)
         self.count = multiprocessing.Value('i', 0)
         for _ in range(num_worker):
             if multi_process is True:
@@ -237,7 +256,11 @@ class SamplerFn:
         for w in self.workers:
             # Check whether the queue of the subprocess is empty.
             if not w.queue_empty():
-                raise Exception("The queue of the subprocess is not empty.")
+                # in failover reset scenario the QUIT flag should be pop first
+                while w.idx_queue.qsize() > 0:
+                    result = w.idx_queue.get()
+                    if result != "QUIT":
+                        raise Exception("The queue of the subprocess is not empty.")
             # Start all workers
             if not w.is_alive():
                 w.start()
@@ -246,6 +269,9 @@ class SamplerFn:
         idx_cursor = 0
         idx_cursor = _fill_worker_indices(self.workers, indices, idx_cursor)
 
+        # worker to quit
+        worker_to_quit = {}
+
         # Fetch results
         for i in range(len(indices)):
             if self.eof.is_set():
@@ -284,6 +310,9 @@ class SamplerFn:
                 return
             if idx_cursor < len(indices):
                 idx_cursor = _fill_worker_indices(self.workers, indices, idx_cursor)
+            else:
+                # send QUIT flag to workers
+                _fill_worker_quit_flag(self.workers, worker_to_quit)
             yield _convert_row(result)
 
     def _log_stuck_warning(self, worker, waiting_time):
@@ -456,6 +485,7 @@ def _generator_worker_loop(dataset, idx_queue, result_queue, eof, is_multiproces
     Multithread or multiprocess generator worker process loop.
     """
     if is_multiprocessing:
+        result_queue.cancel_join_thread()  # Ensure that the process does not hung when exiting
         signal.signal(signal.SIGTERM, partial(_subprocess_handle, eof))
     while True:
         _ignore_sigint(is_multiprocessing=is_multiprocessing)
@@ -470,6 +500,10 @@ def _generator_worker_loop(dataset, idx_queue, result_queue, eof, is_multiproces
                 return
             # If end-of-file (eof) is not set, continue to get data from idx_queue
             continue
+        if idx == "QUIT":
+            # all the data had been processed, so we release the executor which is used by the current thread/process
+            transforms.clean_unused_executors()
+            continue
         if idx is None:
             # When the queue is out of scope from master process, a None item can be fetched from the queue.
             # Upon receiving None, worker process should check if eof is set.
@@ -546,8 +580,7 @@ class _GeneratorWorkerMp(multiprocessing.Process):
             self.res_queue = _SharedQueue(queue_size, count, max_rowsize=max_rowsize)
         else:
             self.res_queue = multiprocessing.Queue(queue_size)
-        self.idx_queue._joincancelled = True  # pylint: disable=W0212
-        self.res_queue._joincancelled = True  # pylint: disable=W0212
+        self.idx_queue.cancel_join_thread()  # Ensure that the process does not hung when exiting
         super().__init__(target=_generator_worker_loop, args=(dataset, self.idx_queue, self.res_queue, eof, True, ppid),
                          name="GeneratorWorkerProcess")
 
@@ -620,9 +653,10 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
             Random accessible input is required.
         python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker process. This
             option could be beneficial if the Python operation is computational heavy. Default: ``True``.
-        max_rowsize(int, optional): Maximum size of row in MB that is used for shared memory allocation to copy
-            data between processes. This is only used if `python_multiprocessing` is set to ``True``.
-            Default: ``6`` MB.
+        max_rowsize(int, optional): Maximum size of row in MB that is used for shared memory
+            allocation to copy data between processes, the total occupied shared memory will increase as
+            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
+            used if python_multiprocessing is set to True. Default: 16.
 
     Raises:
         RuntimeError: If source raises an exception during execution.
@@ -636,7 +670,7 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
 
     Tutorial Examples:
         - `Load & Process Data With Dataset Pipeline
-          <https://www.mindspore.cn/docs/en/r2.1/api_python/samples/dataset/dataset_gallery.html>`_
+          <https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
 
     Note:
         - If you configure `python_multiprocessing=True` (Default: ``True`` ) and `num_parallel_workers>1`
@@ -661,34 +695,10 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
 
         - Input `source` accepts user-defined Python functions (PyFuncs), Do not add network computing operators from
           mindspore.nn and mindspore.ops or others into this `source` .
-        - This dataset can take in a `sampler` . `sampler` and `shuffle` are mutually exclusive.
-          The table below shows what input arguments are allowed and their expected behavior.
-
-    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`
-       :widths: 25 25 50
-       :header-rows: 1
-
-       * - Parameter `sampler`
-         - Parameter `shuffle`
-         - Expected Order Behavior
-       * - None
-         - None
-         - random order
-       * - None
-         - True
-         - random order
-       * - None
-         - False
-         - sequential order
-       * - Sampler object
-         - None
-         - order defined by sampler
-       * - Sampler object
-         - True
-         - not allowed
-       * - Sampler object
-         - False
-         - not allowed
+        - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
+          used in the dataset, and their effects when combined with parameter `sampler` are as follows.
+
+    .. include:: mindspore.dataset.sampler.txt
 
     Examples:
         >>> import mindspore.dataset as ds
@@ -806,38 +816,7 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
     def __deepcopy__(self, memodict):
         if id(self) in memodict:
             return memodict[id(self)]
-        new_op = self.__safe_deepcopy__(memodict, exclude=("source", "__transfer_dataset__"))
-
-        sample_fn = None
-        if new_op.sampler is not None and hasattr(self.source, "__getitem__"):
-            # The reason why there is a try catch here is because when the new op is being constructed with shared
-            # memory enabled, there will be an exception thrown if there is not enough shared memory available
-            if self.source_len == -1:
-                raise RuntimeError("Attempt to construct a random access dataset, '__len__' method is required!")
-
-            if new_op.num_parallel_workers > 1:
-                self.__validate_memory_usage()
-
-                sample_fn = SamplerFn(self.source, new_op.num_parallel_workers, self.python_multiprocessing,
-                                      self.max_rowsize)
-                new_op.prepared_source = (lambda sample_ids: _cpp_sampler_fn_mp(sample_ids, sample_fn))
-            else:
-                new_op.prepared_source = (lambda sample_ids: _cpp_sampler_fn(sample_ids, self.source))
-            new_op.sample_fn = sample_fn
-        else:
-            new_op.sampler = None
-            new_op.sample_fn = sample_fn
-            new_op.source_len = min(new_op.source_len,
-                                    new_op.num_samples) if new_op.num_samples != 0 else new_op.source_len
-            if not hasattr(self.source, "__iter__"):
-                # Use generator function if input callable
-                new_op.prepared_source = (lambda: _generator_fn(self.source, new_op.num_samples))
-            else:
-                # Use iterator function if input is iterable
-                # Random accessible input is also iterable
-                new_op.prepared_source = (lambda: _iter_fn(self.source, new_op.num_samples))
-
-        return new_op
+        return self.__safe_deepcopy__(memodict, exclude=("source", "__transfer_dataset__"))
 
     def is_shuffled(self):
         if self.sampler:
@@ -856,7 +835,38 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
             return super().split(sizes, randomize)
         return super(MappableDataset, self).split(sizes, randomize)
 
+    def prepare_multiprocessing(self):
+        """Preprocessing of prepared_source."""
+        sample_fn = None
+        if self.sampler is not None and hasattr(self.source, "__getitem__"):
+            # The reason why there is a try catch here is because when the new op is being constructed with shared
+            # memory enabled, there will be an exception thrown if there is not enough shared memory available
+            if self.source_len == -1:
+                raise RuntimeError("Attempt to construct a random access dataset, '__len__' method is required!")
+
+            if self.num_parallel_workers > 1:
+                self.__validate_memory_usage()
+
+                sample_fn = SamplerFn(self.source, self.num_parallel_workers, self.python_multiprocessing,
+                                      self.max_rowsize)
+                self.prepared_source = (lambda sample_ids: _cpp_sampler_fn_mp(sample_ids, sample_fn))
+            else:
+                self.prepared_source = (lambda sample_ids: _cpp_sampler_fn(sample_ids, self.source))
+            self.sample_fn = sample_fn
+        else:
+            self.sampler = None
+            self.sample_fn = sample_fn
+            self.source_len = min(self.source_len, self.num_samples) if self.num_samples != 0 else self.source_len
+            if not hasattr(self.source, "__iter__"):
+                # Use generator function if input callable
+                self.prepared_source = (lambda: _generator_fn(self.source, self.num_samples))
+            else:
+                # Use iterator function if input is iterable
+                # Random accessible input is also iterable
+                self.prepared_source = (lambda: _iter_fn(self.source, self.num_samples))
+
     def parse(self, children=None):
+        self.prepare_multiprocessing()
         if self.schema is None:
             return cde.GeneratorNode(self.prepared_source, self.column_names, self.column_types, self.source_len,
                                      self.sampler, self.num_parallel_workers)
@@ -986,34 +996,10 @@ class NumpySlicesDataset(GeneratorDataset):
             specified only when `num_shards` is also specified.
 
     Note:
-        - This dataset can take in a `sampler` . `sampler` and `shuffle` are mutually exclusive.
-          The table below shows what input arguments are allowed and their expected behavior.
-
-    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`
-       :widths: 25 25 50
-       :header-rows: 1
-
-       * - Parameter `sampler`
-         - Parameter `shuffle`
-         - Expected Order Behavior
-       * - None
-         - None
-         - random order
-       * - None
-         - True
-         - random order
-       * - None
-         - False
-         - sequential order
-       * - Sampler object
-         - None
-         - order defined by sampler
-       * - Sampler object
-         - True
-         - not allowed
-       * - Sampler object
-         - False
-         - not allowed
+        - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
+          used in the dataset, and their effects when combined with parameter `sampler` are as follows.
+
+    .. include:: mindspore.dataset.sampler.txt
 
     Raises:
         RuntimeError: If len of column_names does not match output len of data.
@@ -1026,7 +1012,7 @@ class NumpySlicesDataset(GeneratorDataset):
 
     Tutorial Examples:
         - `Load & Process Data With Dataset Pipeline
-          <https://www.mindspore.cn/docs/en/r2.1/api_python/samples/dataset/dataset_gallery.html>`_
+          <https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
 
     Examples:
         >>> import mindspore.dataset as ds
@@ -1092,7 +1078,7 @@ class PaddedDataset(GeneratorDataset):
 
     Tutorial Examples:
         - `Load & Process Data With Dataset Pipeline
-          <https://www.mindspore.cn/docs/en/r2.1/api_python/samples/dataset/dataset_gallery.html>`_
+          <https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
 
     Examples:
         >>> import mindspore.dataset as ds