mindspore 2.2.11__cp39-cp39-win_amd64.whl → 2.3.0__cp39-cp39-win_amd64.whl

mindspore/dataset/engine/datasets_user_defined.py

@@ -19,6 +19,7 @@ After declaring the dataset object, you can further apply dataset operations
 (e.g. filter, skip, concat, map, batch) on it.
 """
 import builtins
+import copy
 import errno
 import math
 import os
@@ -115,33 +116,28 @@ def _cpp_sampler_fn_mp(sample_ids, sample_fn):
     return sample_fn.process(sample_ids)
 
 
-def _fill_worker_indices(workers, indices, idx):
+def _fill_worker_indices(workers, indices, idx_cursor, worker_to_quit):
     """
-    Worker index queue filler, fill worker index queue in round robin order.
+    Worker index queue filler, fill worker index queue in round robin order or QUIT flag.
     """
     num_worker = len(workers)
-    while idx < len(indices):
-        try:
-            workers[idx % num_worker].put(indices[idx])
-            idx += 1
-        except queue.Full:
-            break
-    return idx
-
-
-def _fill_worker_quit_flag(workers, worker_to_quit):
-    """
-    Worker index queue filler, fill worker index queue with QUIT flag.
-    """
-    num_worker = len(workers)
-    for i in range(num_worker):
-        # just put only one QUIT flag to the sub-thread / sub-process
-        if str(i) not in worker_to_quit:
+    if idx_cursor < len(indices):
+        while idx_cursor < len(indices):
             try:
-                workers[i].put("QUIT")
-                worker_to_quit[str(i)] = "QUIT"
+                workers[idx_cursor % num_worker].put(indices[idx_cursor])
+                idx_cursor += 1
             except queue.Full:
-                continue
+                break
+    else:
+        for i in range(num_worker):
+            # just put only one QUIT flag to the sub-thread / sub-process
+            if str(i) not in worker_to_quit:
+                try:
+                    workers[i].put("QUIT")
+                    worker_to_quit[str(i)] = "QUIT"
+                except queue.Full:
+                    continue
+    return idx_cursor, worker_to_quit
 
 
 def _convert_row(row):
@@ -221,10 +217,11 @@ class SamplerFn:
             # res_queue is used shared memory, so it' size is max_rowsize which is defined by user.
             _check_shm_usage(num_worker, queue_size, 0, max_rowsize)
         self.count = multiprocessing.Value('i', 0)
-        for _ in range(num_worker):
+        for worker_id in range(num_worker):
             if multi_process is True:
                 try:
-                    worker = _GeneratorWorkerMp(dataset, self.eof, max_rowsize, queue_size, self.ppid, self.count)
+                    worker = _GeneratorWorkerMp(dataset, self.eof, max_rowsize, queue_size, self.ppid, self.count,
+                                                worker_id)
                     worker.daemon = True
                     # When multi processes fork a subprocess, the lock of the main process is copied to the subprocess,
                     # which may cause deadlock. Therefore, the subprocess startup is performed in the initialization
@@ -243,11 +240,18 @@ class SamplerFn:
                 self.pids.append(worker.pid)
                 self.need_join = True
             else:
-                worker = _GeneratorWorkerMt(dataset, self.eof)
+                worker = _GeneratorWorkerMt(dataset, self.eof, worker_id)
                 worker.daemon = True
             self.workers.append(worker)
         self._launch_cleanup_worker(multi_process=multi_process)
 
+    def _interval_log(self, i, start_time, wait_count):
+        cost_time = int(time.time()) - start_time
+        if cost_time / self.check_interval >= wait_count:
+            wait_count += 1
+            self._log_stuck_warning(self.workers[i % self.num_worker], cost_time)
+        return wait_count
+
     def process(self, indices):
         """
         The main process, start the child process or child thread, and fill the index queue.
@@ -258,19 +262,21 @@ class SamplerFn:
             if not w.queue_empty():
                 # in failover reset scenario the QUIT flag should be pop first
                 while w.idx_queue.qsize() > 0:
-                    result = w.idx_queue.get()
-                    if result != "QUIT":
-                        raise Exception("The queue of the subprocess is not empty.")
+                    try:
+                        result = w.idx_queue.get(timeout=1)
+                        if result != "QUIT":
+                            raise Exception("The queue of the subprocess is not empty.")
+                    except queue.Empty:
+                        continue
             # Start all workers
             if not w.is_alive():
                 w.start()
 
         # Fill initial index queues
         idx_cursor = 0
-        idx_cursor = _fill_worker_indices(self.workers, indices, idx_cursor)
-
         # worker to quit
         worker_to_quit = {}
+        idx_cursor, worker_to_quit = _fill_worker_indices(self.workers, indices, idx_cursor, worker_to_quit)
 
         # Fetch results
         for i in range(len(indices)):
@@ -292,11 +298,16 @@ class SamplerFn:
                         self._stop_subprocess()
                         return
                     time.sleep(0.1)
-                    cost_time = int(time.time()) - start_time
-                    if cost_time / self.check_interval >= wait_count:
-                        wait_count += 1
-                        self._log_stuck_warning(self.workers[i % self.num_worker], cost_time)
+                    wait_count = self._interval_log(i, start_time, wait_count)
                 result = self.workers[i % self.num_worker].get()
+                # Because there is no need to copy when creating Tensors in the C++layer, it reduces the time
+                # from np.ndarray to C++Tensor creation. However, when using shared memory in multiple processes,
+                # the address of the shared memory will always be passed to subsequent nodes in the dataset pipeline,
+                # and the shared memory will also be written by the current node, causing dirty data to be accessed
+                # by subsequent nodes in the pipeline. So make a memory copy here to solve the problem of
+                # shared memory being contaminated.
+                if self.multi_process is True and get_enable_shared_mem():
+                    result = copy.deepcopy(result)
                 if isinstance(result, ExceptionHandler):
                     result.reraise()
             except queue.Empty:
@@ -308,11 +319,9 @@ class SamplerFn:
             if self.eof.is_set():
                 self._stop_subprocess()
                 return
-            if idx_cursor < len(indices):
-                idx_cursor = _fill_worker_indices(self.workers, indices, idx_cursor)
-            else:
-                # send QUIT flag to workers
-                _fill_worker_quit_flag(self.workers, worker_to_quit)
+
+            idx_cursor, worker_to_quit = _fill_worker_indices(self.workers, indices, idx_cursor, worker_to_quit)
+
             yield _convert_row(result)
 
     def _log_stuck_warning(self, worker, waiting_time):
@@ -360,13 +369,16 @@ class SamplerFn:
         """
         if multi_process is True and platform.system().lower() != 'windows':
             _clean_worker_func = _PythonMultiprocessing._clean_process  # pylint: disable=W0212
-            self.cleaning_process = multiprocessing.Process(target=_clean_worker_func, args=(self.ppid, self.workers))
+            self.cleaning_process = multiprocessing.Process(target=_clean_worker_func,
+                                                            name="GeneratorCleanProcess",
+                                                            args=(self.ppid, self.workers, self.eof))
             self.cleaning_process.daemon = True
             self.cleaning_process.start()
 
             if get_enable_watchdog():
                 self.eot = threading.Event()
                 self.watch_dog = threading.Thread(target=_PythonMultiprocessing._watch_dog,  # pylint: disable=W0212
+                                                  name="GeneratorWatchDog",
                                                   args=(self.eot, self.workers + [self.cleaning_process]))
                 self.watch_dog.daemon = True
                 self.watch_dog.start()
@@ -381,11 +393,10 @@ class SamplerFn:
     def _stop_subprocess(self):
         """Only the main process can call join."""
         if self.need_join is True and self.ppid == os.getpid():
+            if hasattr(self, 'eof') and self.eof is not None:
+                self.eof.set()
             # close the watch dog first
             self._abort_watchdog()
-
-            if hasattr(self, 'eof') and self.eof is not None and not self.eof.is_set():
-                self.eof.set()
             self.need_join = False
             for w in self.workers:
                 if self.multi_process is True and hasattr(w, '_closed') and w._closed is False:  # pylint: disable=W0212
@@ -394,10 +405,12 @@ class SamplerFn:
                         del w.res_queue
                         del w.idx_queue
 
-                        # close all the subprocess workers
-                        w.terminate()
-                        w.join()
-                        w.close()
+                        # let the quit event notify the worker process to exit
+                        w.join(timeout=5)
+                        if w.is_alive():
+                            # if the worker process did not exit, it may hang, try to terminate it
+                            w.terminate()
+                            w.close()
                     except Exception:  # pylint: disable=W0703
                         # Block all errors when join
                         continue
@@ -429,10 +442,15 @@ class SamplerFn:
             self.workers = None
 
     def _abort_watchdog(self):
+        """Let watchdog quit."""
         if hasattr(self, 'eot') and self.eot is not None and not self.eot.is_set():
             self.eot.set()
         if hasattr(self, 'cleaning_process') and self.cleaning_process is not None:
-            _PythonMultiprocessing._terminate_processes([self.cleaning_process])  # pylint: disable=W0212
+            # let the quit event notify the cleaning process to exit
+            self.cleaning_process.join(timeout=5)
+            if self.cleaning_process.is_alive():
+                # if the cleaning process did not exit, it may hang, try to terminate it
+                _PythonMultiprocessing._terminate_processes([self.cleaning_process])  # pylint: disable=W0212
             del self.cleaning_process
         if hasattr(self, 'count'):
             del self.count
@@ -487,7 +505,7 @@ def _generator_worker_loop(dataset, idx_queue, result_queue, eof, is_multiproces
     if is_multiprocessing:
         result_queue.cancel_join_thread()  # Ensure that the process does not hung when exiting
         signal.signal(signal.SIGTERM, partial(_subprocess_handle, eof))
-    while True:
+    while not eof.is_set():
         _ignore_sigint(is_multiprocessing=is_multiprocessing)
 
         # Fetch index, block
@@ -522,7 +540,7 @@ def _generator_worker_loop(dataset, idx_queue, result_queue, eof, is_multiproces
         except Exception:  # pylint: disable=broad-except
             result = ExceptionHandler(where="in GeneratorDataset worker process")
         # Send data, block
-        while True:
+        while not eof.is_set():
             try:
                 result_queue.put(result, timeout=5)
             except queue.Full:
@@ -541,11 +559,11 @@ class _GeneratorWorkerMt(threading.Thread):
     Worker process for multi-thread Generator.
     """
 
-    def __init__(self, dataset, eof):
+    def __init__(self, dataset, eof, worker_id):
         self.idx_queue = queue.Queue(16)
         self.res_queue = queue.Queue(16)
         super().__init__(target=_generator_worker_loop, args=(dataset, self.idx_queue, self.res_queue, eof, False),
-                         name="GeneratorWorkerThread")
+                         name="GeneratorWorkerThread" + str(worker_id))
 
     def put(self, item):
         """
@@ -574,7 +592,7 @@ class _GeneratorWorkerMp(multiprocessing.Process):
     Worker process for multiprocess Generator.
     """
 
-    def __init__(self, dataset, eof, max_rowsize, queue_size, ppid, count):
+    def __init__(self, dataset, eof, max_rowsize, queue_size, ppid, count, worker_id):
         self.idx_queue = multiprocessing.Queue(queue_size)
         if get_enable_shared_mem():
             self.res_queue = _SharedQueue(queue_size, count, max_rowsize=max_rowsize)
@@ -582,7 +600,7 @@ class _GeneratorWorkerMp(multiprocessing.Process):
             self.res_queue = multiprocessing.Queue(queue_size)
         self.idx_queue.cancel_join_thread()  # Ensure that the process does not hung when exiting
         super().__init__(target=_generator_worker_loop, args=(dataset, self.idx_queue, self.res_queue, eof, True, ppid),
-                         name="GeneratorWorkerProcess")
+                         name="GeneratorWorkerProcess" + str(worker_id))
 
     def put(self, item):
         """
@@ -655,8 +673,9 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
             option could be beneficial if the Python operation is computational heavy. Default: ``True``.
         max_rowsize(int, optional): Maximum size of row in MB that is used for shared memory
             allocation to copy data between processes, the total occupied shared memory will increase as
-            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
-            used if python_multiprocessing is set to True. Default: 16.
+            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set to -1,
+            shared memory will be dynamically allocated with the actual size of data. This is only used if
+            ``python_multiprocessing`` is set to True. Default: 16.
 
     Raises:
         RuntimeError: If source raises an exception during execution.
@@ -670,7 +689,7 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
 
     Tutorial Examples:
         - `Load & Process Data With Dataset Pipeline
-          <https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
+          <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_
 
     Note:
         - If you configure `python_multiprocessing=True` (Default: ``True`` ) and `num_parallel_workers>1`
@@ -1012,7 +1031,7 @@ class NumpySlicesDataset(GeneratorDataset):
 
     Tutorial Examples:
         - `Load & Process Data With Dataset Pipeline
-          <https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
+          <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_
 
     Examples:
         >>> import mindspore.dataset as ds
@@ -1078,7 +1097,7 @@ class PaddedDataset(GeneratorDataset):
 
     Tutorial Examples:
         - `Load & Process Data With Dataset Pipeline
-          <https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
+          <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_
 
     Examples:
         >>> import mindspore.dataset as ds