mindspore 2.4.10__cp311-cp311-win_amd64.whl → 2.6.0rc1__cp311-cp311-win_amd64.whl

mindspore/dataset/engine/datasets.py

@@ -35,11 +35,9 @@ import stat
 import subprocess
 import warnings
 
-import gc
 import time
 import uuid
 import multiprocessing
-from enum import Enum
 from importlib import import_module
 import sys
 import threading
@@ -53,18 +51,18 @@ import mindspore._c_dataengine as cde
 from mindspore._c_expression import typing
 
 from mindspore import log as logger
-from mindspore.parallel._ps_context import _is_role_pserver, _is_role_sched, _get_ps_context,\
-                                           _enable_distributed_mindrt
+from mindspore.parallel._ps_context import _is_role_pserver, _is_role_sched, _get_ps_context, \
+    _enable_distributed_mindrt
 from mindspore.dataset.engine.offload import GetOffloadModel
-
+from mindspore.communication.management import get_group_size
 import mindspore.dataset.transforms.c_transforms as c_transforms
 import mindspore.dataset.transforms.py_transforms as py_transforms
 import mindspore.dataset.transforms as transforms
 from mindspore.dataset.text.utils import SentencePieceModel, DE_C_INTER_SENTENCEPIECE_MODE
-from mindspore.parallel._utils import _get_device_num
 from mindspore.dataset.debug import DebugHook
 
 from mindspore.dataset.engine import samplers
+from mindspore.dataset.engine.samplers import Shuffle
 from .iterators import DictIterator, TupleIterator, DummyIterator, check_iterator_cleanup, _set_iterator_cleanup, \
     ITERATORS_LIST, _unset_iterator_cleanup, _cleanup_the_iterators_if_created
 from .queue import _SharedQueue, _Queue
@@ -74,13 +72,14 @@ from .validators import check_batch, check_shuffle, check_map, check_filter, che
     check_save, check_tuple_iterator, check_dict_iterator, check_schema, check_to_device_send, check_padded_batch, \
     check_total_batch, check_sync_update
 from ..core.config import get_callback_timeout, _init_device_info, get_enable_shared_mem, get_num_parallel_workers, \
-    get_enable_watchdog, get_seed, set_seed, get_debug_mode, get_multiprocessing_timeout_interval, _get_debug_hook_list
+    get_enable_watchdog, get_seed, set_seed, get_debug_mode, get_multiprocessing_timeout_interval, \
+    _get_debug_hook_list, get_multiprocessing_start_method
 from ..core.datatypes import mstype_to_detype
 from ..core.validator_helpers import replace_none
 from ..core.py_util_helpers import ExceptionHandler
 from ..transforms.py_transforms_util import FuncWrapper, Implementation
 from ..vision.transforms import ToNumpy
-from ...mindrecord.config import _get_enc_key, _get_enc_mode, _get_hash_mode, encrypt, append_hash_to_file
+from ...mindrecord.config import _get_enc_key, _get_enc_mode, encrypt
 
 try:
     context = import_module("mindspore.context")
@@ -136,71 +135,6 @@ def _reset_training_dataset(global_step, dataset_size):
         raise RuntimeError("Training dataset is not set.")
 
 
-class Shuffle(str, Enum):
-    """Specify the shuffle mode.
-
-    - ``Shuffle.GLOBAL`` : Shuffle both the files and samples.
-    - ``Shuffle.FILES`` : Shuffle files only.
-    - ``Shuffle.INFILE`` : Shuffle data within each file.
-    """
-    GLOBAL: str = "global"
-    FILES: str = "files"
-    INFILE: str = "infile"
-
-
-ShuffleToShuffleMode = {Shuffle.FILES: cde.ShuffleMode.FILES,
-                        Shuffle.GLOBAL: cde.ShuffleMode.GLOBAL,
-                        Shuffle.INFILE: cde.ShuffleMode.INFILE}
-
-
-def shuffle_to_shuffle_mode(shuffle):
-    """
-    Shuffle Enum to Shuffle Mode
-
-    Args:
-        shuffle (Shuffle): shuffle flag to shuffle mode in C layer
-
-    Returns:
-        ShuffleMode, shuffle mode
-    """
-    shuffle_mode = cde.ShuffleMode.GLOBAL  # Global shuffle
-    if not isinstance(shuffle, Shuffle):
-        if shuffle is None or shuffle:
-            shuffle_mode = cde.ShuffleMode.GLOBAL  # Global shuffle
-        else:
-            shuffle_mode = cde.ShuffleMode.FALSE  # No shuffle
-    else:
-        shuffle_mode = ShuffleToShuffleMode[shuffle]
-    return shuffle_mode
-
-
-def shuffle_to_bool(shuffle):
-    """
-    Shuffle Enum to bool
-
-    Args:
-        shuffle (Shuffle): shuffle flag to bool
-
-    Returns:
-        bool, True / False
-    """
-    if shuffle is not None and not isinstance(shuffle, (bool, Shuffle)):
-        raise TypeError("shuffle must be of boolean or enum of 'Shuffle' values like 'Shuffle.GLOBAL' or "
-                        "'Shuffle.FILES' or 'Shuffle.INFILE'.")
-
-    shuffle_bool = True
-    if not isinstance(shuffle, Shuffle):
-        if shuffle is None:
-            shuffle_bool = None
-        elif shuffle:
-            shuffle_bool = True
-        else:
-            shuffle_bool = False
-    else:
-        shuffle_bool = True
-    return shuffle_bool
-
-
 @check_zip
 def zip(datasets):
     """
@@ -403,6 +337,7 @@ class Dataset:
         parent = self.parent
         self.parent = []
         dataset = copy.deepcopy(self)
+        dataset = self.pre_process(dataset)
         global _OP_NAME
         _OP_NAME = Dataset._get_operator_id(dataset)
         ir_tree = dataset.parse_tree(getter_mode)
@@ -410,6 +345,19 @@ class Dataset:
         _init_device_info()
         return ir_tree, dataset
 
+    def pre_process(self, dataset):
+        """Insert batch operation for GeneratorDataset with batch_sampler."""
+        if hasattr(dataset, "has_batch_sampler") and dataset.has_batch_sampler:
+            original_parent = dataset.parent
+            dataset.parent = []
+            dataset = dataset.batch(batch_size=-1, num_parallel_workers=dataset.num_parallel_workers,
+                                    per_batch_map=dataset.collate_fn)
+            dataset.parent = original_parent
+        else:
+            for index in range(len(dataset.children)):
+                dataset.children[index] = self.pre_process(dataset.children[index])
+        return dataset
+
     def parse_tree(self, getter_mode=False):
         """
         Internal method to parse the API tree into an IR tree.
@@ -495,8 +443,7 @@ class Dataset:
         .. image:: bucket_batch_by_length_en.png
 
         Note:
-            - When using `Data Sinking <https://www.mindspore.cn/docs/en/master/model_train/train_process/optimize/
-              sink_mode.html#data-sinking>`_ in Graph mode, the input shape of the network should keep consistent.
+            - When using Data Sinking in Graph mode, the input shape of the network should keep consistent.
               You should set `drop_remainder` to "True" to discard the last incomplete batch of data,
               or supplement/remove samples to ensure the dataset size is divisible by `batch_size`.
 
@@ -561,7 +508,7 @@ class Dataset:
     @check_batch
     def batch(self, batch_size, drop_remainder=False, num_parallel_workers=None, **kwargs):
         """
-        Combine batch_size number of consecutive rows into batch which apply per_batch_map to the samples first.
+        Combine `batch_size` number of consecutive rows into batch which apply `per_batch_map` to the samples first.
 
         For any column, all the elements within that column must have the same shape.
 
@@ -572,8 +519,7 @@ class Dataset:
         Note:
             - The order of using repeat and batch reflects the number of batches and per_batch_map.
               It is recommended that the repeat operation applied after the batch operation finished.
-            - When using `Data Sinking <https://www.mindspore.cn/docs/en/master/model_train/train_process/optimize/
-              sink_mode.html#data-sinking>`_ in Graph mode, the input shape of the network should keep consistent.
+            - When using Data Sinking in Graph mode, the input shape of the network should keep consistent.
               You should set `drop_remainder` to "True" to discard the last incomplete batch of data,
               or supplement/remove samples to ensure the dataset size is divisible by `batch_size`.
 
@@ -615,13 +561,19 @@ class Dataset:
 
                 - max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
                   allocation to copy data between processes, the total occupied shared memory will increase as
-                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set
-                  to -1, shared memory will be dynamically allocated with the actual size of data. This is only used if
-                  ``python_multiprocessing`` is set to True. If it is an int value, it represents
-                  ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
-                  If it is a list, the first element represents the ``input_columns`` use this value as the unit to
-                  create shared memory, and the second element represents ``output_columns`` use this value as the unit
-                  to create shared memory. Default: ``None`` , allocate shared memory dynamically.
+                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase.
+                  This is only used if ``python_multiprocessing`` is set to ``True``.
+                  Default: ``None`` , allocate shared memory dynamically (deprecated in future version).
+
+                  - If set to ``-1`` / ``None``, shared memory will be dynamically allocated with the
+                    actual size of data.
+
+                  - If it is an int value, it represents ``input_columns`` and ``output_columns`` use this value as the
+                    unit to create shared memory.
+
+                  - If it is a list, represents the ``input_columns`` use the first element as the unit to
+                    create shared memory, and represents ``output_columns`` use the second element as the
+                    unit to create shared memory.
 
         Returns:
             Dataset, a new dataset with the above operation applied.
@@ -669,8 +621,7 @@ class Dataset:
         Note:
             - The order of using repeat and padded_batch reflects the number of batches.
               It is recommended that the repeat operation applied after the padded_batch operation finished.
-            - When using `Data Sinking <https://www.mindspore.cn/docs/en/master/model_train/train_process/optimize/
-              sink_mode.html#data-sinking>`_ in Graph mode, the input shape of the network should keep consistent.
+            - When using Data Sinking in Graph mode, the input shape of the network should keep consistent.
               You should set `drop_remainder` to "True" to discard the last incomplete batch of data,
               or supplement/remove samples to ensure the dataset size is divisible by `batch_size`.
 
@@ -724,9 +675,9 @@ class Dataset:
 
         Args:
             condition_name (str): The condition name that is used to toggle sending next row.
-            num_batch (int): the number of batches without blocking at the start of each epoch.
+            num_batch (int, optional): the number of batches without blocking at the start of each epoch.
                 Default: ``1``.
-            callback (function): The callback function that will be invoked when sync_update is called.
+            callback (function, optional): The callback function that will be invoked when sync_update is called.
                 Default: ``None``.
 
         Returns:
@@ -911,15 +862,21 @@ class Dataset:
                 - python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker processes.
                   This option could be beneficial if the Python operation is computational heavy. Default: ``False``.
 
-                - max_rowsize (Union[int, list[int]], optional): Maximum size of row in MB that is used for shared
-                  memory allocation to copy data between processes, the total occupied shared memory will increase as
-                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set
-                  to -1, shared memory will be dynamically allocated with the actual size of data. This is only used if
-                  ``python_multiprocessing`` is set to True. If it is an int value, it represents
-                  ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
-                  If it is a list, the first element represents the ``input_columns`` use this value as the unit to
-                  create shared memory, and the second element represents ``output_columns`` use this value as the unit
-                  to create shared memory. Default: ``None`` , allocate shared memory dynamically.
+                - max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
+                  allocation to copy data between processes, the total occupied shared memory will increase as
+                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase.
+                  This is only used if ``python_multiprocessing`` is set to ``True``.
+                  Default: ``None`` , allocate shared memory dynamically (deprecated in future version).
+
+                  - If set to ``-1`` / ``None``, shared memory will be dynamically allocated with the
+                    actual size of data.
+
+                  - If it is an int value, it represents ``input_columns`` and ``output_columns`` use this value as the
+                    unit to create shared memory.
+
+                  - If it is a list, the first element represents the ``input_columns`` use this value as the unit to
+                    create shared memory, and the second element represents ``output_columns`` use this value as the
+                    unit to create shared memory.
 
                 - cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
                   Default: ``None``, which means no cache is used.
@@ -932,8 +889,29 @@ class Dataset:
         Note:
             - Input `operations` accepts TensorOperations defined in mindspore.dataset part, plus user-defined
               Python functions (PyFuncs).
-            - Do not add network computing operators from mindspore.nn and mindspore.ops or others into this
-              `operations` .
+            - Setting the start method of multiprocessing to `spawn` mode by
+              ds.config.set_multiprocessing_start_method("spawn") with `python_ multiprocessing=True`
+              and `num_parallel_workers>1` supports adding network computing operators from mindspore.nn and
+              mindspore.ops or other network computing operators into this `operations` .
+              Otherwise, adding to `operations` is not supported.
+            - Currently only some scenarios support calling DVPP operators in Python functions passed in with the
+              `operations` parameter:
+
+              +---------------+----------------------------+----------------------------+----------------------------+
+              |               |                            |                     Multiprocessing                     |
+              |               |       Multithreading       +----------------------------+----------------------------+
+              |               |                            |           spawn            |            fork            |
+              +===============+============================+============================+============================+
+              |Independent    |Data Processing: support    |Data Processing: support    |Data Processing: support    |
+              |               |                            |                            |                            |
+              |process mode   |Data Processing + Network   |Data Processing + Network   |Data Processing + Network   |
+              |               |training: not support       |training: support           |training: not support       |
+              +---------------+----------------------------+----------------------------+----------------------------+
+              |Non-independent|Data Processing: support    |Data Processing: support    |Data Processing: support    |
+              |               |                            |                            |                            |
+              |process mode   |Data Processing + Network   |Data Processing + Network   |Data Processing + Network   |
+              |               |training: support           |training: support           |training: not support       |
+              +---------------+----------------------------+----------------------------+----------------------------+
 
         Returns:
             Dataset, a new dataset with the above operation applied.
@@ -1557,8 +1535,8 @@ class Dataset:
             >>> d1 = ds.GeneratorDataset(generator_1d, ["data"], shuffle=False)
             >>> d1.save('/path/to/save_file')
         """
-        if (_get_enc_key() is not None or _get_hash_mode() is not None) and num_files > 1:
-            raise RuntimeError("When encode mode or hash check is enabled, " +
+        if _get_enc_key() is not None and num_files > 1:
+            raise RuntimeError("When encode mode is enabled, " +
                                "the automatic sharding function is unavailable.")
 
         ir_tree, api_tree = self.create_ir_tree()
@@ -1571,10 +1549,6 @@ class Dataset:
 
         consumer.Save()
 
-        if _get_hash_mode() is not None:
-            append_hash_to_file(file_name)
-            append_hash_to_file(file_name + ".db")
-
         if _get_enc_key() is not None:
             encrypt(file_name, _get_enc_key(), _get_enc_mode())
             encrypt(file_name + ".db", _get_enc_key(), _get_enc_mode())
@@ -1761,7 +1735,7 @@ class Dataset:
         Get the shapes of output data.
 
         Args:
-            estimate (bool): If `estimate` is ``False`` , will return the shapes of first data row.
+            estimate (bool, optional): If `estimate` is ``False`` , will return the shapes of first data row.
                 Otherwise, will iterate the whole dataset and return the estimated shapes of data row,
                 where dynamic shape is marked as None (used in dynamic data shapes scenario).
                 Default: ``False`` .
@@ -2338,10 +2312,10 @@ class SourceDataset(Dataset):
         self.shard_id = replace_none(shard_id, 0)
 
         if shuffle is not None and not isinstance(shuffle, (bool, Shuffle)):
-            raise TypeError("shuffle must be of boolean or enum of 'Shuffle' values like 'Shuffle.GLOBAL' or "
-                            "'Shuffle.FILES' or 'Shuffle.INFILE'.")
+            raise TypeError("shuffle must be of boolean or enum of 'Shuffle' values like 'Shuffle.ADAPTIVE' or "
+                            "'Shuffle.GLOBAL' or 'Shuffle.PARTIAL' or 'Shuffle.FILES' or 'Shuffle.INFILE'.")
 
-        self.shuffle_flag = 2  # Global shuffle
+        self.shuffle_flag = 5  # Adaptive shuffle
         if not isinstance(shuffle, Shuffle):
             if shuffle is None or shuffle:
                 self.shuffle_flag = 2  # Global shuffle
@@ -2354,6 +2328,10 @@ class SourceDataset(Dataset):
                 self.shuffle_flag = 1  # Files shuffle
             elif shuffle == Shuffle.INFILE:
                 self.shuffle_flag = 3  # Infile shuffle
+            elif shuffle == Shuffle.ADAPTIVE:
+                self.shuffle_flag = 5
+            elif shuffle == Shuffle.PARTIAL:
+                self.shuffle_flag = 4
 
     def parse(self, children=None):
         raise NotImplementedError("Dataset has to implement parse method.")
@@ -2410,15 +2388,23 @@ class MappableDataset(SourceDataset):
     def __init__(self, num_parallel_workers=None, sampler=None, num_samples=None, shuffle=None, num_shards=None,
                  shard_id=None, cache=None):
         num_shards, shard_id = self._update_data_shard(num_shards, shard_id)
+        if sampler is None:
+            if shuffle is None or shuffle is True:
+                shuffle = Shuffle.GLOBAL
+            elif shuffle is False:
+                shuffle = Shuffle.FALSE
         super().__init__(num_parallel_workers=num_parallel_workers, num_samples=num_samples, shuffle=shuffle,
                          num_shards=num_shards, shard_id=shard_id, cache=cache)
-        self.shuffle_flag = replace_none(shuffle, True)
         self.sampler = samplers.select_sampler(num_samples, sampler, shuffle, num_shards, shard_id)
 
     def add_sampler(self, new_sampler):
         """
         Add a child sampler for the current dataset.
 
+        Note:
+            - If the sampler is added and it has a shuffle option, its value must be ``Shuffle.GLOBAL`` .
+              Additionally, the original sampler's shuffle value cannot be ``Shuffle.PARTIAL`` .
+
         Args:
             new_sampler (Sampler): The child sampler to be added.
 
@@ -2432,6 +2418,16 @@ class MappableDataset(SourceDataset):
         # Note: By adding a sampler, the sampled IDs will flow to the new_sampler
         # after first passing through the current samplers attached to this dataset.
         self.dataset_size = None
+
+        if self.sampler is not None and self.sampler.get_shuffle_mode() == Shuffle.PARTIAL:
+            raise RuntimeError("When multiple samplers are used, ensure that the shuffle of the current sampler "
+                               "must not be Shuffle.PARTIAL.")
+
+        if new_sampler.get_shuffle_mode() != Shuffle.GLOBAL and new_sampler.get_shuffle_mode() != Shuffle.FALSE:
+            raise RuntimeError("When multiple samplers are used, ensure that the shuffle of the input sampler "
+                               "must be Shuffle.FALSE or Shuffle.GLOBAL, but got: {}."
+                               .format(new_sampler.get_shuffle_mode()))
+
         new_sampler.add_child(self.sampler)
         self.sampler = new_sampler
 
@@ -2594,7 +2590,7 @@ def _check_shm_usage(num_worker, queue_size, in_rowsize, out_rowsize):
     threshold_ratio = 0.8
     # Verify available size only when using static shared memory on Linux
     if platform.system().lower() not in {"windows", "darwin"} and in_rowsize != -1 and out_rowsize != -1:
-        device_num = _get_device_num()
+        device_num = get_group_size()
         # In the cluster, _get_device_num indicates the number of the entire cluster. The maximum number of cards
         # on the ascend server is 8.
         if device_num > 1:
@@ -2680,11 +2676,6 @@ class BatchDataset(UnionBaseDataset):
         else:
             self.max_rowsize = [max_rowsize[0] * self.batch_size, max_rowsize[1] * self.batch_size]
 
-    def __del__(self):
-        if hasattr(self, "process_pool") and self.process_pool is not None:
-            self.process_pool.terminate()
-            del self.process_pool
-
     def parse(self, children=None):
         return cde.BatchNode(children[0], self.batch_size, self.drop_remainder, False, self.input_columns,
                              self.output_columns, self.batch_size_func, self.per_batch_map, {},
@@ -2747,8 +2738,8 @@ class BatchDataset(UnionBaseDataset):
             if self.num_parallel_workers is None:
                 self.num_parallel_workers = get_num_parallel_workers()
 
-            self.process_pool = _PythonMultiprocessing(str(self), self.num_parallel_workers, [self.per_batch_map],
-                                                       self.max_rowsize)
+            self.process_pool = _PythonMultiprocessing(get_multiprocessing_start_method(), self.num_parallel_workers,
+                                                       str(self), [self.per_batch_map], self.max_rowsize)
             # Wrap per_batch_map into _PythonCallable
             self.per_batch_map = _PythonCallable(self.per_batch_map, 0, self.process_pool)
         else:
@@ -3023,7 +3014,7 @@ class SyncWaitDataset(UnionBaseDataset):
 
 class ShuffleDataset(UnionBaseDataset):
     """
-    The result of applying Shuffle operation to the input Dataset.
+    The result of applying shuffle operation to the input Dataset.
 
     Args:
         input_dataset (Dataset): Input Dataset to be shuffled.
@@ -3200,9 +3191,21 @@ def _worker_loop(operations, pipe, worker_id):
 
 
 def worker_target(operations, worker_id):
+    logger.info("Multiprocessing start method: {}".format(multiprocessing.get_start_method()))
     return lambda pipe: _worker_loop(operations, pipe, worker_id)
 
 
+class WorkerTarget:
+    def __init__(self, operations, pipe, worker_id):
+        self.operations = operations
+        self.pipe = pipe
+        self.worker_id = worker_id
+        logger.info("Multiprocessing start method: {}".format(multiprocessing.get_start_method()))
+
+    def __call__(self):
+        return _worker_loop(self.operations, self.pipe, self.worker_id)
+
+
 class _MPWorker(multiprocessing.Process):
     """
     Worker process for multiprocessing.
@@ -3257,6 +3260,12 @@ class _MPWorker(multiprocessing.Process):
 
                 logger.info(f"Closing worker with PID: {self.pid}")
                 self.pipe.master_close()
+
+                process_dir = os.path.join('/proc', str(self.pid))
+                while self.is_alive() and os.path.exists(process_dir):
+                    logger.info("Waiting for worker {} closed ...".format(self.pid))
+                    time.sleep(0.001)
+
                 # del the handle which hold by master
                 del self.pipe.in_queue
                 del self.pipe.res_queue
@@ -3276,6 +3285,41 @@ class _MPWorker(multiprocessing.Process):
             return False
 
 
+def worker_is_alive(worker):
+    """Check the subprocess worker status in spawn mode"""
+    try:
+        return worker.is_alive()
+    except ValueError:
+        return False
+
+
+def close_worker(worker, pipe):
+    """Close the subprocess worker in spawn mode"""
+    try:
+        if worker_is_alive(worker):
+            # release the eager executor which is used by current process
+            transforms.transforms.clean_unused_executors()
+
+            logger.info(f"Closing worker with PID: {worker.pid}")
+            pipe.master_close()
+
+            process_dir = os.path.join('/proc', str(worker.pid))
+            while worker_is_alive(worker) and os.path.exists(process_dir):
+                logger.info("Waiting for worker {} closed ...".format(worker.pid))
+                time.sleep(0.5)
+
+            # del the handle which hold by master
+            del pipe.in_queue
+            del pipe.res_queue
+            worker.terminate()
+            worker.join()
+            worker.close()
+    except ValueError:
+        # Process has been closed already
+        return
+    return
+
+
 class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
     """
     A wrapper to multiprocessing.pool that performs cleanup and ensure proper termination of forked processes.
@@ -3302,10 +3346,11 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
             self.origin_hook(ex_type, value, tb)
             self.mp_pool_exit_preprocess()
 
-    def __init__(self, op_name, num_parallel_workers, operations, max_rowsize=(-1, -1)):
+    def __init__(self, start_method, num_parallel_workers, op_name, operations, max_rowsize=(-1, -1)):
         super(_PythonMultiprocessing, self).__init__()
-        self.op_name = op_name
+        self.start_method = start_method  # python multiprocssing start method: fork / spawn
         self.num_parallel_workers = num_parallel_workers
+        self.op_name = op_name
         self.operations = operations
         self.max_rowsize = max_rowsize
 
@@ -3316,14 +3361,14 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         self.queues_map = {}
         self.next_queue = 0
 
-        self.eot = None
-        self.watch_dog = None
+        self.cleaning_process = None
         self.ppid = None
         self.hook = None
         self.warning_ctl = None
         # cache thread (get_ident()) to worker_id mapping in Python layer
         self.python_threads_to_workers = {}
         self.eof = None
+        self.running = False
 
     def __del__(self):
         try:
@@ -3331,60 +3376,6 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         except TypeError:
             pass
 
-    # This wait function is for cleaning zombie subprocesses
-    @staticmethod
-    def wait_pid():
-        """
-        This function is used by the main process to release subprocess resources.
-        """
-        try:
-            while True:
-                child_pid, _ = os.waitpid(-1, os.WNOHANG)
-                if child_pid == 0:
-                    break
-        except OSError:
-            # waitpid may fail for some reason, so we ignore this error
-            pass
-
-    # Dataset need watch_dog thread to monitoring fork multiprocessing,
-    # and thread can't be a member function otherwise python won't collect and release resources.
-    @staticmethod
-    def _watch_dog(eot, workers):
-        """
-        This thread is for monitoring subprocesses forked by GeneratorDataset/map/batch
-        """
-        if not isinstance(workers, list):
-            raise TypeError("[Internal Error] The 2nd parameter of watch dog thread should be list of process, "
-                            "but got {}.".format(type(workers)))
-
-        while not eot.is_set():
-            # Monitoring and count how many subprocesses already exit
-            clear_subprocess_timeout = _PythonMultiprocessing._monitor_subprocess_exit(workers)
-            # If find subprocess exit, we will wait for 30s and do some waitpid operations
-            if clear_subprocess_timeout > 0:
-                start = time.time()
-                while time.time() - start < clear_subprocess_timeout:
-                    # We need to distinguishing get_dataset_size or train finished normally and hang scenario.
-                    # If get_dataset_size or train finished normally, _stop_subprocess can be execute and
-                    # self.need_abort can be set to True. If main process is hang in get(), self.need_abort
-                    # will never set to True, then we wait for 30s and kill main process
-                    if eot.is_set():
-                        return
-                    # Sometimes subprocess may be zombie, so in 30s we can wait and do some useful tasks(waitpid).
-                    _PythonMultiprocessing.wait_pid()
-                # multiprocessing.queue may hang in .get() forever when put() process was killed.
-                # We have to exit main process otherwise main process will hang.
-                _PythonMultiprocessing._terminate_processes(workers)
-                logger.critical("The subprocess of dataset may exit unexpected or be killed, "
-                                "main process will exit. If this is not an artificial operation, you can use "
-                                "ds.config.set_enable_watchdog(False) to block this error.")
-                os.kill(os.getpid(), signal.SIGTERM)
-            # sleep to release GIL
-            time.sleep(1)
-
-        # release the workers
-        del workers
-
     @staticmethod
     def _terminate_processes(processes):
         """Terminate subprocesses"""
@@ -3401,45 +3392,12 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
                 # We don't use w.join because join can only used in main process or join will raise an error.
                 p._popen.wait()  # pylint: disable=W0212
 
-    # Monitor the exit number of subprocesses
-    @staticmethod
-    def _monitor_subprocess_exit(workers):
-        """
-        To monitor whether process is exit.
-
-        Args:
-            workers (list of multiprocessing.Process): multiprocessing.Process.
-
-        Returns:
-            int, the timeout(in seconds) when process exit.
-        """
-        for w in workers:
-            try:
-                exit_code = w.exitcode
-                if exit_code is not None:
-                    # For kill -9, we can exit quickly
-                    if exit_code == -9:
-                        return 1
-                    # For kill -15, we still exit after 30s
-                    if exit_code == -15:
-                        return 30
-                # In some cases the subprocess has been killed but the exitcode is still None.
-                # So we use os.kill(pid, 0) to check if it is alive.
-                subprocess_alive = _PythonMultiprocessing.is_process_alive(w.pid)
-                if not subprocess_alive:
-                    # Like kill -15, we wait 30s before exit
-                    return 30
-            except ValueError:
-                # process has been closed already
-                return 0
-        return 0
-
     @staticmethod
     def is_process_alive(pid):
         """
         Check if the process is alive or not.
         Note:  We hit a deadlock when we use psutil or w.exitcode to check whether a process is alive.
-        Instead we use os.kill(ppid, 0).
+        Instead, we use os.kill(ppid, 0).
 
         Args:
             pid: pid of the process to be checked
@@ -3466,6 +3424,8 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
             quit_signal: The flag of quit.
         """
         signal.signal(signal.SIGINT, signal.SIG_IGN)
+        # Initialize C++ side signal handlers
+        cde.register_worker_handlers()
         while _PythonMultiprocessing.is_process_alive(ppid):
             if quit_signal.is_set():
                 return
@@ -3477,6 +3437,8 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
 
             time.sleep(0.1)
 
+        logger.info("Clean process detects that the main process {} has exited, begin to terminate the "
+                    "worker process(es): {}".format(ppid, [worker.pid for worker in workers]))
         _PythonMultiprocessing._terminate_processes(workers)
         del workers
         os.kill(os.getpid(), signal.SIGTERM)
@@ -3493,10 +3455,10 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         """
         self.python_threads_to_workers = {}
         self.op_id = op_id
-        logger.info("Launching new Python Multiprocessing pool for Op:" + str(self.op_id))
+        logger.info("Launching new Python multiprocessing pool for Op: " + str(self.op_id))
         if self.is_mp_enabled():
             message = "Launching a new Python multiprocessing pool while a pool already exists!" + \
-                " The existing pool will be terminated first."
+                      " The existing pool will be terminated first."
             logger.warning(message)
             self.terminate()
             self.reset()
@@ -3515,32 +3477,52 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         if self.workers is not None:
             raise Exception("Pool was already created, close it first.")
 
-        # Let gc collect unreferenced memory to avoid child processes in the pool to do it
-        gc.collect()
-
-        # Construct python worker processes
         self.workers = []
+        self.pipes = []
+        self.check_interval = get_multiprocessing_timeout_interval()
         self.warning_ctl = multiprocessing.Value('i', 0)
-        for worker_id in range(self.num_parallel_workers):
-            worker = _MPWorker(self.operations, self.warning_ctl, self.max_rowsize, worker_id)
-            worker.start()
-            self.workers.append(worker)
+        if self.start_method == "fork":
+            # Construct python worker processes
+            for worker_id in range(self.num_parallel_workers):
+                worker = _MPWorker(self.operations, self.warning_ctl, self.max_rowsize, worker_id)
+                worker.start()
+                self.workers.append(worker)
+        else:
+            multiprocessing.set_start_method(self.start_method, True)
+
+            # Construct python worker processes
+            for worker_id in range(self.num_parallel_workers):
+                shared_memory = get_enable_shared_mem()
+                pipe = Pipe(self.warning_ctl, shared_memory=shared_memory, max_rowsize=self.max_rowsize)
+                self.check_interval = get_multiprocessing_timeout_interval()
+                worker = multiprocessing.Process(target=WorkerTarget(self.operations, pipe, worker_id),
+                                                 name="MapWorker" + str(worker_id), daemon=True)
+                self.workers.append(worker)
+                self.pipes.append(pipe)
+                worker.start()
+
+            multiprocessing.set_start_method("fork", True)
 
-        logger.info("Op: " + str(self.op_id) + " Python multiprocessing pool workers' PIDs: " + str(self.get_pids()))
+        logger.info("Launch worker process(es): {}".format(self.get_pids()))
 
         self.hook = _PythonMultiprocessing._ExceptHookHandler()
 
-        # The op (Map, Batch, etc) multiprocessing will launch a watch dog thread for monitoring sub processes
-        self._launch_watch_dog()
+        # Launch a clean process and register worker processes to be monitored by the watch dog.
+        self._launch_monitor()
+        self.running = True
 
-        atexit.register(self.terminate)
+        # Register a termination function using weakref to avoid the object from unable to properly destruct.
+        atexit.register(lambda cleanup: cleanup()() if cleanup() is not None else None,
+                        weakref.WeakMethod(self.terminate))
 
     def terminate(self):
-        # close watch dog first and then close all the workers
-        self.abort_watchdog()
-        self.close_all_workers()
-        if hasattr(self, "warning_ctl"):
-            del self.warning_ctl
+        if self.running:
+            # abort the monitor first and then close all the workers
+            self._abort_monitor()
+            self.close_all_workers()
+            if hasattr(self, "warning_ctl"):
+                del self.warning_ctl
+            self.running = False
 
     def get_pids(self):
         """
@@ -3596,15 +3578,48 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
 
         # todo check_iterator_cleanup
         if self.is_running() and check_iterator_cleanup() is False:
-            return self.workers[worker_id].execute(idx, *args)
+            if self.start_method == "fork":
+                return self.workers[worker_id].execute(idx, *args)
+            # spawn mode
+            self.pipes[worker_id].master_send(idx, args)
+            time_s = time.time()
+            wait_count = 1
+            while True:
+                cost_time = time.time() - time_s
+                if cost_time / self.check_interval >= wait_count:
+                    wait_count += 1
+                    logger.warning("It has been waiting for " + "%.3f" % cost_time + "s because the sub-process "
+                                   "worker of the map operation is hanging. "
+                                   "Check whether the user defined data transform is too slow or the "
+                                   "output data is too large. You can also set the timeout interval by "
+                                   "ds.config.set_multiprocessing_timeout_interval to adjust the output frequency "
+                                   "of this log.")
+                    pid = self.workers[worker_id].pid
+                    logger.warning("Map worker subprocess ID {} is stuck.".format(pid))
+                    install_status, _ = subprocess.getstatusoutput("py-spy --version")
+                    if install_status == 0:
+                        stack = subprocess.getoutput("py-spy dump -p {} -l".format(pid))
+                        logger.warning("Map worker subprocess stack:\n{}".format(stack))
+                    else:
+                        logger.warning("Please `pip install py-spy` to get the stacks of the stuck process.")
+                try:
+                    res = self.pipes[worker_id].master_receive()
+                except queue.Empty:
+                    continue
+                if res is None:
+                    # receive finish signal
+                    return None
+                if isinstance(res, ExceptionHandler):
+                    res.reraise()
+                return res
 
         return None
 
-    def _launch_watch_dog(self):
+    def _launch_monitor(self):
         """
-        We will launch a watchdog thread and a clean process to cleaning subprocess when there is process was killed.
-        The watchdog thread will cleanup subprocesses and main process when one of the subprocesses was killed.
-        The cleaning subprocess will cleanup subprocesses when main process was killed.
+        Launch a clean process and register subprocess to be monitored by the watch dog.
+        The clean process will clean up subprocesses when main process exited.
+        The watch dog will clean up subprocesses and main process when any subprocess exited.
         """
         if platform.system().lower() != 'windows':
             self.eof = multiprocessing.Event()
@@ -3613,38 +3628,45 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
                                                             args=(self.ppid, self.workers, self.eof),
                                                             daemon=True)
             self.cleaning_process.start()
+            logger.info("Launch clean process {} to monitor worker "
+                        "process(es): {}".format(self.cleaning_process.pid, self.get_pids()))
 
             if get_enable_watchdog():
-                self.eot = threading.Event()
-                self.watch_dog = threading.Thread(target=self._watch_dog,
-                                                  name="MapWatchDog",
-                                                  args=(self.eot, self.workers + [self.cleaning_process]),
-                                                  daemon=True)
-                self.watch_dog.start()
-
-    def _abort_watchdog(self):
-        if not self.eot.is_set():
-            self.eot.set()
-
-    def abort_watchdog(self):
-        if hasattr(self, 'watch_dog') and self.watch_dog is not None and hasattr(self, 'eot') and self.eot is not None:
-            self._abort_watchdog()
+                worker_ids = [worker.pid for worker in self.workers]
+                worker_ids.append(self.cleaning_process.pid)
+                cde.register_worker_pids(id(self), set(worker_ids))
+
+    def _abort_monitor(self):
+        """Deregister workers monitored by the watch dog and join clean process."""
+        if get_enable_watchdog():
+            cde.deregister_worker_pids(id(self))
+        if hasattr(self, 'eof') and self.eof is not None:
+            self.eof.set()
         if hasattr(self, 'cleaning_process') and self.cleaning_process is not None:
-            if hasattr(self, 'eof') and self.eof is not None and not self.eof.is_set():
-                self.eof.set()
-            _PythonMultiprocessing._terminate_processes([self.cleaning_process])
+            # let the quit event notify the cleaning process to exit
+            self.cleaning_process.join(timeout=5)
+            if self.cleaning_process.is_alive():
+                # if the cleaning process did not exit, it may hang, try to terminate it
+                _PythonMultiprocessing._terminate_processes([self.cleaning_process])
             del self.cleaning_process
 
     def is_running(self):
         if hasattr(self, 'workers') and self.workers is not None:
-            return all([w.is_alive() for w in self.workers])
+            if self.start_method == "fork":
+                return all([w.is_alive() for w in self.workers])
+            return all([worker_is_alive(w) for w in self.workers])
         return False
 
     def close_all_workers(self):
         """Close all the subprocess workers"""
         if hasattr(self, 'workers') and self.workers is not None:
-            for w in self.workers:
-                w.close()
+            if self.start_method == "fork":
+                for w in self.workers:
+                    w.close()
+            else:
+                for i, w in enumerate(self.workers):
+                    close_worker(w, self.pipes[i])
+
             check_interval = get_multiprocessing_timeout_interval()
             for w in self.workers:
                 try:
@@ -3660,8 +3682,12 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
                         continue
                     raise e
                 try:
-                    if w.is_alive():
-                        os.close(subprocess_file_descriptor)
+                    if self.start_method == "fork":
+                        if w.is_alive():
+                            os.close(subprocess_file_descriptor)
+                    else:
+                        if worker_is_alive(w):
+                            os.close(subprocess_file_descriptor)
                 except OSError as e:
                     # Maybe the file descriptor had been released, so ignore the 'Bad file descriptor'
                     if "Bad file descriptor" not in str(e):
@@ -3670,6 +3696,8 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
             # use clear to release the handle which is better than self.workers = None
             self.workers.clear()
             self.workers = None
+            self.pipes.clear()
+            self.pipes = None
             self.pids = None
 
 
@@ -3782,11 +3810,6 @@ class MapDataset(UnionBaseDataset):
     def __deepcopy__(self, memodict):
         return self.__safe_deepcopy__(memodict, exclude=("operations", "callbacks", "__transfer_dataset__"))
 
-    def __del__(self):
-        if hasattr(self, "process_pool") and self.process_pool is not None:
-            self.process_pool.terminate()
-            del self.process_pool
-
     @staticmethod
     def __parse_op_name(op):
         """
@@ -3915,8 +3938,9 @@ class MapDataset(UnionBaseDataset):
                     callable_list.append(op)
 
             if callable_list:
-                self.process_pool = _PythonMultiprocessing(str(self), self.num_parallel_workers, callable_list,
-                                                           self.max_rowsize)
+                self.process_pool = _PythonMultiprocessing(get_multiprocessing_start_method(),
+                                                           self.num_parallel_workers, str(self),
+                                                           callable_list, self.max_rowsize)
                 # Pass #2
                 idx = 0
                 for op in self.operations:
@@ -4142,6 +4166,7 @@ class ConcatDataset(UnionBaseDataset):
                     if isinstance(c, ConcatDataset):
                         c.use_sampler(sampler)
                     set_child(c)
+
             set_child(self)
 
             return
@@ -4242,7 +4267,7 @@ class _ToDevice:
         if get_debug_mode():
             logger.error("MindData debugger cannot be used in dataset sink mode. Please manually turn off "
                          "sink mode and try debugger again.")
-        ir_tree, self.api_tree = dataset.create_ir_tree()
+        ir_tree, _ = dataset.create_ir_tree()
 
         self._runtime_context = cde.PythonRuntimeContext()
         self._runtime_context.Init()
@@ -4442,7 +4467,7 @@ class Schema:
     Class to represent a schema of a dataset.
 
     Args:
-        schema_file (str): Path of the schema file. Default: ``None``.
+        schema_file (str, optional): Path of the schema file. Default: ``None``.
 
     Raises:
         RuntimeError: If schema file failed to load.