mindspore 2.4.10__cp39-cp39-win_amd64.whl → 2.6.0rc1__cp39-cp39-win_amd64.whl

mindspore/dataset/engine/datasets_user_defined.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2024 Huawei Technologies Co., Ltd
+# Copyright 2019-2025 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,37 +18,37 @@ You can define your own dataset loading class, and use GeneratorDataset to help
 After declaring the dataset object, you can further apply dataset operations
 (e.g. filter, skip, concat, map, batch) on it.
 """
+import atexit
 import builtins
 import copy
 import errno
 import itertools
 import math
-import os
-import signal
-import time
-from types import GeneratorType
 import multiprocessing
-from multiprocessing.util import Finalize
+import os
+import platform
 import queue
-from functools import partial
+import signal
 import subprocess
 import threading
+import time
 import weakref
-import platform
-import psutil
+from functools import partial
+from types import GeneratorType
+
+import dill
 import numpy as np
+import psutil
 
 import mindspore._c_dataengine as cde
-
-from mindspore.common import Tensor
 from mindspore import log as logger
-
-from .datasets import UnionBaseDataset, MappableDataset, Schema, to_list, _PythonMultiprocessing, _check_shm_usage
+from mindspore.common import Tensor
 from . import samplers
+from .datasets import UnionBaseDataset, MappableDataset, Schema, to_list, _PythonMultiprocessing, _check_shm_usage
 from .queue import _SharedQueue
-from .validators import check_generatordataset, check_numpyslicesdataset, check_paddeddataset
+from .validators import check_generator_dataset, check_numpy_slices_dataset, check_padded_dataset
 from ..core.config import get_enable_shared_mem, get_prefetch_size, get_multiprocessing_timeout_interval, \
-    get_enable_watchdog, get_debug_mode, get_seed, set_seed
+    get_enable_watchdog, get_debug_mode, get_seed, set_seed, get_multiprocessing_start_method
 from ..core.datatypes import mstypelist_to_detypelist
 from ..core.py_util_helpers import ExceptionHandler
 from ..core.validator_helpers import type_check
@@ -221,33 +221,31 @@ class SamplerFn(cde.PythonMultiprocessingRuntime):
         self.ppid = os.getpid()
         self.pids = []
         self.check_interval = get_multiprocessing_timeout_interval()  # the interval of check queue's size
-        self._final_join = True
 
-        # Event for end of epoch
         if self.multi_process is True:
+            multiprocessing.set_start_method(get_multiprocessing_start_method(), True)
+            # Event for end of epoch
             try:
                 self.eof = multiprocessing.Event()
             except Exception:
                 raise RuntimeError("Init multiprocessing.Event() failed, This might be caused by insufficient shm,"
                                    + " and the recommended shm size is at least 5 GB.")
-        else:
-            self.eof = threading.Event()
-        # Create workers
-
-        # get default queue size and adjust queue size per worker if there are large # workers
-        queue_size = get_prefetch_size()
-        queue_size = min(queue_size, queue_size * 4 // self.num_worker)
-        queue_size = max(2, queue_size)
-
-        if self.multi_process and get_enable_shared_mem():
-            # generator dataset use idx_queue and res_queue to transfer data between main and subprocess
-            # idx_queue is used multiprocess.Queue which is not shared memory, so it's size is 0.
-            # res_queue is used shared memory, so its size is max_rowsize which is defined by user.
-            _check_shm_usage(self.num_worker, queue_size, 0, self.max_rowsize)
-        self.count = multiprocessing.Value('i', 0)
-        for worker_id in range(self.num_worker):
-            if self.multi_process is True:
+
+            # Create workers
+            # get default queue size and adjust queue size per worker if there are large # workers
+            queue_size = get_prefetch_size()
+            queue_size = min(queue_size, queue_size * 4 // self.num_worker)
+            queue_size = max(2, queue_size)
+
+            if get_enable_shared_mem():
+                # generator dataset use idx_queue and res_queue to transfer data between main and subprocess
+                # idx_queue is used multiprocess.Queue which is not shared memory, so it's size is 0.
+                # res_queue is used shared memory, so its size is max_rowsize which is defined by user.
+                _check_shm_usage(self.num_worker, queue_size, 0, self.max_rowsize)
+            self.count = multiprocessing.Value('i', 0)
+            for worker_id in range(self.num_worker):
                 try:
+                    logger.info("Multiprocessing start method: {}".format(multiprocessing.get_start_method()))
                     worker = _GeneratorWorkerMp(self.dataset, self.eof, self.max_rowsize, queue_size, self.ppid,
                                                 self.count, worker_id)
                     worker.daemon = True
@@ -267,13 +265,26 @@ class SamplerFn(cde.PythonMultiprocessingRuntime):
                     raise RuntimeError("Failed to launch multiprocessing of GeneratorDataset: {0}".format(e))
                 self.pids.append(worker.pid)
                 self.need_join = True
-            else:
+                self.workers.append(worker)
+            multiprocessing.set_start_method("fork", True)
+
+            logger.info("Launch generator worker process(es): {}".format([worker.pid for worker in self.workers]))
+            if platform.system().lower() != 'windows':
+                self._launch_monitor()
+        else:
+            self.eof = threading.Event()
+            for worker_id in range(self.num_worker):
                 worker = _GeneratorWorkerMt(self.dataset, self.eof, worker_id)
                 worker.daemon = True
                 self.need_join = True
-            self.workers.append(worker)
-        if self.multi_process and platform.system().lower() != 'windows':
-            self._launch_cleanup_worker()
+                self.workers.append(worker)
+
+        # Register a termination function using weakref to avoid the object from unable to properly destruct.
+        atexit.register(lambda cleanup: cleanup()() if cleanup() is not None else None,
+                        weakref.WeakMethod(self.terminate))
+
+    def terminate(self):
+        self._stop_subprocess()
 
     def _interval_log(self, i, start_time, wait_count):
         cost_time = int(time.time()) - start_time
@@ -394,9 +405,11 @@ class SamplerFn(cde.PythonMultiprocessingRuntime):
                            "the `mindspore.dataset.config.set_multiprocessing_timeout_interval` interface."
         logger.warning(warning_message)
 
-    def _launch_cleanup_worker(self):
+    def _launch_monitor(self):
         """
-        We need a extra thread and process if main process or subprocess was killed.
+        Launch a clean process and register subprocess to be monitored by the watch dog.
+        The clean process will clean up subprocesses when main process exited.
+        The watch dog will clean up subprocesses and main process when any subprocess exited.
         """
         _clean_worker_func = _PythonMultiprocessing._clean_process  # pylint: disable=W0212
         self.cleaning_process = multiprocessing.Process(target=_clean_worker_func,
@@ -404,21 +417,13 @@ class SamplerFn(cde.PythonMultiprocessingRuntime):
                                                         args=(self.ppid, self.workers, self.eof))
         self.cleaning_process.daemon = True
         self.cleaning_process.start()
+        logger.info("Launch clean process {} to monitor worker "
+                    "process(es): {}".format(self.cleaning_process.pid, [worker.pid for worker in self.workers]))
 
         if get_enable_watchdog():
-            self.eot = threading.Event()
-            self.watch_dog = threading.Thread(target=_PythonMultiprocessing._watch_dog,  # pylint: disable=W0212
-                                              name="GeneratorWatchDog",
-                                              args=(self.eot, self.workers + [self.cleaning_process]))
-            self.watch_dog.daemon = True
-            self.watch_dog.start()
-
-            if self._final_join is True:
-                self._jointhread = Finalize(
-                    self.watch_dog, self._finalize_join,
-                    args=(weakref.ref(self.watch_dog), self.eot),
-                    exitpriority=-5
-                )
+            worker_ids = [worker.pid for worker in self.workers]
+            worker_ids.append(self.cleaning_process.pid)
+            cde.register_worker_pids(id(self), set(worker_ids))
 
     def _release_fd(self):
         """Release the file descriptor by subprocess"""
@@ -454,16 +459,9 @@ class SamplerFn(cde.PythonMultiprocessingRuntime):
     def _stop_subprocess(self):
         """Only the main process can call join. All the sub-process / sub-thread will be stopped."""
         if self.need_join is True and self.ppid == os.getpid():
-            # the sub-process / sub-thread will stop by self.eof.set()
-            if hasattr(self, 'eof') and self.eof is not None:
-                try:
-                    self.eof.set()
-                except AttributeError:  # maybe occur "'NoneType' object has no attribute 'maxsize'"
-                    pass
-
-            # close the watch dog first
-            self._abort_watchdog()
             self.need_join = False
+            # abort the monitor first
+            self._abort_monitor()
 
             # waiting for the sub-process stop
             for w in self.workers:
@@ -488,11 +486,17 @@ class SamplerFn(cde.PythonMultiprocessingRuntime):
 
             self.workers.clear()
             self.workers = None
+            # Under independent processes, the GeneratorDataset pulls up multiple processes in a spawn manner, and
+            # after the use case exits normally, there will be a warning: UserWarning: resource_tracker: There appear
+            # to be %d leaked semaphore objects to clean up at shutdown.
+            self.eof = None
 
-    def _abort_watchdog(self):
-        """Let watchdog quit."""
-        if hasattr(self, 'eot') and self.eot is not None and not self.eot.is_set():
-            self.eot.set()
+    def _abort_monitor(self):
+        """Deregister workers monitored by the watch dog and join clean process."""
+        if get_enable_watchdog():
+            cde.deregister_worker_pids(id(self))
+        if hasattr(self, 'eof') and self.eof is not None:
+            self.eof.set()
         if hasattr(self, 'cleaning_process') and self.cleaning_process is not None:
             # let the quit event notify the cleaning process to exit
             self.cleaning_process.join(timeout=5)
@@ -503,14 +507,6 @@ class SamplerFn(cde.PythonMultiprocessingRuntime):
         if hasattr(self, 'count'):
             del self.count
 
-    @classmethod
-    def _finalize_join(cls, twr, eot):
-        thread = twr()
-        if thread is not None:
-            if eot is not None and not eot.is_set():
-                eot.set()
-            thread.join()
-
     def __del__(self):
         try:
             self._stop_subprocess()
@@ -521,10 +517,6 @@ class SamplerFn(cde.PythonMultiprocessingRuntime):
         self.__init__(self.dataset, self.num_worker, self.multi_process, self.max_rowsize)
 
 
-def _subprocess_handle(eof, signum, frame):
-    threading.Thread(target=eof.set()).start()
-
-
 def _ignore_sigint(is_multiprocessing):
     """
     We need to ignore sigint signal here so subprocesses can exit normally and clear.
@@ -554,8 +546,7 @@ def _generator_worker_loop(dataset, idx_queue, result_queue, eof, is_multiproces
     cde.register_worker_handlers()
 
     if is_multiprocessing:
-        result_queue.cancel_join_thread()  # Ensure that the process does not hung when exiting
-        signal.signal(signal.SIGTERM, partial(_subprocess_handle, eof))
+        result_queue.cancel_join_thread()  # Ensure that the process does not hang when exiting
 
         # init the random seed and np.random seed for the subprocess
         if get_seed() != 5489:
@@ -694,6 +685,7 @@ class _GeneratorWorkerMp(multiprocessing.Process):
 
 class _GeneratorWrapper:
     """Wrapper the generator so that it can be iterated multiple times in GeneratorDataset."""
+
     def __init__(self, generator):
         self.generator = generator
         self.generator_new, self.generator = itertools.tee(self.generator)
@@ -706,6 +698,25 @@ class _GeneratorWrapper:
         return next(self.generator_new)
 
 
+class _PickleGeneratorSource:
+    """Starting multiple processes in spawn mode requires pickling source object in GeneratorDataset."""
+    def __init__(self, dataset):
+        self.dataset = dataset
+
+    def __getitem__(self, index):
+        return self.dataset[index]
+
+    def __len__(self):
+        return len(self.dataset)
+
+    def __getstate__(self):
+        state = dill.dumps(self.dataset)
+        return state
+
+    def __setstate__(self, state):
+        self.dataset = dill.loads(state)
+
+
 class GeneratorDataset(MappableDataset, UnionBaseDataset):
     """
     A source dataset that generates data from Python by invoking Python data source each epoch.
@@ -713,20 +724,29 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
     The column names and column types of generated dataset depend on Python data defined by users.
 
     Args:
-        source (Union[Callable, Iterable, Random Accessible]):
-            A generator callable object, an iterable Python object or a random accessible Python object.
-            Callable source is required to return a tuple of NumPy arrays as a row of the dataset on source().next().
-            Iterable source is required to return a tuple of NumPy arrays as a row of the dataset on
-            iter(source).next().
-            Random accessible source is required to return a tuple of NumPy arrays as a row of the dataset on
-            source[idx].
+        source (Union[Random Accessible, Iterable]): A custom dataset from which to load the data.
+            MindSpore supports the following types of datasets:
+
+            - Random-accessible (map-style) datasets: A dataset object that implements the `__getitem__()`
+              and `__len__()` methods, represents a mapping from indexes/keys to data samples.
+              For example, such a dataset `source`, when accessed with `source[idx]`, can read the idx-th sample
+              from disk, see `Random-accessible dataset example <https://www.mindspore.cn/tutorials/en/master/
+              beginner/dataset.html#random-accessible-dataset>`_ for details.
+
+            - Iterable-style dataset: An iterable dataset object that implements `__iter__()` and `__next__()` methods,
+              represents an iterable over data samples. This type of dataset is suitable for situations where
+              random reads are costly or even impossible, and where batch sizes depend on the data being acquired.
+              For example, such a dataset `source`, when accessed `iter(source)`, can return a stream of data reading
+              from a database or remote server, see `Iterable-style dataset example
+              <https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html#iterable-dataset>`_ for details.
+
         column_names (Union[str, list[str]], optional): List of column names of the dataset. Default: ``None`` .
             Users are required to provide either column_names or schema.
         column_types (list[mindspore.dtype], optional): List of column data types of the dataset. Default: ``None`` .
-            If provided, sanity check will be performed on generator output.
+            If provided, sanity check will be performed on generator output (deprecated in future version).
         schema (Union[str, Schema], optional): Data format policy, which specifies the data types and shapes of the data
             column to be read. Both JSON file path and objects constructed by :class:`mindspore.dataset.Schema` are
-            acceptable. Default: ``None`` .
+            acceptable (deprecated in future version). Default: ``None`` .
         num_samples (int, optional): The number of samples to be included in the dataset.
             Default: ``None`` , all images.
         num_parallel_workers (int, optional): Number of worker threads/subprocesses used to
@@ -737,7 +757,8 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
             input is required. Default: ``None`` , expected order behavior shown in the table below.
         num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
             Random accessible input is required. When this argument is specified, `num_samples` reflects the maximum
-            sample number of per shard.
+            sample number of per shard. Used in `data parallel training <https://www.mindspore.cn/tutorials/en/master/
+            parallel/data_parallel.html#loading-datasets>`_ .
         shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` .
             This argument must be specified only when `num_shards` is also specified.
             Random accessible input is required.
@@ -745,9 +766,15 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
             option could be beneficial if the Python operation is computational heavy. Default: ``True``.
         max_rowsize(int, optional): Maximum size of data (in MB) that is used for shared memory
             allocation to copy data between processes, the total occupied shared memory will increase as
-            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set to -1,
+            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set to ``-1``,
             shared memory will be dynamically allocated with the actual size of data. This is only used if
-            ``python_multiprocessing`` is set to True. Default: ``None`` , allocate shared memory dynamically.
+            ``python_multiprocessing`` is set to ``True``. Default: ``None`` , allocate shared memory dynamically
+            (deprecated in future version).
+        batch_sampler (Iterable, optional): Similar to `sampler` , but returns a batch of indices at a time, the
+            corresponding data will be combined into a batch. Mutually exclusive with `num_samples` , `shuffle` ,
+            `num_shards` , `shard_id` and `sampler` . Default: ``None`` , do not use batch sampler.
+        collate_fn (Callable[List[numpy.ndarray]], optional): Define how to merge a list of data into a batch.
+            Only valid if `batch_sampler` is used. Default: ``None`` , do not use collation function.
 
     Raises:
         RuntimeError: If source raises an exception during execution.
@@ -758,6 +785,11 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
         ValueError: If `num_shards` is specified but shard_id is None.
         ValueError: If shard_id is specified but `num_shards` is None.
         ValueError: If `shard_id` is not in range of [0, `num_shards` ).
+        TypeError: If `batch_sampler` is not iterable.
+        ValueError: If `batch_sampler` is specified together with `num_samples` ,
+            `shuffle` , `num_shards` , `shard_id` and `sampler`.
+        TypeError: If `collate_fn` is not callable.
+        ValueError: If `collate_fn` is specified while `batch_sampler` is None.
 
     Tutorial Examples:
         - `Load & Process Data With Dataset Pipeline
@@ -777,15 +809,35 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
           (such as Pandas, Numpy or PyArrow objects) for member variables, or load less metadata in member variables,
           or configure `python_multiprocessing=False` to use multi-threading mode.
 
-          There are several classes/functions that can help you reduce the size of member variables, and you can choose
-          to use them:
+          You can use the following classes/functions to reduce the size of member variables:
 
-          1. :class:`mindspore.dataset.utils.LineReader`: Use this class to initialize your text file object in the
+          :class:`mindspore.dataset.utils.LineReader`: Use this class to initialize your text file object in the
           `__init__` function. Then read the file content based on the line number of the object with the `__getitem__`
           function.
 
-        - Input `source` accepts user-defined Python functions (PyFuncs), Do not add network computing operators from
-          mindspore.nn and mindspore.ops or others into this `source` .
+        - Input `source` accepts user-defined Python functions (PyFuncs), and sets the multiprocessing start method
+          to `spawn` mode by ds.config.set_multiprocessing_start_method("spawn") with `python_ multiprocessing=True`
+          and `num_parallel_workers>1` supports adding network computing operators from mindspore.nn and mindspore.ops
+          or others into this `source`, otherwise adding to the `source` is not supported.
+        - When the user defined dataset by `source` calls the DVPP operator during dataset loading and processing,
+          the supported scenarios are as follows:
+
+          +---------------+----------------------------+----------------------------+----------------------------+
+          |               |                            |                     Multiprocessing                     |
+          |               |       Multithreading       +----------------------------+----------------------------+
+          |               |                            |           spawn            |            fork            |
+          +===============+============================+============================+============================+
+          |Independent    |Data Processing: support    |Data Processing: support    |Data Processing: support    |
+          |               |                            |                            |                            |
+          |process mode   |Data Processing + Network   |Data Processing + Network   |Data Processing + Network   |
+          |               |training: not support       |training: support           |training: not support       |
+          +---------------+----------------------------+----------------------------+----------------------------+
+          |Non-independent|Data Processing: support    |Data Processing: support    |Data Processing: support    |
+          |               |                            |                            |                            |
+          |process mode   |Data Processing + Network   |Data Processing + Network   |Data Processing + Network   |
+          |               |training: support           |training: support           |training: not support       |
+          +---------------+----------------------------+----------------------------+----------------------------+
+
         - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
           used in the dataset, and their effects when combined with parameter `sampler` are as follows.
 
@@ -851,10 +903,10 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
         >>> dataset = ds.GeneratorDataset(source=[(np.array(0),), (np.array(1),), (np.array(2),)], column_names=["col"])
     """
 
-    @check_generatordataset
+    @check_generator_dataset
     def __init__(self, source, column_names=None, column_types=None, schema=None, num_samples=None,
                  num_parallel_workers=1, shuffle=None, sampler=None, num_shards=None, shard_id=None,
-                 python_multiprocessing=True, max_rowsize=None):
+                 python_multiprocessing=True, max_rowsize=None, batch_sampler=None, collate_fn=None):
         super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples,
                          shuffle=shuffle, num_shards=num_shards, shard_id=shard_id)
         if isinstance(source, builtins.zip):
@@ -868,7 +920,8 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
             self.source = _GeneratorWrapper(self.source)
 
         self.prepared_source = None  # source to be sent to C++
-        if hasattr(self, 'operator_mixed') and getattr(self, 'operator_mixed') is True:
+        if hasattr(self, 'operator_mixed') and getattr(self, 'operator_mixed') is True and \
+           get_multiprocessing_start_method() == "fork":
             self.num_parallel_workers = 1
             logger.warning(
                 "Input 'source' of 'GeneratorDataset' includes network computing operators like in mindspore.nn, "
@@ -895,18 +948,23 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
             self.schema = schema
             if not isinstance(schema, Schema):
                 self.schema = Schema(schema)
+
+        self.has_batch_sampler = False
+        if batch_sampler is not None:
+            self.has_batch_sampler = True
+            if not isinstance(batch_sampler, samplers.BuiltinSampler):
+                self.sampler = samplers.IterSampler(batch_sampler)
+            else:
+                self.sampler = batch_sampler
+
         # Move get dataset_size by len from parse to here, because self.source will
         # lose attribution of '__len__' after deepcopy.
-        self.source_len = -1  # unknown
-        if hasattr(self.source, "__len__"):
-            self.source_len = len(self.source)
-
-            # if user defined sampler, update the self.source_len
-            if isinstance(self.sampler, samplers.Sampler) or hasattr(self.sampler, "__iter__"):
-                self.source_len = len(list(sampler))
+        self.source_len = len(self.source) if hasattr(self.source, "__len__") else -1
 
         self.max_rowsize = max_rowsize if max_rowsize is not None else -1
         self.sample_fn = None
+        # Ignore batch_info in the input parameter.
+        self.collate_fn = (lambda *args: collate_fn(*args[:-1])) if collate_fn is not None else None
 
     def __deepcopy__(self, memodict):
         if id(self) in memodict:
@@ -917,18 +975,20 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
         type_check(index, (int, np.number), "index")
         if not hasattr(self.source, "__getitem__"):
             raise RuntimeError("Dataset don't support randomized access.")
+        if self.has_batch_sampler:
+            raise RuntimeError("GeneratorDataset with batch_sampler does not support random access.")
         if not hasattr(self, "generator_op"):
             dataset = copy.deepcopy(self)
             self.prepared_source = _generator_fn_wrapper(_cpp_sampler_fn, self.source)
             if self.schema is None:
                 dataset.generator_node = cde.GeneratorNode(self.prepared_source, self.column_names, self.column_types,
-                                                           self.source_len, self.sampler, 1, None)
+                                                           self.source_len, self.sampler, 1, None, False)
             else:
                 schema = self.schema
                 if isinstance(schema, Schema):
                     schema = self.schema.cpp_schema
                 dataset.generator_node = cde.GeneratorNode(self.prepared_source, schema, self.source_len,
-                                                           self.sampler, 1, None)
+                                                           self.sampler, 1, None, False)
             self.generator_op = dataset.generator_node.Build()
         sample_id = self.generator_op.GetMappedIndex(index)
         return self.source[sample_id]
@@ -945,9 +1005,11 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
 
     def split(self, sizes, randomize=True):
         if hasattr(self.source, "__getitem__"):
-            # If the source has __getitem__ attribute, call the split method of MappableDataset.
-            # Otherwise, call the split method of Dataset.
-            return super().split(sizes, randomize)
+            if not self.has_batch_sampler:
+                # If the source has __getitem__ attribute, call the split method of MappableDataset.
+                # Otherwise, call the split method of Dataset.
+                return super().split(sizes, randomize)
+            logger.warning("The performance of split will be degraded since batch_sampler is detected.")
         return super(MappableDataset, self).split(sizes, randomize)
 
     def prepare_multiprocessing(self):
@@ -961,6 +1023,8 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
 
             if self.num_parallel_workers > 1 and not get_debug_mode():
                 self.__validate_memory_usage()
+                # Starting multiple processes in spawn mode requires pickling source object
+                self.source = _PickleGeneratorSource(self.source)
 
                 sample_fn = SamplerFn(self.source, self.num_parallel_workers, self.python_multiprocessing,
                                       self.max_rowsize)
@@ -984,12 +1048,12 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
         self.prepare_multiprocessing()
         if self.schema is None:
             return cde.GeneratorNode(self.prepared_source, self.column_names, self.column_types, self.source_len,
-                                     self.sampler, self.num_parallel_workers, self.sample_fn)
+                                     self.sampler, self.num_parallel_workers, self.sample_fn, self.has_batch_sampler)
         schema = self.schema
         if isinstance(schema, Schema):
             schema = self.schema.cpp_schema
         return cde.GeneratorNode(self.prepared_source, schema, self.source_len, self.sampler,
-                                 self.num_parallel_workers, self.sample_fn)
+                                 self.num_parallel_workers, self.sample_fn, self.has_batch_sampler)
 
     def __validate_memory_usage(self):
         """
@@ -1107,6 +1171,8 @@ class NumpySlicesDataset(GeneratorDataset):
             Default: ``None`` , expected order behavior shown in the table below.
         num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
             When this argument is specified, `num_samples` reflects the max sample number of per shard.
+            Used in `data parallel training <https://www.mindspore.cn/tutorials/en/master/
+            parallel/data_parallel.html#loading-datasets>`_ .
         shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This argument must be
             specified only when `num_shards` is also specified.
 
@@ -1149,7 +1215,7 @@ class NumpySlicesDataset(GeneratorDataset):
         >>> dataset = ds.NumpySlicesDataset(data=dict(df), shuffle=False)
     """
 
-    @check_numpyslicesdataset
+    @check_numpy_slices_dataset
     def __init__(self, data, column_names=None, num_samples=None, num_parallel_workers=1, shuffle=None, sampler=None,
                  num_shards=None, shard_id=None):
         dataset = _NumpySlicesDataset(data, column_names)
@@ -1202,7 +1268,7 @@ class PaddedDataset(GeneratorDataset):
         >>> dataset = ds.PaddedDataset(padded_samples=data)
     """
 
-    @check_paddeddataset
+    @check_padded_dataset
     def __init__(self, padded_samples):
         dataset = _PaddedDataset(padded_samples)
         super().__init__(dataset, column_names=dataset.column_names, num_shards=None, shard_id=None, shuffle=False)