mindspore 2.5.0__cp311-cp311-win_amd64.whl → 2.6.0__cp311-cp311-win_amd64.whl

mindspore/dataset/__init__.py

@@ -17,12 +17,12 @@ It is a `dataset engine <https://www.mindspore.cn/docs/en/master/design/data_eng
 
 This module provides the following data loading methods to help users load datasets into MindSpore.
 
-- User defined dataset loading: allows users to define `Random-accessible(Map-style) datasets
+- User defined dataset loading: allows users to define `Random-accessible(Map-style) dataset
   <https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html#random-accessible-dataset>`_ or
   `Iterable-style dataset <https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html#iterable-dataset>`_
   to customize data reading and processing logic.
 - Standard format dataset loading: support loading dataset files in standard data formats, including
-  `MindRecord <https://www.mindspore.cn/docs/en/master/model_train/dataset/record.html>`_,
+  `MindRecord <https://www.mindspore.cn/tutorials/en/master/dataset/record.html>`_,
   `TFRecord <https://tensorflow.google.cn/tutorials/load_data/tfrecord.md?hl=en>`_ .
 - Open source dataset loading: supports reading `open source datasets <#open-source>`_ ,
   such as MNIST, CIFAR-10, CLUE, LJSpeech, etc.
@@ -53,25 +53,49 @@ As shown in the above figure, the mindspore dataset module makes it easy for use
 pipelines and transform samples in the dataset in the most efficient (multi-process / multi-thread) manner.
 The specific steps are as follows:
 
-- Loading datasets: Users can easily load supported datasets using the `*Dataset` class, or load Python layer
-  customized datasets through `UDF Loader` + `GeneratorDataset` . At the same time, the loading class method can
-  accept a variety of parameters such as sampler, data slicing, and data shuffle;
-- Dataset operation: The user uses the dataset object method `.shuffle` / `.filter` / `.skip` / `.split` /
-  `.take` / ... to further shuffle, filter, skip, and obtain the maximum number of samples of datasets;
+- Loading datasets: Users can easily load supported datasets using the Dataset class
+  (`Standard-format Dataset
+  <https://www.mindspore.cn/docs/en/master/api_python/mindspore.dataset.loading.html#standard-format>`_,
+  `Vision Dataset <https://www.mindspore.cn/docs/en/master/api_python/mindspore.dataset.loading.html#vision>`_,
+  `NLP Dataset <https://www.mindspore.cn/docs/en/master/api_python/mindspore.dataset.loading.html#text>`_,
+  `Audio Dataset <https://www.mindspore.cn/docs/en/master/api_python/mindspore.dataset.loading.html#audio>`_,
+  or load Python layer customized datasets through
+  `User Defined Dataset
+  <https://www.mindspore.cn/docs/en/master/api_python/mindspore.dataset.loading.html#user-defined>`_,
+- Dataset operation: The user uses the dataset object method
+  `.shuffle <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/operation/
+  mindspore.dataset.Dataset.shuffle.html#mindspore.dataset.Dataset.shuffle>`_ /
+  `.filter <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/operation/
+  mindspore.dataset.Dataset.filter.html#mindspore.dataset.Dataset.filter>`_ /
+  `.skip <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/operation/
+  mindspore.dataset.Dataset.skip.html#mindspore.dataset.Dataset.skip>`_ /
+  `.split <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/operation/
+  mindspore.dataset.Dataset.split.html#mindspore.dataset.Dataset.split>`_ /
+  `.take <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/operation/
+  mindspore.dataset.Dataset.take.html#mindspore.dataset.Dataset.take>`_ /
+  … to further shuffle, filter, skip, and obtain the maximum number of samples of datasets.
 - Dataset sample transform operation: The user can add data transform operations
-  ( `vision transform <https://mindspore.cn/docs/en/master/api_python/mindspore.\
-  dataset.transforms.html#module-mindspore.dataset.vision>`_ ,
-  `NLP transform <https://mindspore.cn/docs/en/master/api_python/mindspore.\
-  dataset.transforms.html#module-mindspore.dataset.text>`_ ,
-  `audio transform <https://mindspore.cn/docs/en/master/api_python/mindspore.\
-  dataset.transforms.html#module-mindspore.dataset.audio>`_ ) to the map
-  operation to perform transformations. During data preprocessing, multiple map operations can be defined to
-  perform different transform operations to different fields. The data transform operation can also be a
-  user-defined transform `pyfunc` (Python function);
-- Batch: After the transformation of the samples, the user can use the batch operation to organize multiple samples
-  into batches, or use self-defined batch logic with the parameter `per_batch_map` applied;
-- Iterator: Finally, the user can use the dataset object method `create_dict_iterator` to create an
-  iterator, which can output the preprocessed data cyclically.
+  (`vision transform <https://www.mindspore.cn/docs/en/master/api_python/
+  mindspore.dataset.transforms.html#module-mindspore.dataset.vision>`_,
+  `nlp transform <https://www.mindspore.cn/docs/en/master/api_python/
+  mindspore.dataset.transforms.html#module-mindspore.dataset.text>`_,
+  `audio transform <https://www.mindspore.cn/docs/en/master/api_python/
+  mindspore.dataset.transforms.html#module-mindspore.dataset.audio>`_ )
+  to the `.map <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/operation/
+  mindspore.dataset.Dataset.map.html#mindspore.dataset.Dataset.map>`_ operation to perform transforms.
+  During data preprocessing, multiple map operations can be defined
+  to perform different transform operations to different fields.
+  The data transform operation can also be a user-defined Python function.
+- Batch: After the transforms of the samples, the user can use the
+  `.batch <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/batch/
+  mindspore.dataset.Dataset.batch.html#mindspore.dataset.Dataset.batch>`_ operation to organize multiple samples
+  into batches, or use self-defined batch logic with the parameter per_batch_map applied.
+- Iterator: Finally, user can use the method
+  `.create_dict_iterator <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/iterator/
+  mindspore.dataset.Dataset.create_dict_iterator.html#mindspore.dataset.Dataset.create_dict_iterator>`_ or
+  `.create_tuple_iterator <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/iterator/
+  mindspore.dataset.Dataset.create_tuple_iterator.html#mindspore.dataset.Dataset.create_tuple_iterator>`_ to create
+  an iterator, which can output the preprocessed data cyclically.
 
 Quick start of Dataset Pipeline
 -------------------------------

mindspore/dataset/audio/__init__.py

@@ -26,12 +26,6 @@ Common imported modules in corresponding API examples are as follows:
     import mindspore.dataset.audio as audio
     from mindspore.dataset.audio import utils
 
-Alternative and equivalent imported audio module is as follows:
-
-.. code-block::
-
-    import mindspore.dataset.audio.transforms as audio
-
 Descriptions of common data processing terms are as follows:
 
 - TensorOperation, the base class of all data processing operations implemented in C++.
@@ -41,9 +35,9 @@ The data transform operation can be executed in the data processing pipeline or
 
 - Pipeline mode is generally used to process big datasets. Examples refer to
   `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/master/api_python/
-  mindspore.dataset.html#introduction-to-data-processing-pipeline>`_ .
+  mindspore.dataset.loading.html#introduction-to-data-processing-pipeline>`_ .
 - Eager mode is more like a function call to process data. Examples refer to
-  `Lightweight Data Processing <https://www.mindspore.cn/docs/en/master/model_train/dataset/eager.html>`_ .
+  `Lightweight Data Processing <https://www.mindspore.cn/tutorials/en/master/dataset/eager.html>`_ .
 """
 from __future__ import absolute_import
 

mindspore/dataset/audio/transforms.py

@@ -1423,10 +1423,6 @@ class FrequencyMasking(AudioTensorOperation):
     Tutorial Examples:
         - `Illustration of audio transforms
           <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/audio_gallery.html>`_
-
-    .. image:: frequency_masking_original.png
-
-    .. image:: frequency_masking.png
     """
 
     @check_masking
@@ -1500,10 +1496,10 @@ class GriffinLim(AudioTensorOperation):
     Args:
         n_fft (int, optional): Size of FFT. Default: ``400``.
         n_iter (int, optional): Number of iteration for phase recovery. Default: ``32``.
-        win_length (int, optional): Window size for GriffinLim. Default: ``None``, will be set to `n_fft` .
+        win_length (int, optional): Window size for Griffin-Lim. Default: ``None``, will be set to `n_fft` .
         hop_length (int, optional): Length of hop between STFT windows.
             Default: ``None``, will be set to `win_length // 2` .
-        window_type (WindowType, optional): Window type for GriffinLim, which can be ``WindowType.BARTLETT``,
+        window_type (WindowType, optional): Window type for Griffin-Lim, which can be ``WindowType.BARTLETT``,
             ``WindowType.BLACKMAN``, ``WindowType.HAMMING``, ``WindowType.HANN`` or ``WindowType.KAISER``.
             Default: ``WindowType.HANN``. Currently kaiser window is not supported on macOS.
         power (float, optional): Exponent for the magnitude spectrogram. Default: ``2.0``.
@@ -2075,7 +2071,7 @@ class Magphase(AudioTensorOperation):
         power (float): Power of the norm, which must be non-negative. Default: ``1.0``.
 
     Raises:
-        RuntimeError: If the shape of input audio waveform does not match (..., 2).
+        RuntimeError: If the shape of input audio waveform does not match :math:`(..., 2)`.
 
     Supported Platforms:
         ``CPU``
@@ -3336,10 +3332,6 @@ class TimeMasking(AudioTensorOperation):
     Tutorial Examples:
         - `Illustration of audio transforms
           <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/audio_gallery.html>`_
-
-    .. image:: time_masking_original.png
-
-    .. image:: time_masking.png
     """
 
     @check_masking
@@ -3405,12 +3397,6 @@ class TimeStretch(AudioTensorOperation):
     Tutorial Examples:
         - `Illustration of audio transforms
           <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/audio_gallery.html>`_
-
-    .. image:: time_stretch_rate1.5.png
-
-    .. image:: time_stretch_original.png
-
-    .. image:: time_stretch_rate0.8.png
     """
 
     @check_time_stretch

mindspore/dataset/core/config.py

@@ -513,7 +513,7 @@ def set_enable_autotune(enable, filepath_prefix=None):
 
     Note:
         - Saved parameter profiles can be loaded via the :func:`mindspore.dataset.deserialize` interface to
-          directly obtain a data processing pipeline object configured with optimal parameters.
+          return a data processing pipeline object of optimal parameters.
         - The parameter tuning process can be viewed by turning on INFO level logging.
 
     An example of the generated configuration file is as follows, the "remark" field describes whether or not data
@@ -757,8 +757,8 @@ def get_auto_offload():
 
 def set_enable_watchdog(enable):
     """
-    Set the default state of watchdog Python thread as enabled, the default state of watchdog Python thread is enabled.
-    Watchdog is a thread which cleans up hanging subprocesses.
+    Set the default state of watchdog Python thread as enabled. The watchdog Python thread
+    is responsible for cleaning up stuck or hanging subprocesses, and is enabled by default.
 
     Args:
         enable (bool): Whether to launch a watchdog Python thread.

mindspore/dataset/engine/cache_client.py

@@ -27,7 +27,7 @@ class DatasetCache:
     A client to interface with tensor caching service.
 
     For details, please check
-    `Tutorial <https://www.mindspore.cn/docs/en/master/model_train/dataset/cache.html>`_ .
+    `Tutorial <https://www.mindspore.cn/tutorials/en/master/dataset/cache.html>`_ .
 
     Args:
         session_id (int): A user assigned session id for the current pipeline.

mindspore/dataset/engine/datasets.py

@@ -38,7 +38,6 @@ import warnings
 import time
 import uuid
 import multiprocessing
-from enum import Enum
 from importlib import import_module
 import sys
 import threading
@@ -55,15 +54,15 @@ from mindspore import log as logger
 from mindspore.parallel._ps_context import _is_role_pserver, _is_role_sched, _get_ps_context, \
     _enable_distributed_mindrt
 from mindspore.dataset.engine.offload import GetOffloadModel
-
+from mindspore.communication.management import get_group_size
 import mindspore.dataset.transforms.c_transforms as c_transforms
 import mindspore.dataset.transforms.py_transforms as py_transforms
 import mindspore.dataset.transforms as transforms
 from mindspore.dataset.text.utils import SentencePieceModel, DE_C_INTER_SENTENCEPIECE_MODE
-from mindspore.parallel._utils import _get_device_num
 from mindspore.dataset.debug import DebugHook
 
 from mindspore.dataset.engine import samplers
+from mindspore.dataset.engine.samplers import Shuffle
 from .iterators import DictIterator, TupleIterator, DummyIterator, check_iterator_cleanup, _set_iterator_cleanup, \
     ITERATORS_LIST, _unset_iterator_cleanup, _cleanup_the_iterators_if_created
 from .queue import _SharedQueue, _Queue
@@ -136,71 +135,6 @@ def _reset_training_dataset(global_step, dataset_size):
         raise RuntimeError("Training dataset is not set.")
 
 
-class Shuffle(str, Enum):
-    """Specify the shuffle mode.
-
-    - ``Shuffle.GLOBAL`` : Shuffle both the files and samples.
-    - ``Shuffle.FILES`` : Shuffle files only.
-    - ``Shuffle.INFILE`` : Shuffle data within each file.
-    """
-    GLOBAL: str = "global"
-    FILES: str = "files"
-    INFILE: str = "infile"
-
-
-ShuffleToShuffleMode = {Shuffle.FILES: cde.ShuffleMode.FILES,
-                        Shuffle.GLOBAL: cde.ShuffleMode.GLOBAL,
-                        Shuffle.INFILE: cde.ShuffleMode.INFILE}
-
-
-def shuffle_to_shuffle_mode(shuffle):
-    """
-    Shuffle Enum to Shuffle Mode
-
-    Args:
-        shuffle (Shuffle): shuffle flag to shuffle mode in C layer
-
-    Returns:
-        ShuffleMode, shuffle mode
-    """
-    shuffle_mode = cde.ShuffleMode.GLOBAL  # Global shuffle
-    if not isinstance(shuffle, Shuffle):
-        if shuffle is None or shuffle:
-            shuffle_mode = cde.ShuffleMode.GLOBAL  # Global shuffle
-        else:
-            shuffle_mode = cde.ShuffleMode.FALSE  # No shuffle
-    else:
-        shuffle_mode = ShuffleToShuffleMode[shuffle]
-    return shuffle_mode
-
-
-def shuffle_to_bool(shuffle):
-    """
-    Shuffle Enum to bool
-
-    Args:
-        shuffle (Shuffle): shuffle flag to bool
-
-    Returns:
-        bool, True / False
-    """
-    if shuffle is not None and not isinstance(shuffle, (bool, Shuffle)):
-        raise TypeError("shuffle must be of boolean or enum of 'Shuffle' values like 'Shuffle.GLOBAL' or "
-                        "'Shuffle.FILES' or 'Shuffle.INFILE'.")
-
-    shuffle_bool = True
-    if not isinstance(shuffle, Shuffle):
-        if shuffle is None:
-            shuffle_bool = None
-        elif shuffle:
-            shuffle_bool = True
-        else:
-            shuffle_bool = False
-    else:
-        shuffle_bool = True
-    return shuffle_bool
-
-
 @check_zip
 def zip(datasets):
     """
@@ -509,8 +443,7 @@ class Dataset:
         .. image:: bucket_batch_by_length_en.png
 
         Note:
-            - When using `Data Sinking <https://www.mindspore.cn/docs/en/master/model_train/train_process/optimize/
-              sink_mode.html#data-sinking>`_ in Graph mode, the input shape of the network should keep consistent.
+            - When using Data Sinking in Graph mode, the input shape of the network should keep consistent.
               You should set `drop_remainder` to "True" to discard the last incomplete batch of data,
               or supplement/remove samples to ensure the dataset size is divisible by `batch_size`.
 
@@ -575,7 +508,7 @@ class Dataset:
     @check_batch
     def batch(self, batch_size, drop_remainder=False, num_parallel_workers=None, **kwargs):
         """
-        Combine batch_size number of consecutive rows into batch which apply per_batch_map to the samples first.
+        Combine `batch_size` number of consecutive rows into batch which apply `per_batch_map` to the samples first.
 
         For any column, all the elements within that column must have the same shape.
 
@@ -586,8 +519,7 @@ class Dataset:
         Note:
             - The order of using repeat and batch reflects the number of batches and per_batch_map.
               It is recommended that the repeat operation applied after the batch operation finished.
-            - When using `Data Sinking <https://www.mindspore.cn/docs/en/master/model_train/train_process/optimize/
-              sink_mode.html#data-sinking>`_ in Graph mode, the input shape of the network should keep consistent.
+            - When using Data Sinking in Graph mode, the input shape of the network should keep consistent.
               You should set `drop_remainder` to "True" to discard the last incomplete batch of data,
               or supplement/remove samples to ensure the dataset size is divisible by `batch_size`.
 
@@ -629,13 +561,19 @@ class Dataset:
 
                 - max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
                   allocation to copy data between processes, the total occupied shared memory will increase as
-                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set
-                  to -1, shared memory will be dynamically allocated with the actual size of data. This is only used if
-                  ``python_multiprocessing`` is set to True. If it is an int value, it represents
-                  ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
-                  If it is a list, the first element represents the ``input_columns`` use this value as the unit to
-                  create shared memory, and the second element represents ``output_columns`` use this value as the unit
-                  to create shared memory. Default: ``None`` , allocate shared memory dynamically.
+                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase.
+                  This is only used if ``python_multiprocessing`` is set to ``True``.
+                  Default: ``None`` , allocate shared memory dynamically (deprecated in future version).
+
+                  - If set to ``-1`` / ``None``, shared memory will be dynamically allocated with the
+                    actual size of data.
+
+                  - If it is an int value, it represents ``input_columns`` and ``output_columns`` use this value as the
+                    unit to create shared memory.
+
+                  - If it is a list, represents the ``input_columns`` use the first element as the unit to
+                    create shared memory, and represents ``output_columns`` use the second element as the
+                    unit to create shared memory.
 
         Returns:
             Dataset, a new dataset with the above operation applied.
@@ -683,8 +621,7 @@ class Dataset:
         Note:
             - The order of using repeat and padded_batch reflects the number of batches.
               It is recommended that the repeat operation applied after the padded_batch operation finished.
-            - When using `Data Sinking <https://www.mindspore.cn/docs/en/master/model_train/train_process/optimize/
-              sink_mode.html#data-sinking>`_ in Graph mode, the input shape of the network should keep consistent.
+            - When using Data Sinking in Graph mode, the input shape of the network should keep consistent.
               You should set `drop_remainder` to "True" to discard the last incomplete batch of data,
               or supplement/remove samples to ensure the dataset size is divisible by `batch_size`.
 
@@ -738,9 +675,9 @@ class Dataset:
 
         Args:
             condition_name (str): The condition name that is used to toggle sending next row.
-            num_batch (int): the number of batches without blocking at the start of each epoch.
+            num_batch (int, optional): the number of batches without blocking at the start of each epoch.
                 Default: ``1``.
-            callback (function): The callback function that will be invoked when sync_update is called.
+            callback (function, optional): The callback function that will be invoked when sync_update is called.
                 Default: ``None``.
 
         Returns:
@@ -925,15 +862,21 @@ class Dataset:
                 - python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker processes.
                   This option could be beneficial if the Python operation is computational heavy. Default: ``False``.
 
-                - max_rowsize (Union[int, list[int]], optional): Maximum size of row in MB that is used for shared
-                  memory allocation to copy data between processes, the total occupied shared memory will increase as
-                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set
-                  to -1, shared memory will be dynamically allocated with the actual size of data. This is only used if
-                  ``python_multiprocessing`` is set to True. If it is an int value, it represents
-                  ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
-                  If it is a list, the first element represents the ``input_columns`` use this value as the unit to
-                  create shared memory, and the second element represents ``output_columns`` use this value as the unit
-                  to create shared memory. Default: ``None`` , allocate shared memory dynamically.
+                - max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
+                  allocation to copy data between processes, the total occupied shared memory will increase as
+                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase.
+                  This is only used if ``python_multiprocessing`` is set to ``True``.
+                  Default: ``None`` , allocate shared memory dynamically (deprecated in future version).
+
+                  - If set to ``-1`` / ``None``, shared memory will be dynamically allocated with the
+                    actual size of data.
+
+                  - If it is an int value, it represents ``input_columns`` and ``output_columns`` use this value as the
+                    unit to create shared memory.
+
+                  - If it is a list, the first element represents the ``input_columns`` use this value as the unit to
+                    create shared memory, and the second element represents ``output_columns`` use this value as the
+                    unit to create shared memory.
 
                 - cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
                   Default: ``None``, which means no cache is used.
@@ -946,8 +889,29 @@ class Dataset:
         Note:
             - Input `operations` accepts TensorOperations defined in mindspore.dataset part, plus user-defined
               Python functions (PyFuncs).
-            - Do not add network computing operators from mindspore.nn and mindspore.ops or others into this
-              `operations` .
+            - Setting the start method of multiprocessing to `spawn` mode by
+              ds.config.set_multiprocessing_start_method("spawn") with `python_ multiprocessing=True`
+              and `num_parallel_workers>1` supports adding network computing operators from mindspore.nn and
+              mindspore.ops or other network computing operators into this `operations` .
+              Otherwise, adding to `operations` is not supported.
+            - Currently only some scenarios support calling DVPP operators in Python functions passed in with the
+              `operations` parameter:
+
+              +---------------+----------------------------+----------------------------+----------------------------+
+              |               |                            |                     Multiprocessing                     |
+              |               |       Multithreading       +----------------------------+----------------------------+
+              |               |                            |           spawn            |            fork            |
+              +===============+============================+============================+============================+
+              |Independent    |Data Processing: support    |Data Processing: support    |Data Processing: support    |
+              |               |                            |                            |                            |
+              |process mode   |Data Processing + Network   |Data Processing + Network   |Data Processing + Network   |
+              |               |training: not support       |training: support           |training: not support       |
+              +---------------+----------------------------+----------------------------+----------------------------+
+              |Non-independent|Data Processing: support    |Data Processing: support    |Data Processing: support    |
+              |               |                            |                            |                            |
+              |process mode   |Data Processing + Network   |Data Processing + Network   |Data Processing + Network   |
+              |               |training: support           |training: support           |training: not support       |
+              +---------------+----------------------------+----------------------------+----------------------------+
 
         Returns:
             Dataset, a new dataset with the above operation applied.
@@ -1771,7 +1735,7 @@ class Dataset:
         Get the shapes of output data.
 
         Args:
-            estimate (bool): If `estimate` is ``False`` , will return the shapes of first data row.
+            estimate (bool, optional): If `estimate` is ``False`` , will return the shapes of first data row.
                 Otherwise, will iterate the whole dataset and return the estimated shapes of data row,
                 where dynamic shape is marked as None (used in dynamic data shapes scenario).
                 Default: ``False`` .
@@ -2348,10 +2312,10 @@ class SourceDataset(Dataset):
         self.shard_id = replace_none(shard_id, 0)
 
         if shuffle is not None and not isinstance(shuffle, (bool, Shuffle)):
-            raise TypeError("shuffle must be of boolean or enum of 'Shuffle' values like 'Shuffle.GLOBAL' or "
-                            "'Shuffle.FILES' or 'Shuffle.INFILE'.")
+            raise TypeError("shuffle must be of boolean or enum of 'Shuffle' values like 'Shuffle.ADAPTIVE' or "
+                            "'Shuffle.GLOBAL' or 'Shuffle.PARTIAL' or 'Shuffle.FILES' or 'Shuffle.INFILE'.")
 
-        self.shuffle_flag = 2  # Global shuffle
+        self.shuffle_flag = 5  # Adaptive shuffle
         if not isinstance(shuffle, Shuffle):
             if shuffle is None or shuffle:
                 self.shuffle_flag = 2  # Global shuffle
@@ -2364,6 +2328,10 @@ class SourceDataset(Dataset):
                 self.shuffle_flag = 1  # Files shuffle
             elif shuffle == Shuffle.INFILE:
                 self.shuffle_flag = 3  # Infile shuffle
+            elif shuffle == Shuffle.ADAPTIVE:
+                self.shuffle_flag = 5
+            elif shuffle == Shuffle.PARTIAL:
+                self.shuffle_flag = 4
 
     def parse(self, children=None):
         raise NotImplementedError("Dataset has to implement parse method.")
@@ -2420,15 +2388,23 @@ class MappableDataset(SourceDataset):
     def __init__(self, num_parallel_workers=None, sampler=None, num_samples=None, shuffle=None, num_shards=None,
                  shard_id=None, cache=None):
         num_shards, shard_id = self._update_data_shard(num_shards, shard_id)
+        if sampler is None:
+            if shuffle is None or shuffle is True:
+                shuffle = Shuffle.GLOBAL
+            elif shuffle is False:
+                shuffle = Shuffle.FALSE
         super().__init__(num_parallel_workers=num_parallel_workers, num_samples=num_samples, shuffle=shuffle,
                          num_shards=num_shards, shard_id=shard_id, cache=cache)
-        self.shuffle_flag = replace_none(shuffle, True)
         self.sampler = samplers.select_sampler(num_samples, sampler, shuffle, num_shards, shard_id)
 
     def add_sampler(self, new_sampler):
         """
         Add a child sampler for the current dataset.
 
+        Note:
+            - If the sampler is added and it has a shuffle option, its value must be ``Shuffle.GLOBAL`` .
+              Additionally, the original sampler's shuffle value cannot be ``Shuffle.PARTIAL`` .
+
         Args:
             new_sampler (Sampler): The child sampler to be added.
 
@@ -2442,6 +2418,16 @@ class MappableDataset(SourceDataset):
         # Note: By adding a sampler, the sampled IDs will flow to the new_sampler
         # after first passing through the current samplers attached to this dataset.
         self.dataset_size = None
+
+        if self.sampler is not None and self.sampler.get_shuffle_mode() == Shuffle.PARTIAL:
+            raise RuntimeError("When multiple samplers are used, ensure that the shuffle of the current sampler "
+                               "must not be Shuffle.PARTIAL.")
+
+        if new_sampler.get_shuffle_mode() != Shuffle.GLOBAL and new_sampler.get_shuffle_mode() != Shuffle.FALSE:
+            raise RuntimeError("When multiple samplers are used, ensure that the shuffle of the input sampler "
+                               "must be Shuffle.FALSE or Shuffle.GLOBAL, but got: {}."
+                               .format(new_sampler.get_shuffle_mode()))
+
         new_sampler.add_child(self.sampler)
         self.sampler = new_sampler
 
@@ -2604,7 +2590,7 @@ def _check_shm_usage(num_worker, queue_size, in_rowsize, out_rowsize):
     threshold_ratio = 0.8
     # Verify available size only when using static shared memory on Linux
     if platform.system().lower() not in {"windows", "darwin"} and in_rowsize != -1 and out_rowsize != -1:
-        device_num = _get_device_num()
+        device_num = get_group_size()
         # In the cluster, _get_device_num indicates the number of the entire cluster. The maximum number of cards
         # on the ascend server is 8.
         if device_num > 1:
@@ -2690,11 +2676,6 @@ class BatchDataset(UnionBaseDataset):
         else:
             self.max_rowsize = [max_rowsize[0] * self.batch_size, max_rowsize[1] * self.batch_size]
 
-    def __del__(self):
-        if hasattr(self, "process_pool") and self.process_pool is not None:
-            self.process_pool.terminate()
-            del self.process_pool
-
     def parse(self, children=None):
         return cde.BatchNode(children[0], self.batch_size, self.drop_remainder, False, self.input_columns,
                              self.output_columns, self.batch_size_func, self.per_batch_map, {},
@@ -3033,7 +3014,7 @@ class SyncWaitDataset(UnionBaseDataset):
 
 class ShuffleDataset(UnionBaseDataset):
     """
-    The result of applying Shuffle operation to the input Dataset.
+    The result of applying shuffle operation to the input Dataset.
 
     Args:
         input_dataset (Dataset): Input Dataset to be shuffled.
@@ -3387,6 +3368,7 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         # cache thread (get_ident()) to worker_id mapping in Python layer
         self.python_threads_to_workers = {}
         self.eof = None
+        self.running = False
 
     def __del__(self):
         try:
@@ -3527,15 +3509,20 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
 
         # Launch a clean process and register worker processes to be monitored by the watch dog.
         self._launch_monitor()
+        self.running = True
 
-        atexit.register(self.terminate)
+        # Register a termination function using weakref to avoid the object from unable to properly destruct.
+        atexit.register(lambda cleanup: cleanup()() if cleanup() is not None else None,
+                        weakref.WeakMethod(self.terminate))
 
     def terminate(self):
-        # abort the monitor first and then close all the workers
-        self._abort_monitor()
-        self.close_all_workers()
-        if hasattr(self, "warning_ctl"):
-            del self.warning_ctl
+        if self.running:
+            # abort the monitor first and then close all the workers
+            self._abort_monitor()
+            self.close_all_workers()
+            if hasattr(self, "warning_ctl"):
+                del self.warning_ctl
+            self.running = False
 
     def get_pids(self):
         """
@@ -3823,11 +3810,6 @@ class MapDataset(UnionBaseDataset):
     def __deepcopy__(self, memodict):
         return self.__safe_deepcopy__(memodict, exclude=("operations", "callbacks", "__transfer_dataset__"))
 
-    def __del__(self):
-        if hasattr(self, "process_pool") and self.process_pool is not None:
-            self.process_pool.terminate()
-            del self.process_pool
-
     @staticmethod
     def __parse_op_name(op):
         """
@@ -4285,7 +4267,7 @@ class _ToDevice:
         if get_debug_mode():
             logger.error("MindData debugger cannot be used in dataset sink mode. Please manually turn off "
                          "sink mode and try debugger again.")
-        ir_tree, self.api_tree = dataset.create_ir_tree()
+        ir_tree, _ = dataset.create_ir_tree()
 
         self._runtime_context = cde.PythonRuntimeContext()
         self._runtime_context.Init()
@@ -4485,7 +4467,7 @@ class Schema:
     Class to represent a schema of a dataset.
 
     Args:
-        schema_file (str): Path of the schema file. Default: ``None``.
+        schema_file (str, optional): Path of the schema file. Default: ``None``.
 
     Raises:
         RuntimeError: If schema file failed to load.