mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/dataset/engine/datasets.py

@@ -1,4 +1,4 @@
-# Copyright 2022-2023 Huawei Technologies Co., Ltd
+# Copyright 2022-2024 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,13 +16,13 @@
 1. This file is an abstraction of the dataset loading class. It contains
 some basic dataset operations(skip, filter, map, batch, ...).
 2. Specific dataset loading classes can be found in datasets_vision.py, datasets_text.py,
-datasets_audio.py, datasets_standard_format.py and dataets_user_defined.py files.
+datasets_audio.py, datasets_standard_format.py and datasets_user_defined.py files.
     datasets_vision.py: contains vision dataset loading classes.
     datasets_text.py: contains text dataset loading classes.
     datasets_audio.py: contains audio dataset loading classes.
     datasets_standard_format.py: contains standard format loading classes which
                                  any other kinds of datasets can be converted to.
-    dataets_user_defined.py: contains basic classes that help users to define
+    datasets_user_defined.py: contains basic classes that help users to define
                              flexible ways to load dataset.
 """
 import atexit
@@ -66,12 +66,13 @@ from mindspore.dataset.debug import DebugHook
 
 from mindspore.dataset.engine import samplers
 from .iterators import DictIterator, TupleIterator, DummyIterator, check_iterator_cleanup, _set_iterator_cleanup, \
-    ITERATORS_LIST, _unset_iterator_cleanup
+    ITERATORS_LIST, _unset_iterator_cleanup, _cleanup_the_iterators_if_created
 from .queue import _SharedQueue, _Queue
 from .validators import check_batch, check_shuffle, check_map, check_filter, check_repeat, check_skip, check_zip, \
     check_rename, check_device_send, check_take, check_output_shape, check_project, \
     check_sync_wait, check_zip_dataset, check_add_column, check_concat, check_split, check_bucket_batch_by_length, \
-    check_save, check_tuple_iterator, check_dict_iterator, check_schema, check_to_device_send, check_padded_batch
+    check_save, check_tuple_iterator, check_dict_iterator, check_schema, check_to_device_send, check_padded_batch, \
+    check_total_batch, check_sync_update
 from ..core.config import get_callback_timeout, _init_device_info, get_enable_shared_mem, get_num_parallel_workers, \
     get_enable_watchdog, get_seed, set_seed, get_debug_mode, get_multiprocessing_timeout_interval, _get_debug_hook_list
 from ..core.datatypes import mstype_to_detype
@@ -79,6 +80,7 @@ from ..core.validator_helpers import replace_none
 from ..core.py_util_helpers import ExceptionHandler
 from ..transforms.py_transforms_util import FuncWrapper, Implementation
 from ..vision.transforms import ToNumpy
+from ...mindrecord.config import _get_enc_key, _get_enc_mode, _get_hash_mode, encrypt, append_hash_to_file
 
 try:
     context = import_module("mindspore.context")
@@ -492,6 +494,12 @@ class Dataset:
 
         .. image:: bucket_batch_by_length_en.png
 
+        Note:
+            - When using `Data Sinking <https://www.mindspore.cn/docs/en/master/model_train/train_process/optimize/
+              sink_mode.html#data-sinking>`_ in Graph mode, the input shape of the network should keep consistent.
+              You should set `drop_remainder` to "True" to discard the last incomplete batch of data,
+              or supplement/remove samples to ensure the dataset size is divisible by `batch_size`.
+
         Args:
             column_names (list[str]): Columns passed to element_length_function.
             bucket_boundaries (list[int]): A list consisting of the upper boundaries
@@ -562,8 +570,12 @@ class Dataset:
         .. image:: batch_en.png
 
         Note:
-            The order of using repeat and batch reflects the number of batches and per_batch_map.
-            It is recommended that the repeat operation applied after the batch operation finished.
+            - The order of using repeat and batch reflects the number of batches and per_batch_map.
+              It is recommended that the repeat operation applied after the batch operation finished.
+            - When using `Data Sinking <https://www.mindspore.cn/docs/en/master/model_train/train_process/optimize/
+              sink_mode.html#data-sinking>`_ in Graph mode, the input shape of the network should keep consistent.
+              You should set `drop_remainder` to "True" to discard the last incomplete batch of data,
+              or supplement/remove samples to ensure the dataset size is divisible by `batch_size`.
 
         Args:
             batch_size (Union[int, Callable]): The number of rows each batch is created with. An
@@ -596,19 +608,20 @@ class Dataset:
                   name as the input columns, i.e., the columns will be replaced.
 
                 - python_multiprocessing (bool, optional): Parallelize Python function `per_batch_map` with
-                  multi-processing or multi-threading mode, ``True`` means multi-processing,
-                  ``False`` means multi-threading If `per_batch_map` is a I/O bound task, use
-                  multi-threading mode. If `per_batch_map` is a CPU bound task, it is recommended to use
-                  multi-processing mode. Default: ``False`` , use python multi-threading mode.
+                  multiprocessing or multithreading mode, ``True`` means multiprocessing,
+                  ``False`` means multithreading If `per_batch_map` is a I/O bound task, use
+                  multithreading mode. If `per_batch_map` is a CPU bound task, it is recommended to use
+                  multiprocessing mode. Default: ``False`` , use python multithreading mode.
 
                 - max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
                   allocation to copy data between processes, the total occupied shared memory will increase as
-                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
-                  used if python_multiprocessing is set to True. If it is an int value, it represents
+                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set
+                  to -1, shared memory will be dynamically allocated with the actual size of data. This is only used if
+                  ``python_multiprocessing`` is set to True. If it is an int value, it represents
                   ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
                   If it is a list, the first element represents the ``input_columns`` use this value as the unit to
                   create shared memory, and the second element represents ``output_columns`` use this value as the unit
-                  to create shared memory. Default: 16.
+                  to create shared memory. Default: ``None`` , allocate shared memory dynamically.
 
         Returns:
             Dataset, a new dataset with the above operation applied.
@@ -654,8 +667,12 @@ class Dataset:
         .. image:: padded_batch_en.png
 
         Note:
-            The order of using repeat and padded_batch reflects the number of batches.
-            It is recommended that the repeat operation applied after the padded_batch operation finished.
+            - The order of using repeat and padded_batch reflects the number of batches.
+              It is recommended that the repeat operation applied after the padded_batch operation finished.
+            - When using `Data Sinking <https://www.mindspore.cn/docs/en/master/model_train/train_process/optimize/
+              sink_mode.html#data-sinking>`_ in Graph mode, the input shape of the network should keep consistent.
+              You should set `drop_remainder` to "True" to discard the last incomplete batch of data,
+              or supplement/remove samples to ensure the dataset size is divisible by `batch_size`.
 
         Args:
             batch_size (Union[int, Callable]): The number of rows each batch is created with. An
@@ -666,15 +683,17 @@ class Dataset:
                 be dropped and not propagated to the child node.
             num_parallel_workers (int, optional): Number of workers(threads) to process the dataset in parallel.
                 Default: ``None``.
-            pad_info (dict, optional): The information about how to batch each column. The key
+            pad_info (dict, optional): The pad information about how to batch each column. The key
                 corresponds to the column name, and the value must be a tuple of 2 elements.
                 The first element corresponds to the shape to pad to, and the second
                 element corresponds to the value to pad with. If a column is not
                 specified, then that column will be padded to the longest in the current
-                batch, and 0 will be used as the padding value. Any None dimensions will
-                be padded to the longest in the current batch, unless if
-                pad_to_bucket_boundary is True. If no padding is wanted, set `pad_info`
-                to ``None``. Default: ``None``.
+                batch, and 0 will be used as the padding value. If ``pad_info={"col1": ([224, 224], 0)}``,
+                expand the data column named ``col1`` to shape (224, 224), and fill in the missing values with 0.
+                If ``pad_info={}``, all samples in the batch will be filled to the shape with the largest sample
+                in the current batch. If ``pad_info={"col1": (None, 100)}``, all samples in the batch will be filled
+                to the shape with the largest sample in the current batch, and fill in the missing values with 100.
+                If no padding is wanted, set `pad_info` to ``None``. Default: ``None``.
 
         Returns:
             Dataset, a new dataset with the above operation applied.
@@ -858,11 +877,11 @@ class Dataset:
         `output_columns` , and if not specified, the column name of output column is same as that of `input_columns` .
 
         - If you use transformations (
-          `vision transform <https://mindspore.cn/docs/en/r2.2/api_python/mindspore.\
+          `vision transform <https://mindspore.cn/docs/en/master/api_python/mindspore.\
           dataset.transforms.html#module-mindspore.dataset.vision>`_ ,
-          `nlp transform <https://mindspore.cn/docs/en/r2.2/api_python/mindspore.\
+          `nlp transform <https://mindspore.cn/docs/en/master/api_python/mindspore.\
           dataset.transforms.html#module-mindspore.dataset.text>`_ ,
-          `audio transform <https://mindspore.cn/docs/en/r2.2/api_python/mindspore.\
+          `audio transform <https://mindspore.cn/docs/en/master/api_python/mindspore.\
           dataset.transforms.html#module-mindspore.dataset.audio>`_ )
           provided by mindspore dataset, please use the following parameters:
 
@@ -894,12 +913,13 @@ class Dataset:
 
                 - max_rowsize (Union[int, list[int]], optional): Maximum size of row in MB that is used for shared
                   memory allocation to copy data between processes, the total occupied shared memory will increase as
-                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
-                  used if python_multiprocessing is set to True. If it is an int value, it represents
+                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set
+                  to -1, shared memory will be dynamically allocated with the actual size of data. This is only used if
+                  ``python_multiprocessing`` is set to True. If it is an int value, it represents
                   ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
                   If it is a list, the first element represents the ``input_columns`` use this value as the unit to
                   create shared memory, and the second element represents ``output_columns`` use this value as the unit
-                  to create shared memory. Default: 16.
+                  to create shared memory. Default: ``None`` , allocate shared memory dynamically.
 
                 - cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
                   Default: ``None``, which means no cache is used.
@@ -983,8 +1003,8 @@ class Dataset:
             num_parallel_workers = 1
             logger.warning(
                 "Input 'operations' of 'map' includes network computing operators like in mindspore.nn, mindspore.ops, "
-                "mindspore.numpy module and etc, which do not support multi-thread compiling, recommend to replace it "
-                "with python implemented operator like numpy etc. Here decrease 'num_parallel_workers' into 1.")
+                "mindspore.numpy module and etc, which do not support multithreading compiling, recommend to replace "
+                "it with python implemented operator like numpy etc. Here decrease 'num_parallel_workers' into 1.")
 
         return MapDataset(self, operations, input_columns, output_columns, num_parallel_workers, **kwargs)
 
@@ -1030,20 +1050,28 @@ class Dataset:
 
         Examples:
             >>> import mindspore.dataset as ds
+            >>>
+            >>> # Create a dataset with 10 elements
             >>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
+            >>> ori_size = dataset.get_dataset_size()
             >>>
-            >>> # Create a dataset where the dataset is repeated for 50 epochs
+            >>> # Repeat the dataset 50 times.
             >>> dataset = dataset.repeat(50)
+            >>> repeated_size = dataset.get_dataset_size()
+            >>> print("ori_size", ori_size, ", repeated_size", repeated_size)
+            ori_size 10 , repeated_size 500
             >>>
-            >>> # Create a dataset where each epoch is shuffled individually
-            >>> dataset = dataset.shuffle(10)
-            >>> dataset = dataset.repeat(50)
+            >>> # Since the original dataset size is less than batch_size, thus no data is returned
+            >>> dataset1 = ds.GeneratorDataset([i for i in range(10)], "column1")
+            >>> dataset1 = dataset1.batch(batch_size=20, drop_remainder=True)
+            >>> dataset1 = dataset1.repeat(6)
             >>>
-            >>> # Create a dataset where the dataset is first repeated for
-            >>> # 50 epochs before shuffling. The shuffle operation will treat
-            >>> # the entire 50 epochs as one big dataset.
-            >>> dataset = dataset.repeat(50)
-            >>> dataset = dataset.shuffle(10)
+            >>> # Repeat the original dataset to 60 elements, thus 3 batches are returned
+            >>> dataset2 = ds.GeneratorDataset([i for i in range(10)], "column1")
+            >>> dataset2 = dataset2.repeat(6)
+            >>> dataset2 = dataset2.batch(batch_size=20, drop_remainder=True)
+            >>> print("dataset1 size", dataset1.get_dataset_size(), ", dataset2 size", dataset2.get_dataset_size())
+            dataset1 size 0 , dataset2 size 3
         """
         return RepeatDataset(self, count)
 
@@ -1462,14 +1490,14 @@ class Dataset:
              - Type in `mindrecord`
              - Details
            * - bool
-             - None
-             - Not supported
+             - int32
+             - transform to int32
            * - int8
              - int32
              -
            * - uint8
-             - bytes(1D uint8)
-             - Drop dimension
+             - int32
+             -
            * - int16
              - int32
              -
@@ -1486,8 +1514,8 @@ class Dataset:
              - int64
              -
            * - uint64
-             - None
-             - Not supported
+             - int64
+             - Maybe reverse
            * - float16
              - float32
              -
@@ -1500,15 +1528,17 @@ class Dataset:
            * - string
              - string
              - Multi-dimensional string not supported
+           * - bytes
+             - bytes
+             - Multi-dimensional bytes not supported
 
         Note:
             1. To save the samples in order, set dataset's `shuffle` to ``False`` and `num_files` to ``1``.
             2. Before calling the function, do not use batch operation, repeat operation or data augmentation operations
                with random attribute in map operation.
             3. When array dimension is variable, one-dimensional arrays or
-               multi-dimensional arrays with variable dimension 0 are supported.
-            4. MindRecord does not support uint64, multi-dimensional uint8(drop dimension) nor
-               multi-dimensional string.
+               multidimensional arrays with variable dimension 0 are supported.
+            4. MindRecord does not support multidimensional string or multidimensional bytes.
 
         Args:
             file_name (str): Path to dataset file.
@@ -1527,6 +1557,10 @@ class Dataset:
             >>> d1 = ds.GeneratorDataset(generator_1d, ["data"], shuffle=False)
             >>> d1.save('/path/to/save_file')
         """
+        if (_get_enc_key() is not None or _get_hash_mode() is not None) and num_files > 1:
+            raise RuntimeError("When encode mode or hash check is enabled, " +
+                               "the automatic sharding function is unavailable.")
+
         ir_tree, api_tree = self.create_ir_tree()
 
         runtime_context = cde.PythonRuntimeContext()
@@ -1536,38 +1570,49 @@ class Dataset:
         runtime_context.AssignConsumer(consumer)
 
         consumer.Save()
+
+        if _get_hash_mode() is not None:
+            append_hash_to_file(file_name)
+            append_hash_to_file(file_name + ".db")
+
+        if _get_enc_key() is not None:
+            encrypt(file_name, _get_enc_key(), _get_enc_mode())
+            encrypt(file_name + ".db", _get_enc_key(), _get_enc_mode())
+
         _set_dataset_permissions(file_name, num_files)
         del api_tree
 
     @check_tuple_iterator
     def create_tuple_iterator(self, columns=None, num_epochs=-1, output_numpy=False, do_copy=True):
         """
-        Create an iterator over the dataset. The datatype retrieved back will be a list of `numpy.ndarray` .
-
-        To specify which columns to list and the order needed, use columns_list. If columns_list
-        is not provided, the order of the columns will remain unchanged.
+        Create an iterator over the dataset that yields samples of type list, whose elements are
+        the data for each column.
 
         Args:
-            columns (list[str], optional): List of columns to be used to specify the order of columns.
-                Default: ``None``, means all columns.
-            num_epochs (int, optional): Maximum number of epochs that iterator can be iterated.
-                Default: ``-1``, iterator can be iterated infinite number of epochs.
-            output_numpy (bool, optional): Whether or not to output NumPy datatype.
-                If `output_numpy` is ``False``, iterator will output MSTensor. Default: ``False``.
-            do_copy (bool, optional): When output data type is :class:`mindspore.Tensor`,
-                use this param to select the conversion method, only take False for better performance.
-                Default: ``True``.
+            columns (list[str], optional): Specify the output columns and the order.
+                Default: ``None``, keep all the output columns and their original order.
+            num_epochs (int, optional): The number of epochs to iterate over the entire dataset.
+                Default: ``-1`` , the dataset can be iterated indefinitely.
+            output_numpy (bool, optional): Whether to keep the output data as NumPy ndarray, or
+                convert it to Tensor. Default: ``False`` .
+            do_copy (bool, optional): Whether to copy the data when converting output to Tensor,
+                or reuse the buffer for better performance, only works when `output_numpy` is ``False`` .
+                Default: ``True`` .
 
         Returns:
-            Iterator, a dataset iterator that returns data of type Tuple.
+            Iterator, a dataset iterator that yields samples of type list.
 
         Examples:
             >>> import mindspore.dataset as ds
-            >>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
-            >>> iterator = dataset.create_tuple_iterator()
-            >>> for item in iterator:
-            ...     # item is a list
-            ...     print(type(item))
+            >>>
+            >>> dataset = ds.GeneratorDataset([i for i in range(10)], "data")
+            >>> num_epochs = 3
+            >>> iterator = dataset.create_tuple_iterator(num_epochs=num_epochs)
+            >>> for epoch in range(num_epochs):
+            ...     for item in iterator:
+            ...         # output is of type tuple
+            ...         print(type(item))
+            ...         break
             ...     break
             <class 'list'>
         """
@@ -1581,27 +1626,32 @@ class Dataset:
     @check_dict_iterator
     def create_dict_iterator(self, num_epochs=-1, output_numpy=False, do_copy=True):
         """
-        Create an iterator over the dataset. The data retrieved will be a dictionary datatype.
+        Create an iterator over the dataset that yields samples of type dict,
+        while the key is the column name and the value is the data.
 
         Args:
-            num_epochs (int, optional): Maximum number of epochs that iterator can be iterated.
-                Default: ``-1`` , iterator can be iterated infinite number of epochs.
-            output_numpy (bool, optional): Whether or not to output NumPy datatype,
-                if `output_numpy` is ``False``, iterator will output MSTensor. Default: ``False`` .
-            do_copy (bool, optional): When output data type is :class:`mindspore.Tensor`,
-                use this param to select the conversion method, only take False for better performance.
+            num_epochs (int, optional): The number of epochs to iterate over the entire dataset.
+                Default: ``-1`` , the dataset can be iterated indefinitely.
+            output_numpy (bool, optional): Whether to keep the output data as NumPy ndarray, or
+                convert it to Tensor. Default: ``False`` .
+            do_copy (bool, optional): Whether to copy the data when converting output to Tensor,
+                or reuse the buffer for better performance, only works when `output_numpy` is ``False`` .
                 Default: ``True`` .
 
         Returns:
-            Iterator, a dataset iterator that returns data of type Dict.
+            Iterator, a dataset iterator that yields samples of type dict.
 
         Examples:
             >>> import mindspore.dataset as ds
-            >>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
-            >>> iterator = dataset.create_dict_iterator()
-            >>> for item in iterator:
-            ...     # item is a dict
-            ...     print(type(item))
+            >>>
+            >>> dataset = ds.GeneratorDataset([i for i in range(10)], "data")
+            >>> num_epochs = 3
+            >>> iterator = dataset.create_dict_iterator(num_epochs=num_epochs)
+            >>> for epoch in range(num_epochs):
+            ...     for item in iterator:
+            ...         # output is of type dict
+            ...         print(type(item))
+            ...         break
             ...     break
             <class 'dict'>
         """
@@ -1705,6 +1755,7 @@ class Dataset:
         return self._col_names
 
     @check_output_shape
+    @_cleanup_the_iterators_if_created
     def output_shapes(self, estimate=False):
         """
         Get the shapes of output data.
@@ -1756,6 +1807,7 @@ class Dataset:
             self.saved_output_shapes = output_shapes
         return output_shapes
 
+    @_cleanup_the_iterators_if_created
     def output_types(self):
         """
         Get the types of output data.
@@ -1790,6 +1842,7 @@ class Dataset:
             del self.runtime_context
         return self.saved_output_types
 
+    @_cleanup_the_iterators_if_created
     def get_dataset_size(self):
         """
         Return the number of batches in an epoch.
@@ -1857,6 +1910,7 @@ class Dataset:
             return self.children[0].is_sync()
         return False
 
+    @check_sync_update
     def sync_update(self, condition_name, num_batch=None, data=None):
         """
         Release a blocking condition and trigger callback with given data.
@@ -2138,7 +2192,7 @@ class TextBaseDataset(Dataset):
                 Japanese or Chinese character sets, and 1.0 for other languages with small character sets
                 like English or Latin.
             model_type(SentencePieceModel): Model type. Choose from unigram (default), bpe, char, or word.
-                The input sentence must be pretokenized when using word type.
+                The input sentence must be pre-tokenized when using word type.
             params(dict): Any extra optional parameters of sentencepiece library according to your raw data
 
         Returns:
@@ -2215,7 +2269,7 @@ class TextBaseDataset(Dataset):
                 Japanese or Chinese character sets, and 1.0 for other languages with small character sets
                 like English or Latin.
             model_type(SentencePieceModel): Model type. Choose from unigram (default), bpe, char, or word.
-                The input sentence must be pretokenized when using word type.
+                The input sentence must be pre-tokenized when using word type.
             params(dict): Any extra optional parameters of sentencepiece library according to your raw data
 
         Returns:
@@ -2538,7 +2592,8 @@ def _check_shm_usage(num_worker, queue_size, in_rowsize, out_rowsize):
     when training in parallel mode.
     """
     threshold_ratio = 0.8
-    if platform.system().lower() not in {"windows", "darwin"}:
+    # Verify available size only when using static shared memory on Linux
+    if platform.system().lower() not in {"windows", "darwin"} and in_rowsize != -1 and out_rowsize != -1:
         device_num = _get_device_num()
         # In the cluster, _get_device_num indicates the number of the entire cluster. The maximum number of cards
         # on the ascend server is 8.
@@ -2586,17 +2641,18 @@ class BatchDataset(UnionBaseDataset):
             name as the input columns, i.e., the columns will be replaced.
         max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
             allocation to copy data between processes, the total occupied shared memory will increase as
-            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
-            used if python_multiprocessing is set to True. If it is an int value, it represents
+            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set to -1,
+            shared memory will be dynamically allocated with the actual size of data. This is only used if
+            ``python_multiprocessing`` is set to True. If it is an int value, it represents
             ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
             If it is a list, the first element represents the ``input_columns`` use this value as the unit to
             create shared memory, and the second element represents ``output_columns`` use this value as the unit
-            to create shared memory. Default: 16.
+            to create shared memory. Default: ``None`` , allocate shared memory dynamically.
 
     """
 
     def __init__(self, input_dataset, batch_size, drop_remainder=False, num_parallel_workers=None, per_batch_map=None,
-                 input_columns=None, output_columns=None, python_multiprocessing=False, max_rowsize=16):
+                 input_columns=None, output_columns=None, python_multiprocessing=False, max_rowsize=None):
         super().__init__(children=input_dataset, num_parallel_workers=num_parallel_workers)
 
         if BatchDataset._is_ancestor_of_repeat(input_dataset):
@@ -2617,8 +2673,10 @@ class BatchDataset(UnionBaseDataset):
 
         self.python_multiprocessing = python_multiprocessing
         self.process_pool = None
-        if isinstance(max_rowsize, int):
-            self.max_rowsize = [max_rowsize * self.batch_size] * 2
+        if max_rowsize is None:
+            self.max_rowsize = [-1, -1]
+        elif isinstance(max_rowsize, int):
+            self.max_rowsize = [max_rowsize * self.batch_size] * 2 if max_rowsize != -1 else [max_rowsize, max_rowsize]
         else:
             self.max_rowsize = [max_rowsize[0] * self.batch_size, max_rowsize[1] * self.batch_size]
 
@@ -3040,7 +3098,7 @@ class Pipe:
     Class to handle communication between the master process and the worker processes.
     """
 
-    def __init__(self, warning_ctl, shared_memory=False, max_rowsize=16):
+    def __init__(self, warning_ctl, shared_memory=False, max_rowsize=(-1, -1)):
         self.shared_memory = shared_memory
         self.eof = multiprocessing.Event()
         if self.shared_memory:
@@ -3097,11 +3155,14 @@ def _main_process_already_exit():
     return False
 
 
-def _worker_loop(operations, pipe, seed=get_seed()):
+def _worker_loop(operations, pipe, worker_id):
     """
     Multiprocess worker process loop.
     """
-    # Ensure that the process does not hung when exiting
+    # Initialize C++ side signal handlers
+    cde.register_worker_handlers()
+
+    # Ensure that the process does not hang when exiting
     pipe.res_queue.cancel_join_thread()
 
     def _ignore_sigint():
@@ -3110,9 +3171,12 @@ def _worker_loop(operations, pipe, seed=get_seed()):
         """
         signal.signal(signal.SIGINT, signal.SIG_IGN)
 
-    # We set the seed here so the main process will have the same seed, while the child process
-    # will have different seed depending on what is being passed
-    set_seed(seed)
+    # If the default random seed has not been changed, there is no need to fix the randomness.
+    # Otherwise, set the random seed for each child process to "base_seed + worker_id" to ensure
+    # that the random results of each process are different.
+    if get_seed() != 5489:
+        set_seed(get_seed() + worker_id)
+
     while not _main_process_already_exit():
         _ignore_sigint()
 
@@ -3135,8 +3199,8 @@ def _worker_loop(operations, pipe, seed=get_seed()):
     del pipe.res_queue
 
 
-def worker_target(operations, seed=get_seed()):
-    return lambda pipe: _worker_loop(operations, pipe, seed)
+def worker_target(operations, worker_id):
+    return lambda pipe: _worker_loop(operations, pipe, worker_id)
 
 
 class _MPWorker(multiprocessing.Process):
@@ -3144,11 +3208,12 @@ class _MPWorker(multiprocessing.Process):
     Worker process for multiprocessing.
     """
 
-    def __init__(self, operations, warning_ctl, max_rowsize=16, seed=get_seed()):
+    def __init__(self, operations, warning_ctl, max_rowsize=(-1, -1), worker_id=0):
         shared_memory = get_enable_shared_mem()
         self.pipe = Pipe(warning_ctl, shared_memory=shared_memory, max_rowsize=max_rowsize)
         self.check_interval = get_multiprocessing_timeout_interval()
-        super().__init__(target=worker_target(operations, seed), args=(self.pipe,), daemon=True)
+        super().__init__(target=worker_target(operations, worker_id), name="MapWorker" + str(worker_id),
+                         args=(self.pipe,), daemon=True)
 
     def execute(self, idx, *args):
         """Acquiring data from a worker in an infinite loop"""
@@ -3237,7 +3302,7 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
             self.origin_hook(ex_type, value, tb)
             self.mp_pool_exit_preprocess()
 
-    def __init__(self, op_name, num_parallel_workers, operations, max_rowsize=16):
+    def __init__(self, op_name, num_parallel_workers, operations, max_rowsize=(-1, -1)):
         super(_PythonMultiprocessing, self).__init__()
         self.op_name = op_name
         self.num_parallel_workers = num_parallel_workers
@@ -3253,11 +3318,12 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
 
         self.eot = None
         self.watch_dog = None
-        self.ppid = os.getpid()
+        self.ppid = None
         self.hook = None
         self.warning_ctl = None
         # cache thread (get_ident()) to worker_id mapping in Python layer
         self.python_threads_to_workers = {}
+        self.eof = None
 
     def __del__(self):
         try:
@@ -3277,10 +3343,10 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
                 if child_pid == 0:
                     break
         except OSError:
-            # waitpid may be failed for some reasons so we ignore this error
+            # waitpid may fail for some reason, so we ignore this error
             pass
 
-    # Dataset need watch_dog thread to monitoring fork multi-processing,
+    # Dataset need watch_dog thread to monitoring fork multiprocessing,
     # and thread can't be a member function otherwise python won't collect and release resources.
     @staticmethod
     def _watch_dog(eot, workers):
@@ -3313,6 +3379,8 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
                                 "main process will exit. If this is not an artificial operation, you can use "
                                 "ds.config.set_enable_watchdog(False) to block this error.")
                 os.kill(os.getpid(), signal.SIGTERM)
+            # sleep to release GIL
+            time.sleep(1)
 
         # release the workers
         del workers
@@ -3388,17 +3456,25 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
 
     # When main process exit, subprocesses will be terminate
     @staticmethod
-    def _clean_process(ppid, workers):
+    def _clean_process(ppid, workers, quit_signal):
         """
             This is the execute function of clean process, if we found main process exited, we will clean subprocesses.
 
         Args:
             ppid: The process id of main process.
             workers: The list of subprocesses.
-
+            quit_signal: The flag of quit.
         """
         signal.signal(signal.SIGINT, signal.SIG_IGN)
         while _PythonMultiprocessing.is_process_alive(ppid):
+            if quit_signal.is_set():
+                return
+
+            # independent dataset mode, the subprocess of GeneratorDataset / map / batch should exit when
+            # independent dataset process have exit
+            if os.getppid() != ppid:
+                break
+
             time.sleep(0.1)
 
         _PythonMultiprocessing._terminate_processes(workers)
@@ -3410,10 +3486,10 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         Launch Python multiprocessing pool.
 
         Args:
-            pop_id: ID for operation to have Python multiprocessing pool launched
+            op_id: ID for operation to have Python multiprocessing pool launched
 
         Returns:
-            Python multiprocssing pool is launched.
+            Python multiprocessing pool is launched.
         """
         self.python_threads_to_workers = {}
         self.op_id = op_id
@@ -3424,6 +3500,7 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
             logger.warning(message)
             self.terminate()
             self.reset()
+        self.ppid = os.getpid()
         self.create_pool()
 
     def create_pool(self):
@@ -3444,8 +3521,8 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         # Construct python worker processes
         self.workers = []
         self.warning_ctl = multiprocessing.Value('i', 0)
-        for i in range(self.num_parallel_workers):
-            worker = _MPWorker(self.operations, self.warning_ctl, self.max_rowsize, i + get_seed())
+        for worker_id in range(self.num_parallel_workers):
+            worker = _MPWorker(self.operations, self.warning_ctl, self.max_rowsize, worker_id)
             worker.start()
             self.workers.append(worker)
 
@@ -3530,17 +3607,18 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         The cleaning subprocess will cleanup subprocesses when main process was killed.
         """
         if platform.system().lower() != 'windows':
+            self.eof = multiprocessing.Event()
             self.cleaning_process = multiprocessing.Process(target=self._clean_process,
-                                                            args=(self.ppid, self.workers),
-                                                            name="OrphanCleaner",
+                                                            name="MapCleanProcess",
+                                                            args=(self.ppid, self.workers, self.eof),
                                                             daemon=True)
             self.cleaning_process.start()
 
             if get_enable_watchdog():
                 self.eot = threading.Event()
                 self.watch_dog = threading.Thread(target=self._watch_dog,
+                                                  name="MapWatchDog",
                                                   args=(self.eot, self.workers + [self.cleaning_process]),
-                                                  name="WatchDog",
                                                   daemon=True)
                 self.watch_dog.start()
 
@@ -3552,6 +3630,8 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         if hasattr(self, 'watch_dog') and self.watch_dog is not None and hasattr(self, 'eot') and self.eot is not None:
             self._abort_watchdog()
         if hasattr(self, 'cleaning_process') and self.cleaning_process is not None:
+            if hasattr(self, 'eof') and self.eof is not None and not self.eof.is_set():
+                self.eof.set()
             _PythonMultiprocessing._terminate_processes([self.cleaning_process])
             del self.cleaning_process
 
@@ -3617,16 +3697,18 @@ class MapDataset(UnionBaseDataset):
         callbacks (DSCallback, list[DSCallback], optional): List of Dataset callbacks to be called. Default: ``None``.
         max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
             allocation to copy data between processes, the total occupied shared memory will increase as
-            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
-            used if python_multiprocessing is set to True. If it is an int value, it represents ``input_columns`` and
+            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set to -1,
+            shared memory will be dynamically allocated with the actual size of data. This is only used if
+            ``python_multiprocessing`` is set to True. If it is an int value, it represents ``input_columns`` and
             ``output_columns`` use this value as the unit to create shared memory. If it is a list, the first element
             represents the ``input_columns`` use this value as the unit to create shared memory, and the second element
-            represents ``output_columns`` use this value as the unit to create shared memory. Default: 16.
+            represents ``output_columns`` use this value as the unit to create shared memory. Default: ``None`` ,
+            allocate shared memory dynamically.
         offload (bool, optional): Flag to indicate whether offload is used. Default: ``None``.
     """
 
     def __init__(self, input_dataset, operations=None, input_columns=None, output_columns=None,
-                 num_parallel_workers=None, python_multiprocessing=False, cache=None, callbacks=None, max_rowsize=16,
+                 num_parallel_workers=None, python_multiprocessing=False, cache=None, callbacks=None, max_rowsize=None,
                  offload=None):
         super().__init__(children=input_dataset, num_parallel_workers=num_parallel_workers, cache=cache)
         self.operations = to_list(operations)
@@ -3652,7 +3734,9 @@ class MapDataset(UnionBaseDataset):
         self.process_pool = None
 
         self.callbacks = to_list(callbacks)
-        if isinstance(max_rowsize, int):
+        if max_rowsize is None:
+            self.max_rowsize = [-1, -1]
+        elif isinstance(max_rowsize, int):
             self.max_rowsize = [max_rowsize] * 2
         else:
             self.max_rowsize = max_rowsize
@@ -4269,6 +4353,7 @@ class TransferDataset(Dataset):
         total_batch = 0
         if hasattr(self.children[0], "__total_batch__"):
             total_batch = self.children[0].__total_batch__
+            check_total_batch(total_batch)
         return cde.DataQueueNode(children[0], self.queue_name, self.device_type, self.device_id, self._send_epoch_end,
                                  total_batch, self._create_data_info_queue)