mindspore 2.2.11__cp39-cp39-win_amd64.whl → 2.3.0__cp39-cp39-win_amd64.whl

mindspore/dataset/callback/ds_callback.py

@@ -160,7 +160,7 @@ class WaitedDSCallback(Callback, DSCallback):
     r"""
     Abstract base class used to build dataset callback classes that are synchronized with the training callback class
     `mindspore.train.Callback \
-    <https://www.mindspore.cn/docs/en/r2.2/api_python/train/
+    <https://www.mindspore.cn/docs/en/master/api_python/train/
     mindspore.train.Callback.html#mindspore.train.Callback>`_ .
 
     It can be used to execute a custom callback method before a step or an epoch, such as
@@ -171,7 +171,7 @@ class WaitedDSCallback(Callback, DSCallback):
     `device_number` , `list_callback` , `cur_epoch_num` , `cur_step_num` , `dataset_sink_mode` ,
     `net_outputs` , etc., see
     `mindspore.train.Callback \
-    <https://www.mindspore.cn/docs/en/r2.2/api_python/train/
+    <https://www.mindspore.cn/docs/en/master/api_python/train/
     mindspore.train.Callback.html#mindspore.train.Callback>`_ .
 
     Users can obtain the dataset pipeline context through `ds_run_context` , including

mindspore/dataset/engine/cache_client.py

@@ -26,8 +26,8 @@ class DatasetCache:
     r"""
     A client to interface with tensor caching service.
 
-    For details, please check `Tutorial <https://www.mindspore.cn/
-    tutorials/experts/en/r2.2/dataset/cache.html>`_ .
+    For details, please check
+    `Tutorial <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ .
 
     Args:
         session_id (int): A user assigned session id for the current pipeline.

mindspore/dataset/engine/datasets.py

@@ -71,7 +71,8 @@ from .queue import _SharedQueue, _Queue
 from .validators import check_batch, check_shuffle, check_map, check_filter, check_repeat, check_skip, check_zip, \
     check_rename, check_device_send, check_take, check_output_shape, check_project, \
     check_sync_wait, check_zip_dataset, check_add_column, check_concat, check_split, check_bucket_batch_by_length, \
-    check_save, check_tuple_iterator, check_dict_iterator, check_schema, check_to_device_send, check_padded_batch
+    check_save, check_tuple_iterator, check_dict_iterator, check_schema, check_to_device_send, check_padded_batch, \
+    check_total_batch
 from ..core.config import get_callback_timeout, _init_device_info, get_enable_shared_mem, get_num_parallel_workers, \
     get_enable_watchdog, get_seed, set_seed, get_debug_mode, get_multiprocessing_timeout_interval, _get_debug_hook_list
 from ..core.datatypes import mstype_to_detype
@@ -79,6 +80,7 @@ from ..core.validator_helpers import replace_none
 from ..core.py_util_helpers import ExceptionHandler
 from ..transforms.py_transforms_util import FuncWrapper, Implementation
 from ..vision.transforms import ToNumpy
+from ...mindrecord.config import _get_enc_key, _get_enc_mode, _get_hash_mode, encrypt, append_hash_to_file
 
 try:
     context = import_module("mindspore.context")
@@ -603,8 +605,9 @@ class Dataset:
 
                 - max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
                   allocation to copy data between processes, the total occupied shared memory will increase as
-                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
-                  used if python_multiprocessing is set to True. If it is an int value, it represents
+                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set
+                  to -1, shared memory will be dynamically allocated with the actual size of data. This is only used if
+                  ``python_multiprocessing`` is set to True. If it is an int value, it represents
                   ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
                   If it is a list, the first element represents the ``input_columns`` use this value as the unit to
                   create shared memory, and the second element represents ``output_columns`` use this value as the unit
@@ -666,15 +669,17 @@ class Dataset:
                 be dropped and not propagated to the child node.
             num_parallel_workers (int, optional): Number of workers(threads) to process the dataset in parallel.
                 Default: ``None``.
-            pad_info (dict, optional): The information about how to batch each column. The key
+            pad_info (dict, optional): The pad information about how to batch each column. The key
                 corresponds to the column name, and the value must be a tuple of 2 elements.
                 The first element corresponds to the shape to pad to, and the second
                 element corresponds to the value to pad with. If a column is not
                 specified, then that column will be padded to the longest in the current
-                batch, and 0 will be used as the padding value. Any None dimensions will
-                be padded to the longest in the current batch, unless if
-                pad_to_bucket_boundary is True. If no padding is wanted, set `pad_info`
-                to ``None``. Default: ``None``.
+                batch, and 0 will be used as the padding value. If ``pad_info={"col1": ([224, 224], 0)}``,
+                expand the data column named ``col1`` to shape (224, 224), and fill in the missing values with 0.
+                If ``pad_info={}``, all samples in the batch will be filled to the shape with the largest sample
+                in the current batch. If ``pad_info={"col1": (None, 100)}``, all samples in the batch will be filled
+                to the shape with the largest sample in the current batch, and fill in the missing values with 100.
+                If no padding is wanted, set `pad_info` to ``None``. Default: ``None``.
 
         Returns:
             Dataset, a new dataset with the above operation applied.
@@ -858,11 +863,11 @@ class Dataset:
         `output_columns` , and if not specified, the column name of output column is same as that of `input_columns` .
 
         - If you use transformations (
-          `vision transform <https://mindspore.cn/docs/en/r2.2/api_python/mindspore.\
+          `vision transform <https://mindspore.cn/docs/en/master/api_python/mindspore.\
           dataset.transforms.html#module-mindspore.dataset.vision>`_ ,
-          `nlp transform <https://mindspore.cn/docs/en/r2.2/api_python/mindspore.\
+          `nlp transform <https://mindspore.cn/docs/en/master/api_python/mindspore.\
           dataset.transforms.html#module-mindspore.dataset.text>`_ ,
-          `audio transform <https://mindspore.cn/docs/en/r2.2/api_python/mindspore.\
+          `audio transform <https://mindspore.cn/docs/en/master/api_python/mindspore.\
           dataset.transforms.html#module-mindspore.dataset.audio>`_ )
           provided by mindspore dataset, please use the following parameters:
 
@@ -894,8 +899,9 @@ class Dataset:
 
                 - max_rowsize (Union[int, list[int]], optional): Maximum size of row in MB that is used for shared
                   memory allocation to copy data between processes, the total occupied shared memory will increase as
-                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
-                  used if python_multiprocessing is set to True. If it is an int value, it represents
+                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set
+                  to -1, shared memory will be dynamically allocated with the actual size of data. This is only used if
+                  ``python_multiprocessing`` is set to True. If it is an int value, it represents
                   ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
                   If it is a list, the first element represents the ``input_columns`` use this value as the unit to
                   create shared memory, and the second element represents ``output_columns`` use this value as the unit
@@ -1030,20 +1036,28 @@ class Dataset:
 
         Examples:
             >>> import mindspore.dataset as ds
+            >>>
+            >>> # Create a dataset with 10 elements
             >>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
+            >>> ori_size = dataset.get_dataset_size()
             >>>
-            >>> # Create a dataset where the dataset is repeated for 50 epochs
+            >>> # Repeat the dataset 50 times.
             >>> dataset = dataset.repeat(50)
+            >>> repeated_size = dataset.get_dataset_size()
+            >>> print("ori_size", ori_size, ", repeated_size", repeated_size)
+            ori_size 10 , repeated_size 500
             >>>
-            >>> # Create a dataset where each epoch is shuffled individually
-            >>> dataset = dataset.shuffle(10)
-            >>> dataset = dataset.repeat(50)
+            >>> # Since the original dataset size is less than batch_size, thus no data is returned
+            >>> dataset1 = ds.GeneratorDataset([i for i in range(10)], "column1")
+            >>> dataset1 = dataset1.batch(batch_size=20, drop_remainder=True)
+            >>> dataset1 = dataset1.repeat(6)
             >>>
-            >>> # Create a dataset where the dataset is first repeated for
-            >>> # 50 epochs before shuffling. The shuffle operation will treat
-            >>> # the entire 50 epochs as one big dataset.
-            >>> dataset = dataset.repeat(50)
-            >>> dataset = dataset.shuffle(10)
+            >>> # Repeat the original dataset to 60 elements, thus 3 batches are returned
+            >>> dataset2 = ds.GeneratorDataset([i for i in range(10)], "column1")
+            >>> dataset2 = dataset2.repeat(6)
+            >>> dataset2 = dataset2.batch(batch_size=20, drop_remainder=True)
+            >>> print("dataset1 size", dataset1.get_dataset_size(), ", dataset2 size", dataset2.get_dataset_size())
+            dataset1 size 0 , dataset2 size 3
         """
         return RepeatDataset(self, count)
 
@@ -1462,14 +1476,14 @@ class Dataset:
              - Type in `mindrecord`
              - Details
            * - bool
-             - None
-             - Not supported
+             - int32
+             - transform to int32
            * - int8
              - int32
              -
            * - uint8
-             - bytes(1D uint8)
-             - Drop dimension
+             - int32
+             -
            * - int16
              - int32
              -
@@ -1486,8 +1500,8 @@ class Dataset:
              - int64
              -
            * - uint64
-             - None
-             - Not supported
+             - int64
+             - Maybe reverse
            * - float16
              - float32
              -
@@ -1500,6 +1514,9 @@ class Dataset:
            * - string
              - string
              - Multi-dimensional string not supported
+           * - bytes
+             - bytes
+             - Multi-dimensional bytes not supported
 
         Note:
             1. To save the samples in order, set dataset's `shuffle` to ``False`` and `num_files` to ``1``.
@@ -1507,8 +1524,7 @@ class Dataset:
                with random attribute in map operation.
             3. When array dimension is variable, one-dimensional arrays or
                multi-dimensional arrays with variable dimension 0 are supported.
-            4. MindRecord does not support uint64, multi-dimensional uint8(drop dimension) nor
-               multi-dimensional string.
+            4. MindRecord does not support multi-dimensional string or multi-dimensional bytes.
 
         Args:
             file_name (str): Path to dataset file.
@@ -1527,6 +1543,10 @@ class Dataset:
             >>> d1 = ds.GeneratorDataset(generator_1d, ["data"], shuffle=False)
             >>> d1.save('/path/to/save_file')
         """
+        if (_get_enc_key() is not None or _get_hash_mode() is not None) and num_files > 1:
+            raise RuntimeError("When encode mode or hash check is enabled, " +
+                               "the automatic sharding function is unavailable.")
+
         ir_tree, api_tree = self.create_ir_tree()
 
         runtime_context = cde.PythonRuntimeContext()
@@ -1536,38 +1556,49 @@ class Dataset:
         runtime_context.AssignConsumer(consumer)
 
         consumer.Save()
+
+        if _get_hash_mode() is not None:
+            append_hash_to_file(file_name)
+            append_hash_to_file(file_name + ".db")
+
+        if _get_enc_key() is not None:
+            encrypt(file_name, _get_enc_key(), _get_enc_mode())
+            encrypt(file_name + ".db", _get_enc_key(), _get_enc_mode())
+
         _set_dataset_permissions(file_name, num_files)
         del api_tree
 
     @check_tuple_iterator
     def create_tuple_iterator(self, columns=None, num_epochs=-1, output_numpy=False, do_copy=True):
         """
-        Create an iterator over the dataset. The datatype retrieved back will be a list of `numpy.ndarray` .
-
-        To specify which columns to list and the order needed, use columns_list. If columns_list
-        is not provided, the order of the columns will remain unchanged.
+        Create an iterator over the dataset that yields samples of type list, whose elements are
+        the data for each column.
 
         Args:
-            columns (list[str], optional): List of columns to be used to specify the order of columns.
-                Default: ``None``, means all columns.
-            num_epochs (int, optional): Maximum number of epochs that iterator can be iterated.
-                Default: ``-1``, iterator can be iterated infinite number of epochs.
-            output_numpy (bool, optional): Whether or not to output NumPy datatype.
-                If `output_numpy` is ``False``, iterator will output MSTensor. Default: ``False``.
-            do_copy (bool, optional): When output data type is :class:`mindspore.Tensor`,
-                use this param to select the conversion method, only take False for better performance.
-                Default: ``True``.
+            columns (list[str], optional): Specify the output columns and the order.
+                Default: ``None``, keep all the output columns and their original order.
+            num_epochs (int, optional): The number of epochs to iterate over the entire dataset.
+                Default: ``-1`` , the dataset can be iterated indefinitely.
+            output_numpy (bool, optional): Whether to keep the output data as NumPy ndarray, or
+                convert it to Tensor. Default: ``False`` .
+            do_copy (bool, optional): Whether to copy the data when converting output to Tensor,
+                or reuse the buffer for better performance, only works when `output_numpy` is ``False`` .
+                Default: ``True`` .
 
         Returns:
-            Iterator, a dataset iterator that returns data of type Tuple.
+            Iterator, a dataset iterator that yields samples of type list.
 
         Examples:
             >>> import mindspore.dataset as ds
-            >>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
-            >>> iterator = dataset.create_tuple_iterator()
-            >>> for item in iterator:
-            ...     # item is a list
-            ...     print(type(item))
+            >>>
+            >>> dataset = ds.GeneratorDataset([i for i in range(10)], "data")
+            >>> num_epochs = 3
+            >>> iterator = dataset.create_tuple_iterator(num_epochs=num_epochs)
+            >>> for epoch in range(num_epochs):
+            ...     for item in iterator:
+            ...         # output is of type tuple
+            ...         print(type(item))
+            ...         break
             ...     break
             <class 'list'>
         """
@@ -1581,27 +1612,32 @@ class Dataset:
     @check_dict_iterator
     def create_dict_iterator(self, num_epochs=-1, output_numpy=False, do_copy=True):
         """
-        Create an iterator over the dataset. The data retrieved will be a dictionary datatype.
+        Create an iterator over the dataset that yields samples of type dict,
+        while the key is the column name and the value is the data.
 
         Args:
-            num_epochs (int, optional): Maximum number of epochs that iterator can be iterated.
-                Default: ``-1`` , iterator can be iterated infinite number of epochs.
-            output_numpy (bool, optional): Whether or not to output NumPy datatype,
-                if `output_numpy` is ``False``, iterator will output MSTensor. Default: ``False`` .
-            do_copy (bool, optional): When output data type is :class:`mindspore.Tensor`,
-                use this param to select the conversion method, only take False for better performance.
+            num_epochs (int, optional): The number of epochs to iterate over the entire dataset.
+                Default: ``-1`` , the dataset can be iterated indefinitely.
+            output_numpy (bool, optional): Whether to keep the output data as NumPy ndarray, or
+                convert it to Tensor. Default: ``False`` .
+            do_copy (bool, optional): Whether to copy the data when converting output to Tensor,
+                or reuse the buffer for better performance, only works when `output_numpy` is ``False`` .
                 Default: ``True`` .
 
         Returns:
-            Iterator, a dataset iterator that returns data of type Dict.
+            Iterator, a dataset iterator that yields samples of type dict.
 
         Examples:
             >>> import mindspore.dataset as ds
-            >>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
-            >>> iterator = dataset.create_dict_iterator()
-            >>> for item in iterator:
-            ...     # item is a dict
-            ...     print(type(item))
+            >>>
+            >>> dataset = ds.GeneratorDataset([i for i in range(10)], "data")
+            >>> num_epochs = 3
+            >>> iterator = dataset.create_dict_iterator(num_epochs=num_epochs)
+            >>> for epoch in range(num_epochs):
+            ...     for item in iterator:
+            ...         # output is of type dict
+            ...         print(type(item))
+            ...         break
             ...     break
             <class 'dict'>
         """
@@ -2538,7 +2574,8 @@ def _check_shm_usage(num_worker, queue_size, in_rowsize, out_rowsize):
     when training in parallel mode.
     """
     threshold_ratio = 0.8
-    if platform.system().lower() not in {"windows", "darwin"}:
+    # Verify available size only when using static shared memory on Linux
+    if platform.system().lower() not in {"windows", "darwin"} and in_rowsize != -1 and out_rowsize != -1:
         device_num = _get_device_num()
         # In the cluster, _get_device_num indicates the number of the entire cluster. The maximum number of cards
         # on the ascend server is 8.
@@ -2586,8 +2623,9 @@ class BatchDataset(UnionBaseDataset):
             name as the input columns, i.e., the columns will be replaced.
         max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
             allocation to copy data between processes, the total occupied shared memory will increase as
-            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
-            used if python_multiprocessing is set to True. If it is an int value, it represents
+            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set to -1,
+            shared memory will be dynamically allocated with the actual size of data. This is only used if
+            ``python_multiprocessing`` is set to True. If it is an int value, it represents
             ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
             If it is a list, the first element represents the ``input_columns`` use this value as the unit to
             create shared memory, and the second element represents ``output_columns`` use this value as the unit
@@ -2618,7 +2656,7 @@ class BatchDataset(UnionBaseDataset):
         self.python_multiprocessing = python_multiprocessing
         self.process_pool = None
         if isinstance(max_rowsize, int):
-            self.max_rowsize = [max_rowsize * self.batch_size] * 2
+            self.max_rowsize = [max_rowsize * self.batch_size] * 2 if max_rowsize != -1 else [max_rowsize, max_rowsize]
         else:
             self.max_rowsize = [max_rowsize[0] * self.batch_size, max_rowsize[1] * self.batch_size]
 
@@ -3097,7 +3135,7 @@ def _main_process_already_exit():
     return False
 
 
-def _worker_loop(operations, pipe, seed=get_seed()):
+def _worker_loop(operations, pipe, worker_id):
     """
     Multiprocess worker process loop.
     """
@@ -3110,9 +3148,11 @@ def _worker_loop(operations, pipe, seed=get_seed()):
         """
         signal.signal(signal.SIGINT, signal.SIG_IGN)
 
-    # We set the seed here so the main process will have the same seed, while the child process
-    # will have different seed depending on what is being passed
-    set_seed(seed)
+    # If the default random seed has not been changed, there is no need to fix the randomness.
+    # Otherwise, set the random seed for each child process to "base_seed + worker_id" to ensure
+    # that the random results of each process are different.
+    if get_seed() != 5489:
+        set_seed(get_seed() + worker_id)
     while not _main_process_already_exit():
         _ignore_sigint()
 
@@ -3135,8 +3175,8 @@ def _worker_loop(operations, pipe, seed=get_seed()):
     del pipe.res_queue
 
 
-def worker_target(operations, seed=get_seed()):
-    return lambda pipe: _worker_loop(operations, pipe, seed)
+def worker_target(operations, worker_id):
+    return lambda pipe: _worker_loop(operations, pipe, worker_id)
 
 
 class _MPWorker(multiprocessing.Process):
@@ -3144,11 +3184,12 @@ class _MPWorker(multiprocessing.Process):
     Worker process for multiprocessing.
     """
 
-    def __init__(self, operations, warning_ctl, max_rowsize=16, seed=get_seed()):
+    def __init__(self, operations, warning_ctl, max_rowsize=16, worker_id=0):
         shared_memory = get_enable_shared_mem()
         self.pipe = Pipe(warning_ctl, shared_memory=shared_memory, max_rowsize=max_rowsize)
         self.check_interval = get_multiprocessing_timeout_interval()
-        super().__init__(target=worker_target(operations, seed), args=(self.pipe,), daemon=True)
+        super().__init__(target=worker_target(operations, worker_id), name="MapWorker" + str(worker_id),
+                         args=(self.pipe,), daemon=True)
 
     def execute(self, idx, *args):
         """Acquiring data from a worker in an infinite loop"""
@@ -3175,6 +3216,14 @@ class _MPWorker(multiprocessing.Process):
                     logger.warning("Please `pip install py-spy` to get the stacks of the stuck process.")
             try:
                 res = self.pipe.master_receive()
+                # Because there is no need to copy when creating Tensors in the C++layer, it reduces the time
+                # from np.ndarray to C++Tensor creation. However, when using shared memory in multiple processes,
+                # the address of the shared memory will always be passed to subsequent nodes in the dataset pipeline,
+                # and the shared memory will also be written by the current node, causing dirty data to be accessed
+                # by subsequent nodes in the pipeline. So make a memory copy here to solve the problem of
+                # shared memory being contaminated.
+                if get_enable_shared_mem():
+                    res = copy.deepcopy(res)
             except queue.Empty:
                 continue
             if res is None:
@@ -3258,6 +3307,7 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         self.warning_ctl = None
         # cache thread (get_ident()) to worker_id mapping in Python layer
         self.python_threads_to_workers = {}
+        self.eof = None
 
     def __del__(self):
         try:
@@ -3388,17 +3438,19 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
 
     # When main process exit, subprocesses will be terminate
     @staticmethod
-    def _clean_process(ppid, workers):
+    def _clean_process(ppid, workers, quit_signal):
         """
             This is the execute function of clean process, if we found main process exited, we will clean subprocesses.
 
         Args:
             ppid: The process id of main process.
             workers: The list of subprocesses.
-
+            quit_signal: The flag of quit.
         """
         signal.signal(signal.SIGINT, signal.SIG_IGN)
         while _PythonMultiprocessing.is_process_alive(ppid):
+            if quit_signal.is_set():
+                return
             time.sleep(0.1)
 
         _PythonMultiprocessing._terminate_processes(workers)
@@ -3444,8 +3496,8 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         # Construct python worker processes
         self.workers = []
         self.warning_ctl = multiprocessing.Value('i', 0)
-        for i in range(self.num_parallel_workers):
-            worker = _MPWorker(self.operations, self.warning_ctl, self.max_rowsize, i + get_seed())
+        for worker_id in range(self.num_parallel_workers):
+            worker = _MPWorker(self.operations, self.warning_ctl, self.max_rowsize, worker_id)
             worker.start()
             self.workers.append(worker)
 
@@ -3530,17 +3582,18 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         The cleaning subprocess will cleanup subprocesses when main process was killed.
         """
         if platform.system().lower() != 'windows':
+            self.eof = multiprocessing.Event()
             self.cleaning_process = multiprocessing.Process(target=self._clean_process,
-                                                            args=(self.ppid, self.workers),
-                                                            name="OrphanCleaner",
+                                                            name="MapCleanProcess",
+                                                            args=(self.ppid, self.workers, self.eof),
                                                             daemon=True)
             self.cleaning_process.start()
 
             if get_enable_watchdog():
                 self.eot = threading.Event()
                 self.watch_dog = threading.Thread(target=self._watch_dog,
+                                                  name="MapWatchDog",
                                                   args=(self.eot, self.workers + [self.cleaning_process]),
-                                                  name="WatchDog",
                                                   daemon=True)
                 self.watch_dog.start()
 
@@ -3552,6 +3605,8 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         if hasattr(self, 'watch_dog') and self.watch_dog is not None and hasattr(self, 'eot') and self.eot is not None:
             self._abort_watchdog()
         if hasattr(self, 'cleaning_process') and self.cleaning_process is not None:
+            if hasattr(self, 'eof') and self.eof is not None and not self.eof.is_set():
+                self.eof.set()
             _PythonMultiprocessing._terminate_processes([self.cleaning_process])
             del self.cleaning_process
 
@@ -3617,8 +3672,9 @@ class MapDataset(UnionBaseDataset):
         callbacks (DSCallback, list[DSCallback], optional): List of Dataset callbacks to be called. Default: ``None``.
         max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
             allocation to copy data between processes, the total occupied shared memory will increase as
-            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
-            used if python_multiprocessing is set to True. If it is an int value, it represents ``input_columns`` and
+            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set to -1,
+            shared memory will be dynamically allocated with the actual size of data. This is only used if
+            ``python_multiprocessing`` is set to True. If it is an int value, it represents ``input_columns`` and
             ``output_columns`` use this value as the unit to create shared memory. If it is a list, the first element
             represents the ``input_columns`` use this value as the unit to create shared memory, and the second element
             represents ``output_columns`` use this value as the unit to create shared memory. Default: 16.
@@ -4051,6 +4107,15 @@ class ConcatDataset(UnionBaseDataset):
 
             self._sampler = sampler
             self._children_sizes = [c.get_dataset_size() for c in self.children]
+
+            # Recursive access to other child concat nodes
+            def set_child(node):
+                for c in node.children:
+                    if isinstance(c, ConcatDataset):
+                        c.use_sampler(sampler)
+                    set_child(c)
+            set_child(self)
+
             return
 
         if sampler.is_shuffled():
@@ -4186,6 +4251,12 @@ class _ToDevice:
         """
         return self._to_device.GetDataInfo()
 
+    def get_mbuf_queue_size(self):
+        """
+        Get element numbers inside mbuf.
+        """
+        return self._to_device.GetMbufQueueSize()
+
     def get_send_info(self):
         """
         In sink mode, it returns the send information of dataset at this moment.
@@ -4254,6 +4325,7 @@ class TransferDataset(Dataset):
         total_batch = 0
         if hasattr(self.children[0], "__total_batch__"):
             total_batch = self.children[0].__total_batch__
+            check_total_batch(total_batch)
         return cde.DataQueueNode(children[0], self.queue_name, self.device_type, self.device_id, self._send_epoch_end,
                                  total_batch, self._create_data_info_queue)
 
@@ -4300,6 +4372,14 @@ class TransferDataset(Dataset):
             return self._to_device.get_data_info()
         raise RuntimeError("Calling get_data_info with bad state.")
 
+    def get_mbuf_queue_size(self):
+        """
+        Get element numbers inside mbuf.
+        """
+        if self._to_device is not None:
+            return self._to_device.get_mbuf_queue_size()
+        raise RuntimeError("Device queue is not init, call get_mbuf_queue_size failed.")
+
     def get_send_info(self):
         """
         In sink mode, it returns the send information of dataset at this moment.