mindspore 2.1.0__cp37-cp37m-manylinux1_x86_64.whl → 2.2.10__cp37-cp37m-manylinux1_x86_64.whl

mindspore/dataset/engine/datasets.py

@@ -129,8 +129,7 @@ def _reset_training_dataset(global_step, dataset_size):
     """
     dataset = _get_training_dataset()
     if dataset is not None:
-        epoch = global_step // dataset_size
-        dataset._reset(global_step, epoch)  # pylint: disable=protected-access
+        dataset._reset(global_step, dataset_size)  # pylint: disable=protected-access
     else:
         raise RuntimeError("Training dataset is not set.")
 
@@ -210,7 +209,7 @@ def zip(datasets):
             The number of datasets must be more than 1.
 
     Returns:
-        Dataset, dataset zipped.
+        Dataset, a new dataset with the above operation applied.
 
     Raises:
         ValueError: If the number of datasets is 1.
@@ -352,6 +351,7 @@ class Dataset:
         self._repeat_count = None
         self._class_indexing = None
         self._sync = False
+        self._global_step = None
 
     @staticmethod
     def _get_operator_id(dataset):
@@ -388,36 +388,42 @@ class Dataset:
             _OP_PROCESS.update(generator_process)
         return op_name
 
-    def create_ir_tree(self):
+    def create_ir_tree(self, getter_mode=False):
         """
         Internal method to build an IR tree.
 
+        Args:
+            getter_mode (bool, optional): Whether to build IR tree in pull mode. Default: ``False``.
+
         Returns:
-            DatasetNode, the root node of the IR tree.
-            Dataset, the root dataset of the IR tree.
+            Union[DatasetNode, Dataset], the root node of the IR tree and the root dataset of the IR tree.
         """
         parent = self.parent
         self.parent = []
         dataset = copy.deepcopy(self)
         global _OP_NAME
         _OP_NAME = Dataset._get_operator_id(dataset)
-        ir_tree = dataset.parse_tree()
+        ir_tree = dataset.parse_tree(getter_mode)
         self.parent = parent
         _init_device_info()
         return ir_tree, dataset
 
-    def parse_tree(self):
+    def parse_tree(self, getter_mode=False):
         """
         Internal method to parse the API tree into an IR tree.
 
+        Args:
+            getter_mode (bool, optional): Whether to build IR tree in pull mode. Default: ``False``.
+
         Returns:
             DatasetNode, the root node of the IR tree.
         """
         if len(self.parent) > 1:
             raise ValueError("The data pipeline is not a tree (i.e., one node has 2 consumers)")
-        ir_children = [d.parse_tree() for d in self.children]
+        ir_children = [d.parse_tree(getter_mode) for d in self.children]
         # Bootstrap can only be performed on a copy of the original dataset node.
         # Bootstrap on original dataset node will make all iterators share the same process pool
+        self.pre_parse(getter_mode)
         self.iterator_bootstrap()
         ir_node = self.parse(ir_children)
         ir_node = self.post_parse(ir_node)
@@ -516,7 +522,7 @@ class Dataset:
                 bucket if it is not a full batch. Default: ``False``.
 
         Returns:
-            Dataset, dataset bucketized and batched by length.
+            Dataset, a new dataset with the above operation applied.
 
         Examples:
             >>> # Create a dataset where certain counts rows are combined into a batch
@@ -595,12 +601,17 @@ class Dataset:
                   multi-threading mode. If `per_batch_map` is a CPU bound task, it is recommended to use
                   multi-processing mode. Default: ``False`` , use python multi-threading mode.
 
-                - max_rowsize(int, optional): Maximum size of row in MB that is used for shared memory allocation to
-                  copy data between processes. This is only used if `python_multiprocessing` is set to ``True``.
-                  Default: ``16`` .
+                - max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
+                  allocation to copy data between processes, the total occupied shared memory will increase as
+                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
+                  used if python_multiprocessing is set to True. If it is an int value, it represents
+                  ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
+                  If it is a list, the first element represents the ``input_columns`` use this value as the unit to
+                  create shared memory, and the second element represents ``output_columns`` use this value as the unit
+                  to create shared memory. Default: 16.
 
         Returns:
-            BatchDataset, dataset batched.
+            Dataset, a new dataset with the above operation applied.
 
         Examples:
             >>> # 1) Create a dataset where every 5 rows are combined into a batch
@@ -666,7 +677,7 @@ class Dataset:
                 to ``None``. Default: ``None``.
 
         Returns:
-            PaddedBatchDataset, dataset batched.
+            Dataset, a new dataset with the above operation applied.
 
         Examples:
             >>> # 1) Pad every sample to the largest sample's shape and batch the samples
@@ -700,7 +711,7 @@ class Dataset:
                 Default: ``None``.
 
         Returns:
-            SyncWaitDataset, dataset added a blocking condition.
+            Dataset, a new dataset with the above operation applied.
 
         Raises:
             RuntimeError: If condition name already exists.
@@ -758,7 +769,7 @@ class Dataset:
                 dataset will result in a global shuffle.
 
         Returns:
-            Dataset, dataset shuffled.
+            Dataset, a new dataset with the above operation applied.
 
         Raises:
             RuntimeError: If exist sync operations before shuffle.
@@ -784,7 +795,7 @@ class Dataset:
                 return a `Dataset` .
 
         Returns:
-            Dataset, dataset applied by the function.
+            Dataset, a new dataset with the above operation applied.
 
         Examples:
             >>> import mindspore.dataset as ds
@@ -847,11 +858,11 @@ class Dataset:
         `output_columns` , and if not specified, the column name of output column is same as that of `input_columns` .
 
         - If you use transformations (
-          `vision transform <https://mindspore.cn/docs/en/r2.1/api_python/mindspore.\
+          `vision transform <https://mindspore.cn/docs/en/r2.2/api_python/mindspore.\
           dataset.transforms.html#module-mindspore.dataset.vision>`_ ,
-          `nlp transform <https://mindspore.cn/docs/en/r2.1/api_python/mindspore.\
+          `nlp transform <https://mindspore.cn/docs/en/r2.2/api_python/mindspore.\
           dataset.transforms.html#module-mindspore.dataset.text>`_ ,
-          `audio transform <https://mindspore.cn/docs/en/r2.1/api_python/mindspore.\
+          `audio transform <https://mindspore.cn/docs/en/r2.2/api_python/mindspore.\
           dataset.transforms.html#module-mindspore.dataset.audio>`_ )
           provided by mindspore dataset, please use the following parameters:
 
@@ -881,9 +892,14 @@ class Dataset:
                 - python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker processes.
                   This option could be beneficial if the Python operation is computational heavy. Default: ``False``.
 
-                - max_rowsize (int, optional): Maximum size of row in MB that is used for shared memory allocation to
-                  copy data between processes.  This is only used if `python_multiprocessing` is set to ``True``.
-                  Default: ``16``.
+                - max_rowsize (Union[int, list[int]], optional): Maximum size of row in MB that is used for shared
+                  memory allocation to copy data between processes, the total occupied shared memory will increase as
+                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
+                  used if python_multiprocessing is set to True. If it is an int value, it represents
+                  ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
+                  If it is a list, the first element represents the ``input_columns`` use this value as the unit to
+                  create shared memory, and the second element represents ``output_columns`` use this value as the unit
+                  to create shared memory. Default: 16.
 
                 - cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
                   Default: ``None``, which means no cache is used.
@@ -900,7 +916,7 @@ class Dataset:
               `operations` .
 
         Returns:
-            Dataset, dataset after mapping operation.
+            Dataset, a new dataset with the above operation applied.
 
         Examples:
             >>> import mindspore.dataset as ds
@@ -986,7 +1002,7 @@ class Dataset:
                 in parallel. Default: ``None``.
 
         Returns:
-            Dataset, dataset filtered.
+            Dataset, a new dataset with the above operation applied.
 
         Examples:
             >>> # generator data(0 ~ 19)
@@ -1010,7 +1026,7 @@ class Dataset:
             count (int): Number of times the dataset is going to be repeated. Default: ``None``.
 
         Returns:
-            Dataset, dataset repeated.
+            Dataset, a new dataset with the above operation applied.
 
         Examples:
             >>> import mindspore.dataset as ds
@@ -1040,7 +1056,7 @@ class Dataset:
             count (int): Number of elements in the dataset to be skipped.
 
         Returns:
-            Dataset, dataset that containing rows like origin rows subtract skipped rows.
+            Dataset, a new dataset with the above operation applied.
 
         Examples:
             >>> import mindspore.dataset as ds
@@ -1053,19 +1069,22 @@ class Dataset:
     @check_take
     def take(self, count=-1):
         """
-        Takes at most given numbers of elements from the dataset.
-
-        Note:
-            1. If count is greater than the number of elements in the dataset or equal to -1,
-               all the elements in dataset will be taken.
-            2. The order of using take and batch matters. If take is before batch operation,
-               then take the given number of rows; otherwise take the given number of batches.
+        Take the first specified number of samples from the dataset.
 
         Args:
-            count (int, optional): Number of elements to be taken from the dataset. Default: ``-1`` .
+            count (int, optional): The desired number of samples to take. If the value exceeds
+                the total number of samples in the dataset, all data will be returned.
+                Default: ``-1`` , will return all data.
+
+        Note:
+            When there are operations that will change the number of samples of the dataset in
+            the data pipeline, the location of the `take` operation can change its effect.
+            For example, `batch` operation will combine the successive samples of the specified
+            `batch_size` into 1 sample, so `.batch(batch_size).take(1)` will be equivalent to
+            `.take(batch_size).batch(batch_size)`.
 
         Returns:
-            Dataset, dataset taken.
+            Dataset, a new dataset with the above operation applied.
 
         Examples:
             >>> import mindspore.dataset as ds
@@ -1163,7 +1182,7 @@ class Dataset:
                will be different in each epoch.
 
         Returns:
-            tuple(Dataset), a tuple of datasets that have been split.
+            Tuple[Dataset], a tuple of new datasets split from the original one.
 
         Raises:
             RuntimeError: If get_dataset_size returns None or is not supported for this dataset.
@@ -1218,7 +1237,7 @@ class Dataset:
                 to be zipped together with this dataset.
 
         Returns:
-            Dataset, dataset zipped.
+            Dataset, a new dataset with the above operation applied.
 
         Raises:
             TypeError: The parameter is not dataset object or tuple of dataset objects.
@@ -1244,6 +1263,12 @@ class Dataset:
         Concatenate the dataset objects in the input list.
         Performing "+" operation on dataset objects can achieve the same effect.
 
+        For a dataset concatenated by many other dataset objects, it returns the data in the order of
+        datasets passed in. If you want to change the data order(such as random selection from each dataset
+        instead of in sequence), apply `use_sampler` method on the concatenated dataset object.
+        Currently `use_sampler` supports `dataset.DistributedSampler` for sharding selection from each dataset
+        or `dataset.RandomSampler` for random selection from each dataset, see examples below.
+
         Note:
             The column name, and rank and type of the column data must be the same in the input datasets.
 
@@ -1252,17 +1277,45 @@ class Dataset:
                 to be concatenated together with this dataset.
 
         Returns:
-            Dataset, dataset concatenated.
+            Dataset, a new dataset with the above operation applied.
 
         Examples:
             >>> import mindspore.dataset as ds
-            >>> dataset_1 = ds.GeneratorDataset([1, 2, 3], "column1")
-            >>> dataset_2 = ds.GeneratorDataset([2, 3, 4], "column1")
+            >>> dataset_1 = ds.GeneratorDataset([1, 2, 3], "column1", shuffle=False)
+            >>> dataset_2 = ds.GeneratorDataset([4, 5, 6], "column1", shuffle=False)
             >>>
             >>> # Create a dataset by concatenating dataset_1 and dataset_2 with "+" operator
             >>> dataset = dataset_1 + dataset_2
             >>> # Create a dataset by concatenating dataset_1 and dataset_2 with concat operation
             >>> dataset = dataset_1.concat(dataset_2)
+            >>>
+            >>> # Check the data order of dataset
+            >>> dataset_1 = ds.GeneratorDataset([1, 2, 3], "column1", shuffle=False)
+            >>> dataset_2 = ds.GeneratorDataset([4, 5, 6], "column1", shuffle=False)
+            >>> dataset = dataset_1 + dataset_2
+            >>> result = list(dataset)
+            >>> # [[Tensor(shape=[], dtype=Int64, value= 1)], [Tensor(shape=[], dtype=Int64, value= 2)],
+            >>> #  [Tensor(shape=[], dtype=Int64, value= 3)], [Tensor(shape=[], dtype=Int64, value= 4)],
+            >>> #  [Tensor(shape=[], dtype=Int64, value= 5)], [Tensor(shape=[], dtype=Int64, value= 6)]]
+            >>>
+            >>> # Change the data order of concatenated dataset with sharding selection
+            >>> dataset_1 = ds.GeneratorDataset([1, 2, 3], "column1", shuffle=False)
+            >>> dataset_2 = ds.GeneratorDataset([4, 5, 6], "column1", shuffle=False)
+            >>> dataset = dataset_1.concat(dataset_2)
+            >>> dataset.use_sampler(ds.DistributedSampler(num_shards=2, shard_id=1, shuffle=False))
+            >>> result = list(dataset)
+            >>> # [[Tensor(shape=[], dtype=Int64, value= 2)], [Tensor(shape=[], dtype=Int64, value= 4)],
+            >>> #  [Tensor(shape=[], dtype=Int64, value= 6)]]
+            >>>
+            >>> # Change the data order of concatenated dataset with random selection
+            >>> dataset_1 = ds.GeneratorDataset([1, 2, 3], "column1", shuffle=False)
+            >>> dataset_2 = ds.GeneratorDataset([4, 5, 6], "column1", shuffle=False)
+            >>> dataset = dataset_1.concat(dataset_2)
+            >>> dataset.use_sampler(ds.RandomSampler())
+            >>> result = list(dataset)
+            >>> # [[Tensor(shape=[], dtype=Int64, value= 1)], [Tensor(shape=[], dtype=Int64, value= 4)],
+            >>> #  [Tensor(shape=[], dtype=Int64, value= 2)], [Tensor(shape=[], dtype=Int64, value= 5)],
+            >>> #  [Tensor(shape=[], dtype=Int64, value= 6)], [Tensor(shape=[], dtype=Int64, value= 3)]]
         """
         if isinstance(datasets, Dataset):
             datasets = [self] + [datasets]
@@ -1282,7 +1335,7 @@ class Dataset:
             output_columns (Union[str, list[str]]): List of names of the output columns.
 
         Returns:
-            Dataset, dataset renamed.
+            Dataset, a new dataset with the above operation applied.
 
         Examples:
             >>> import mindspore.dataset as ds
@@ -1308,7 +1361,7 @@ class Dataset:
             columns(Union[str, list[str]]): List of names of the columns to project.
 
         Returns:
-            Dataset, dataset projected.
+            Dataset, a new dataset with the above operation applied.
 
         Examples:
             >>> import mindspore.dataset as ds
@@ -1332,7 +1385,7 @@ class Dataset:
                                    return a preprocessed `Dataset` .
 
         Returns:
-            Dataset, dataset applied by the function.
+            Dataset, a new dataset with the above operation applied.
 
         Examples:
             >>> import mindspore.dataset as ds
@@ -1360,7 +1413,7 @@ class Dataset:
         return dataset
 
     @check_device_send
-    def device_que(self, send_epoch_end=True, create_data_info_queue=False):
+    def device_que(self, send_epoch_end=True, create_data_info_queue=False, queue_name=""):
         """
         Return a transferred Dataset that transfers data through a device.
 
@@ -1369,13 +1422,15 @@ class Dataset:
                 Default: ``True``.
             create_data_info_queue (bool, optional): Whether to create queue which stores
                 types and shapes of data or not. Default: ``False``.
+            queue_name (str, optional): Name of queue which connects dataset processing and model
+                computing. Default: ``""``.
 
         Note:
             If device is Ascend, features of data will be transferred one by one. The limitation
             of data transmission per time is 256M.
 
         Returns:
-            Dataset, dataset for transferring.
+            Dataset, a new dataset with the above operation applied.
 
         Examples:
             >>> import mindspore.dataset as ds
@@ -1387,7 +1442,7 @@ class Dataset:
             >>> time.sleep(0.1)
             >>> data.stop_send()
         """
-        return TransferDataset(self, send_epoch_end, create_data_info_queue)
+        return TransferDataset(self, send_epoch_end, create_data_info_queue, queue_name)
 
     @check_save
     def save(self, file_name, num_files=1, file_type='mindrecord'):
@@ -1504,7 +1559,7 @@ class Dataset:
                 Default: ``True``.
 
         Returns:
-            Iterator, tuple iterator over the dataset.
+            Iterator, a dataset iterator that returns data of type Tuple.
 
         Examples:
             >>> import mindspore.dataset as ds
@@ -1538,7 +1593,7 @@ class Dataset:
                 Default: ``True`` .
 
         Returns:
-            Iterator, dictionary iterator over the dataset.
+            Iterator, a dataset iterator that returns data of type Dict.
 
         Examples:
             >>> import mindspore.dataset as ds
@@ -1599,11 +1654,14 @@ class Dataset:
     def copy_batch_size(self, value):
         self._batch_size = value
 
-    def _init_tree_getters(self):
+    def _init_tree_getters(self, getter_mode=True):
         """
         Get pipeline information.
+
+        Args:
+            getter_mode (bool, optional): Whether to build IR tree in pull mode. Default: ``True``.
         """
-        ir_tree, api_tree = self.create_ir_tree()
+        ir_tree, api_tree = self.create_ir_tree(getter_mode)
 
         runtime_context = cde.PythonRuntimeContext()
         runtime_context.Init()
@@ -1904,12 +1962,12 @@ class Dataset:
 
     def get_class_indexing(self):
         """
-        Return the class index.
+        Get the mapping dictionary from category names to category indexes.
+
+        This dictionary can be used to look up which category name corresponds to a particular category index.
 
         Returns:
-            dict, a str-to-int mapping from label name to index.
-            dict, a str-to-list<int> mapping from label name to index for Coco ONLY. The second number
-            in the list is used to indicate the super category.
+            Dict[str, int], the mappings from category names to category indexes.
 
         Examples:
             >>> import mindspore.dataset as ds
@@ -1981,6 +2039,13 @@ class Dataset:
             shard_id = 0
         return num_shards, shard_id
 
+    def pre_parse(self, getter_mode):
+        if getter_mode:
+            if hasattr(self, "python_multiprocessing"):
+                self.python_multiprocessing = False
+            if hasattr(self, "num_parallel_workers"):
+                self.num_parallel_workers = 1
+
     def post_parse(self, ir_node):
         if self.cache:
             ir_node = ir_node.set_cache_client(self.cache.cache_client)
@@ -1989,6 +2054,18 @@ class Dataset:
 
         return ir_node
 
+    def set_init_step(self, init_step):
+        self._global_step = init_step
+
+    def get_init_step(self):
+        if self._global_step is not None:
+            return self._global_step
+        if len(self.children) == 1:
+            return self.children[0].get_init_step()
+        # When there are multiple children, we cannot tell from which child to get the initial step,
+        # so we initialize from the beginning
+        return 0
+
 
 class VisionBaseDataset(Dataset):
     """
@@ -2373,7 +2450,7 @@ class MappableDataset(SourceDataset):
                shard may not be part of the same split.
 
         Returns:
-            tuple(Dataset), a tuple of datasets that have been split.
+            Tuple[Dataset], a tuple of new datasets split from the original one.
 
         Raises:
             RuntimeError: If get_dataset_size returns None or is not supported for this dataset.
@@ -2455,7 +2532,7 @@ class BucketBatchByLengthDataset(UnionBaseDataset):
                                            self.pad_to_bucket_boundary, self.drop_remainder)
 
 
-def _check_shm_usage(num_worker, queue_size, max_rowsize, num_queues=1):
+def _check_shm_usage(num_worker, queue_size, in_rowsize, out_rowsize):
     """
     Check sufficient shared memory is available for shared memory queues
     when training in parallel mode.
@@ -2467,8 +2544,8 @@ def _check_shm_usage(num_worker, queue_size, max_rowsize, num_queues=1):
         # on the ascend server is 8.
         if device_num > 1:
             device_num = min(device_num, 8)
-        shm_estimate_usage = device_num * num_worker * num_queues * \
-                             (queue_size + 2) * max_rowsize * 1024 * 1024
+        shm_estimate_usage = device_num * num_worker * \
+                             (queue_size + 2) * (in_rowsize + out_rowsize) * 1024 * 1024
         try:
             shm_available = psutil.disk_usage('/dev/shm').free
             if shm_estimate_usage >= threshold_ratio * shm_available:
@@ -2507,8 +2584,14 @@ class BatchDataset(UnionBaseDataset):
             len(output_columns). The size of this list must match the number of output
             columns of the last operation. Default: ``None``, output columns will have the same
             name as the input columns, i.e., the columns will be replaced.
-        max_rowsize(int, optional): Maximum size of row in MB that is used for shared memory allocation to copy
-            data between processes.  This is only used if python_multiprocessing is set to True. Default: 16.
+        max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
+            allocation to copy data between processes, the total occupied shared memory will increase as
+            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
+            used if python_multiprocessing is set to True. If it is an int value, it represents
+            ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
+            If it is a list, the first element represents the ``input_columns`` use this value as the unit to
+            create shared memory, and the second element represents ``output_columns`` use this value as the unit
+            to create shared memory. Default: 16.
 
     """
 
@@ -2534,7 +2617,10 @@ class BatchDataset(UnionBaseDataset):
 
         self.python_multiprocessing = python_multiprocessing
         self.process_pool = None
-        self.max_rowsize = max_rowsize
+        if isinstance(max_rowsize, int):
+            self.max_rowsize = [max_rowsize * self.batch_size] * 2
+        else:
+            self.max_rowsize = [max_rowsize[0] * self.batch_size, max_rowsize[1] * self.batch_size]
 
     def __del__(self):
         if hasattr(self, "process_pool") and self.process_pool is not None:
@@ -2604,7 +2690,7 @@ class BatchDataset(UnionBaseDataset):
                 self.num_parallel_workers = get_num_parallel_workers()
 
             self.process_pool = _PythonMultiprocessing(str(self), self.num_parallel_workers, [self.per_batch_map],
-                                                       self.max_rowsize * self.batch_size)
+                                                       self.max_rowsize)
             # Wrap per_batch_map into _PythonCallable
             self.per_batch_map = _PythonCallable(self.per_batch_map, 0, self.process_pool)
         else:
@@ -2614,39 +2700,53 @@ class BatchDataset(UnionBaseDataset):
 
 class BatchInfo(cde.CBatchInfo):
     """
-    Only the batch size function and per_batch_map of the batch operation can dynamically adjust parameters
-    based on the number of batches and epochs during training.
+    This class helps to get dataset information dynamically when the input of `batch_size` or `per_batch_map`
+    in `batch` operation is a callable object.
     """
 
     def get_batch_num(self):
         """
-        Return the batch number of the current batch.
+        Return the batch number being processed in current epoch, start from 0.
 
         Examples:
             >>> # Create a dataset where its batch size is dynamic
             >>> # Define a callable batch size function and let batch size increase 1 each time.
             >>> import mindspore.dataset as ds
             >>> from mindspore.dataset import BatchInfo
-            >>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
+            >>>
+            >>> dataset = ds.GeneratorDataset([i for i in range(3)], "column1", shuffle=False)
             >>> def add_one(BatchInfo):
             ...     return BatchInfo.get_batch_num() + 1
             >>> dataset = dataset.batch(batch_size=add_one)
+            >>> print(list(dataset))
+            [[Tensor(shape=[1], dtype=Int64, value= [0])], [Tensor(shape=[2], dtype=Int64, value= [1, 2])]]
         """
         return
 
     def get_epoch_num(self):
         """
-        Return the epoch number of the current batch.
+        Return the epoch number, start from 0.
 
         Examples:
             >>> # Create a dataset where its batch size is dynamic
             >>> # Define a callable batch size function and let batch size increase 1 each epoch.
             >>> import mindspore.dataset as ds
             >>> from mindspore.dataset import BatchInfo
-            >>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
+            >>>
+            >>> dataset = ds.GeneratorDataset([i for i in range(4)], "column1", shuffle=False)
             >>> def add_one_by_epoch(BatchInfo):
             ...     return BatchInfo.get_epoch_num() + 1
             >>> dataset = dataset.batch(batch_size=add_one_by_epoch)
+            >>>
+            >>> result = []
+            >>> epoch = 2
+            >>> iterator = dataset.create_tuple_iterator(num_epochs=epoch)
+            >>> for i in range(epoch):
+            ...    result.extend(list(iterator))
+            >>> # result:
+            >>> # [[Tensor(shape=[1], dtype=Int64, value= [0])], [Tensor(shape=[1], dtype=Int64, value= [1])],
+            >>> #  [Tensor(shape=[1], dtype=Int64, value= [2])], [Tensor(shape=[1], dtype=Int64, value= [3])],
+            >>> #  [Tensor(shape=[2], dtype=Int64, value= [0, 1])], [Tensor(shape=[2], dtype=Int64, value= [2, 3])]]
         """
         return
 
@@ -2934,6 +3034,7 @@ class _PythonCallable:
         return self.py_callable.to_json()
 
 
+# used when python_multiprocessing=True in map
 class Pipe:
     """
     Class to handle communication between the master process and the worker processes.
@@ -2943,13 +3044,12 @@ class Pipe:
         self.shared_memory = shared_memory
         self.eof = multiprocessing.Event()
         if self.shared_memory:
-            self.in_queue = _SharedQueue(1, warning_ctl, max_rowsize=max_rowsize)
-            self.res_queue = _SharedQueue(1, warning_ctl, max_rowsize=max_rowsize)
+            self.in_queue = _SharedQueue(1, warning_ctl, max_rowsize=max_rowsize[0])
+            self.res_queue = _SharedQueue(1, warning_ctl, max_rowsize=max_rowsize[1])
         else:
             self.in_queue = _Queue(1)
             self.res_queue = _Queue(1)
-        self.in_queue._joincancelled = True  # pylint: disable=W0212
-        self.res_queue._joincancelled = True  # pylint: disable=W0212
+        self.in_queue.cancel_join_thread()  # Ensure that the process does not hung when exiting
 
     def master_send(self, func_index, data):
         self.in_queue.put_nowait((func_index, *data))
@@ -2965,8 +3065,6 @@ class Pipe:
         self.eof.set()
         self.send_finish_signal_to_worker()
         self.send_finish_signal()
-        self.res_queue.cancel_join_thread()
-        self.in_queue.cancel_join_thread()
 
     def send_finish_signal(self):
         self.worker_send(None)
@@ -2986,10 +3084,6 @@ class Pipe:
         func_index, *data = result
         return func_index, tuple(data)
 
-    def worker_close(self):
-        self.res_queue.cancel_join_thread()
-        self.in_queue.cancel_join_thread()
-
 
 def _main_process_already_exit():
     """
@@ -3007,6 +3101,8 @@ def _worker_loop(operations, pipe, seed=get_seed()):
     """
     Multiprocess worker process loop.
     """
+    # Ensure that the process does not hung when exiting
+    pipe.res_queue.cancel_join_thread()
 
     def _ignore_sigint():
         """
@@ -3022,11 +3118,9 @@ def _worker_loop(operations, pipe, seed=get_seed()):
 
         result = pipe.worker_receive()
         if result is None:
-            pipe.worker_close()
             return
         (idx, input_tensors) = result
         if input_tensors == "QUIT":
-            pipe.worker_close()
             break
         try:
             output_tensors = operations[idx](*input_tensors)
@@ -3093,6 +3187,9 @@ class _MPWorker(multiprocessing.Process):
     def close(self):
         try:
             if self.is_alive():
+                # release the eager executor which is used by current process
+                transforms.transforms.clean_unused_executors()
+
                 logger.info(f"Closing worker with PID: {self.pid}")
                 self.pipe.master_close()
                 # del the handle which hold by master
@@ -3140,12 +3237,12 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
             self.origin_hook(ex_type, value, tb)
             self.mp_pool_exit_preprocess()
 
-    def __init__(self, op_name, num_parallel_workers, operations, max_row_size=16):
+    def __init__(self, op_name, num_parallel_workers, operations, max_rowsize=16):
         super(_PythonMultiprocessing, self).__init__()
         self.op_name = op_name
         self.num_parallel_workers = num_parallel_workers
         self.operations = operations
-        self.max_row_size = max_row_size
+        self.max_rowsize = max_rowsize
 
         self.workers = None
         self.pids = None
@@ -3336,7 +3433,7 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
 
         """
         if get_enable_shared_mem():
-            self.check_shared_memory()
+            _check_shm_usage(self.num_parallel_workers, 1, self.max_rowsize[0], self.max_rowsize[1])
 
         if self.workers is not None:
             raise Exception("Pool was already created, close it first.")
@@ -3348,7 +3445,7 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         self.workers = []
         self.warning_ctl = multiprocessing.Value('i', 0)
         for i in range(self.num_parallel_workers):
-            worker = _MPWorker(self.operations, self.warning_ctl, self.max_row_size, i + get_seed())
+            worker = _MPWorker(self.operations, self.warning_ctl, self.max_rowsize, i + get_seed())
             worker.start()
             self.workers.append(worker)
 
@@ -3410,12 +3507,6 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
     def is_mp_enabled(self):
         return self.workers is not None
 
-    def check_shared_memory(self):
-        """
-        Check if there is enough shared memory in the system.
-        """
-        _check_shm_usage(self.num_parallel_workers, 1, self.max_row_size, 2)
-
     def execute(self, idx, *args):
         """
         Execute
@@ -3524,8 +3615,13 @@ class MapDataset(UnionBaseDataset):
         cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
             Default: ``None``, which means no cache is used.
         callbacks (DSCallback, list[DSCallback], optional): List of Dataset callbacks to be called. Default: ``None``.
-        max_rowsize(int, optional): Maximum size of row in MB that is used for shared memory allocation to copy
-            data between processes. This is only used if python_multiprocessing is set to True. Default: 16.
+        max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
+            allocation to copy data between processes, the total occupied shared memory will increase as
+            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
+            used if python_multiprocessing is set to True. If it is an int value, it represents ``input_columns`` and
+            ``output_columns`` use this value as the unit to create shared memory. If it is a list, the first element
+            represents the ``input_columns`` use this value as the unit to create shared memory, and the second element
+            represents ``output_columns`` use this value as the unit to create shared memory. Default: 16.
         offload (bool, optional): Flag to indicate whether offload is used. Default: ``None``.
     """
 
@@ -3556,7 +3652,10 @@ class MapDataset(UnionBaseDataset):
         self.process_pool = None
 
         self.callbacks = to_list(callbacks)
-        self.max_rowsize = max_rowsize
+        if isinstance(max_rowsize, int):
+            self.max_rowsize = [max_rowsize] * 2
+        else:
+            self.max_rowsize = max_rowsize
         self.offload = offload
 
     def parse(self, children=None):
@@ -3594,7 +3693,7 @@ class MapDataset(UnionBaseDataset):
 
         callbacks = [cb.create_runtime_obj() for cb in self.callbacks]
         return cde.MapNode(children[0], self.operations, self.input_columns, self.output_columns,
-                           callbacks, self.max_rowsize, OffloadToManualOffloadMode.get(self.offload), self.process_pool)
+                           callbacks, OffloadToManualOffloadMode.get(self.offload), self.process_pool)
 
     def __deepcopy__(self, memodict):
         return self.__safe_deepcopy__(memodict, exclude=("operations", "callbacks", "__transfer_dataset__"))
@@ -3898,6 +3997,8 @@ class ConcatDataset(UnionBaseDataset):
                                  "valid samples in the dataset." % child_index)
             child_index += 1
 
+        self._children_sizes = self.children_sizes_.copy()
+
         # _children_flag_and_nums: A list of pair<int ,int>.The first element of pair is flag that characterizes
         # whether the dataset is mappable. The second element of pair is length of the dataset
         self._children_flag_and_nums = []
@@ -3921,7 +4022,8 @@ class ConcatDataset(UnionBaseDataset):
                 self._children_flag_and_nums.append((1, dataset_len))
 
     def parse(self, children=None):
-        return cde.ConcatNode(children, self._sampler, self._children_flag_and_nums, self._children_start_end_index_)
+        return cde.ConcatNode(children, self._sampler, self._children_flag_and_nums, self._children_start_end_index_,
+                              self._children_sizes)
 
     def use_sampler(self, sampler):
         """
@@ -3937,8 +4039,19 @@ class ConcatDataset(UnionBaseDataset):
             ValueError: If the parameter NumSamples of sampler is not None.
             ValueError: If num_shards <=0.
         """
-        if not isinstance(sampler, samplers.DistributedSampler):
-            raise TypeError("The parameter %s of concat must be DistributedSampler!" % sampler)
+        if not isinstance(sampler, (samplers.DistributedSampler, samplers.RandomSampler)):
+            raise TypeError("The parameter %s of concat must be DistributedSampler or RandomSampler!" % sampler)
+
+        if isinstance(sampler, samplers.RandomSampler):
+            if sampler.replacement:
+                raise ValueError("The parameter replacement of RandomSampler must be False!")
+
+            if sampler.get_num_samples() is not None:
+                raise ValueError("The parameter num_samples of RandomSampler is not support to be set!")
+
+            self._sampler = sampler
+            self._children_sizes = [c.get_dataset_size() for c in self.children]
+            return
 
         if sampler.is_shuffled():
             raise ValueError("The parameter shuffle of DistributedSampler must be False!")
@@ -4041,7 +4154,12 @@ class _ToDevice:
         self._runtime_context = cde.PythonRuntimeContext()
         self._runtime_context.Init()
         self._to_device = cde.ToDevice(num_epochs)
-        self._to_device.Init(ir_tree)
+        if dataset.get_init_step() != 0:
+            init_step = dataset.get_init_step()
+            dataset_size = dataset.get_dataset_size()
+            self._to_device.Init(ir_tree, init_step, dataset_size)
+        else:
+            self._to_device.Init(ir_tree, 0, -1)
         self._runtime_context.AssignConsumer(self._to_device)
 
         ITERATORS_LIST.append(weakref.ref(self))
@@ -4068,6 +4186,14 @@ class _ToDevice:
         """
         return self._to_device.GetDataInfo()
 
+    def get_send_info(self):
+        """
+        In sink mode, it returns the send information of dataset at this moment.
+        Send information includes number of send batches, time summary of fetching data on host
+        and time summary of sending data.
+        """
+        return self._to_device.GetSendInfo()
+
     def release(self):
         """
         Manually terminate Device Queue instead of relying on out of scope destruction.
@@ -4088,8 +4214,8 @@ class _ToDevice:
         offload_model = GetOffloadModel(self._to_device, col_names)
         return offload_model
 
-    def _reset(self, step, epoch):
-        self._to_device.Reset(step, epoch)
+    def _reset(self, step, dataset_size):
+        self._to_device.Reset(step, dataset_size)
 
 
 class TransferDataset(Dataset):
@@ -4108,9 +4234,14 @@ class TransferDataset(Dataset):
         RuntimeError: If dataset is unknown.
     """
 
-    def __init__(self, input_dataset, send_epoch_end=True, create_data_info_queue=False):
+    def __init__(self, input_dataset, send_epoch_end=True, create_data_info_queue=False, queue_name=""):
         super().__init__(children=input_dataset)
-        self.queue_name = str(uuid.uuid1())
+        if queue_name == "":
+            self.queue_name = str(uuid.uuid1())
+            logger.info(f"queue_name is newly generated. value is {self.queue_name}")
+        else:
+            self.queue_name = queue_name
+            logger.info(f"queue_name is read from compile cache. value is {self.queue_name}")
         self.device_type = context.get_context("device_target") if context else "CPU"
         self.device_id = context.get_context("device_id") if context else 0
 
@@ -4169,6 +4300,16 @@ class TransferDataset(Dataset):
             return self._to_device.get_data_info()
         raise RuntimeError("Calling get_data_info with bad state.")
 
+    def get_send_info(self):
+        """
+        In sink mode, it returns the send information of dataset at this moment.
+        Send information includes number of send batches, time summary of fetching data on host
+        and time summary of sending data.
+        """
+        if self._to_device is not None:
+            return self._to_device.get_send_info()
+        raise RuntimeError("Calling get_send_info with bad state, data queue is not initialized.")
+
     def get_offload_model(self):
         if self._to_device is not None:
             return self._to_device.get_offload_model(self.column_name)
@@ -4182,10 +4323,10 @@ class TransferDataset(Dataset):
         if self._to_device is not None:
             self._to_device.release()
 
-    def _reset(self, step, epoch):
+    def _reset(self, step, dataset_size):
         if self._to_device is not None:
-            logger.info("Reset the dataset pipeline to step: " + str(step) + ", epoch: " + str(epoch))
-            self._to_device._reset(step, epoch)  # pylint: disable=protected-access
+            logger.info("Reset the dataset pipeline to step: " + str(step) + ", epoch: " + str(step // dataset_size))
+            self._to_device._reset(step, dataset_size)  # pylint: disable=protected-access
 
 
 class Schema:
@@ -4193,10 +4334,7 @@ class Schema:
     Class to represent a schema of a dataset.
 
     Args:
-        schema_file(str): Path of the schema file. Default: ``None``.
-
-    Returns:
-        Schema object, schema info about dataset.
+        schema_file (str): Path of the schema file. Default: ``None``.
 
     Raises:
         RuntimeError: If schema file failed to load.
@@ -4306,7 +4444,7 @@ class Schema:
             >>> import json
             >>> from mindspore.dataset import Schema
             >>>
-            >>> with open("/path/to/schema_file") as file:
+            >>> with open("/path/to/schema_file", "r") as file:
             ...     json_obj = json.load(file)
             ...     schema = Schema()
             ...     schema.from_json(json_obj)