mindspore 2.3.0__cp310-cp310-win_amd64.whl → 2.4.1__cp310-cp310-win_amd64.whl

mindspore/dataset/engine/datasets.py

@@ -1,4 +1,4 @@
-# Copyright 2022-2023 Huawei Technologies Co., Ltd
+# Copyright 2022-2024 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,13 +16,13 @@
 1. This file is an abstraction of the dataset loading class. It contains
 some basic dataset operations(skip, filter, map, batch, ...).
 2. Specific dataset loading classes can be found in datasets_vision.py, datasets_text.py,
-datasets_audio.py, datasets_standard_format.py and dataets_user_defined.py files.
+datasets_audio.py, datasets_standard_format.py and datasets_user_defined.py files.
     datasets_vision.py: contains vision dataset loading classes.
     datasets_text.py: contains text dataset loading classes.
     datasets_audio.py: contains audio dataset loading classes.
     datasets_standard_format.py: contains standard format loading classes which
                                  any other kinds of datasets can be converted to.
-    dataets_user_defined.py: contains basic classes that help users to define
+    datasets_user_defined.py: contains basic classes that help users to define
                              flexible ways to load dataset.
 """
 import atexit
@@ -66,13 +66,13 @@ from mindspore.dataset.debug import DebugHook
 
 from mindspore.dataset.engine import samplers
 from .iterators import DictIterator, TupleIterator, DummyIterator, check_iterator_cleanup, _set_iterator_cleanup, \
-    ITERATORS_LIST, _unset_iterator_cleanup
+    ITERATORS_LIST, _unset_iterator_cleanup, _cleanup_the_iterators_if_created
 from .queue import _SharedQueue, _Queue
 from .validators import check_batch, check_shuffle, check_map, check_filter, check_repeat, check_skip, check_zip, \
     check_rename, check_device_send, check_take, check_output_shape, check_project, \
     check_sync_wait, check_zip_dataset, check_add_column, check_concat, check_split, check_bucket_batch_by_length, \
     check_save, check_tuple_iterator, check_dict_iterator, check_schema, check_to_device_send, check_padded_batch, \
-    check_total_batch
+    check_total_batch, check_sync_update
 from ..core.config import get_callback_timeout, _init_device_info, get_enable_shared_mem, get_num_parallel_workers, \
     get_enable_watchdog, get_seed, set_seed, get_debug_mode, get_multiprocessing_timeout_interval, _get_debug_hook_list
 from ..core.datatypes import mstype_to_detype
@@ -494,6 +494,12 @@ class Dataset:
 
         .. image:: bucket_batch_by_length_en.png
 
+        Note:
+            - When using `Data Sinking <https://www.mindspore.cn/docs/en/master/model_train/train_process/optimize/
+              sink_mode.html#data-sinking>`_ in Graph mode, the input shape of the network should keep consistent.
+              You should set `drop_remainder` to "True" to discard the last incomplete batch of data,
+              or supplement/remove samples to ensure the dataset size is divisible by `batch_size`.
+
         Args:
             column_names (list[str]): Columns passed to element_length_function.
             bucket_boundaries (list[int]): A list consisting of the upper boundaries
@@ -564,8 +570,12 @@ class Dataset:
         .. image:: batch_en.png
 
         Note:
-            The order of using repeat and batch reflects the number of batches and per_batch_map.
-            It is recommended that the repeat operation applied after the batch operation finished.
+            - The order of using repeat and batch reflects the number of batches and per_batch_map.
+              It is recommended that the repeat operation applied after the batch operation finished.
+            - When using `Data Sinking <https://www.mindspore.cn/docs/en/master/model_train/train_process/optimize/
+              sink_mode.html#data-sinking>`_ in Graph mode, the input shape of the network should keep consistent.
+              You should set `drop_remainder` to "True" to discard the last incomplete batch of data,
+              or supplement/remove samples to ensure the dataset size is divisible by `batch_size`.
 
         Args:
             batch_size (Union[int, Callable]): The number of rows each batch is created with. An
@@ -598,10 +608,10 @@ class Dataset:
                   name as the input columns, i.e., the columns will be replaced.
 
                 - python_multiprocessing (bool, optional): Parallelize Python function `per_batch_map` with
-                  multi-processing or multi-threading mode, ``True`` means multi-processing,
-                  ``False`` means multi-threading If `per_batch_map` is a I/O bound task, use
-                  multi-threading mode. If `per_batch_map` is a CPU bound task, it is recommended to use
-                  multi-processing mode. Default: ``False`` , use python multi-threading mode.
+                  multiprocessing or multithreading mode, ``True`` means multiprocessing,
+                  ``False`` means multithreading If `per_batch_map` is a I/O bound task, use
+                  multithreading mode. If `per_batch_map` is a CPU bound task, it is recommended to use
+                  multiprocessing mode. Default: ``False`` , use python multithreading mode.
 
                 - max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
                   allocation to copy data between processes, the total occupied shared memory will increase as
@@ -611,7 +621,7 @@ class Dataset:
                   ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
                   If it is a list, the first element represents the ``input_columns`` use this value as the unit to
                   create shared memory, and the second element represents ``output_columns`` use this value as the unit
-                  to create shared memory. Default: 16.
+                  to create shared memory. Default: ``None`` , allocate shared memory dynamically.
 
         Returns:
             Dataset, a new dataset with the above operation applied.
@@ -657,8 +667,12 @@ class Dataset:
         .. image:: padded_batch_en.png
 
         Note:
-            The order of using repeat and padded_batch reflects the number of batches.
-            It is recommended that the repeat operation applied after the padded_batch operation finished.
+            - The order of using repeat and padded_batch reflects the number of batches.
+              It is recommended that the repeat operation applied after the padded_batch operation finished.
+            - When using `Data Sinking <https://www.mindspore.cn/docs/en/master/model_train/train_process/optimize/
+              sink_mode.html#data-sinking>`_ in Graph mode, the input shape of the network should keep consistent.
+              You should set `drop_remainder` to "True" to discard the last incomplete batch of data,
+              or supplement/remove samples to ensure the dataset size is divisible by `batch_size`.
 
         Args:
             batch_size (Union[int, Callable]): The number of rows each batch is created with. An
@@ -905,7 +919,7 @@ class Dataset:
                   ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
                   If it is a list, the first element represents the ``input_columns`` use this value as the unit to
                   create shared memory, and the second element represents ``output_columns`` use this value as the unit
-                  to create shared memory. Default: 16.
+                  to create shared memory. Default: ``None`` , allocate shared memory dynamically.
 
                 - cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
                   Default: ``None``, which means no cache is used.
@@ -989,8 +1003,8 @@ class Dataset:
             num_parallel_workers = 1
             logger.warning(
                 "Input 'operations' of 'map' includes network computing operators like in mindspore.nn, mindspore.ops, "
-                "mindspore.numpy module and etc, which do not support multi-thread compiling, recommend to replace it "
-                "with python implemented operator like numpy etc. Here decrease 'num_parallel_workers' into 1.")
+                "mindspore.numpy module and etc, which do not support multithreading compiling, recommend to replace "
+                "it with python implemented operator like numpy etc. Here decrease 'num_parallel_workers' into 1.")
 
         return MapDataset(self, operations, input_columns, output_columns, num_parallel_workers, **kwargs)
 
@@ -1523,8 +1537,8 @@ class Dataset:
             2. Before calling the function, do not use batch operation, repeat operation or data augmentation operations
                with random attribute in map operation.
             3. When array dimension is variable, one-dimensional arrays or
-               multi-dimensional arrays with variable dimension 0 are supported.
-            4. MindRecord does not support multi-dimensional string or multi-dimensional bytes.
+               multidimensional arrays with variable dimension 0 are supported.
+            4. MindRecord does not support multidimensional string or multidimensional bytes.
 
         Args:
             file_name (str): Path to dataset file.
@@ -1741,6 +1755,7 @@ class Dataset:
         return self._col_names
 
     @check_output_shape
+    @_cleanup_the_iterators_if_created
     def output_shapes(self, estimate=False):
         """
         Get the shapes of output data.
@@ -1792,6 +1807,7 @@ class Dataset:
             self.saved_output_shapes = output_shapes
         return output_shapes
 
+    @_cleanup_the_iterators_if_created
     def output_types(self):
         """
         Get the types of output data.
@@ -1826,6 +1842,7 @@ class Dataset:
             del self.runtime_context
         return self.saved_output_types
 
+    @_cleanup_the_iterators_if_created
     def get_dataset_size(self):
         """
         Return the number of batches in an epoch.
@@ -1893,6 +1910,7 @@ class Dataset:
             return self.children[0].is_sync()
         return False
 
+    @check_sync_update
     def sync_update(self, condition_name, num_batch=None, data=None):
         """
         Release a blocking condition and trigger callback with given data.
@@ -2174,7 +2192,7 @@ class TextBaseDataset(Dataset):
                 Japanese or Chinese character sets, and 1.0 for other languages with small character sets
                 like English or Latin.
             model_type(SentencePieceModel): Model type. Choose from unigram (default), bpe, char, or word.
-                The input sentence must be pretokenized when using word type.
+                The input sentence must be pre-tokenized when using word type.
             params(dict): Any extra optional parameters of sentencepiece library according to your raw data
 
         Returns:
@@ -2251,7 +2269,7 @@ class TextBaseDataset(Dataset):
                 Japanese or Chinese character sets, and 1.0 for other languages with small character sets
                 like English or Latin.
             model_type(SentencePieceModel): Model type. Choose from unigram (default), bpe, char, or word.
-                The input sentence must be pretokenized when using word type.
+                The input sentence must be pre-tokenized when using word type.
             params(dict): Any extra optional parameters of sentencepiece library according to your raw data
 
         Returns:
@@ -2629,12 +2647,12 @@ class BatchDataset(UnionBaseDataset):
             ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
             If it is a list, the first element represents the ``input_columns`` use this value as the unit to
             create shared memory, and the second element represents ``output_columns`` use this value as the unit
-            to create shared memory. Default: 16.
+            to create shared memory. Default: ``None`` , allocate shared memory dynamically.
 
     """
 
     def __init__(self, input_dataset, batch_size, drop_remainder=False, num_parallel_workers=None, per_batch_map=None,
-                 input_columns=None, output_columns=None, python_multiprocessing=False, max_rowsize=16):
+                 input_columns=None, output_columns=None, python_multiprocessing=False, max_rowsize=None):
         super().__init__(children=input_dataset, num_parallel_workers=num_parallel_workers)
 
         if BatchDataset._is_ancestor_of_repeat(input_dataset):
@@ -2655,7 +2673,9 @@ class BatchDataset(UnionBaseDataset):
 
         self.python_multiprocessing = python_multiprocessing
         self.process_pool = None
-        if isinstance(max_rowsize, int):
+        if max_rowsize is None:
+            self.max_rowsize = [-1, -1]
+        elif isinstance(max_rowsize, int):
             self.max_rowsize = [max_rowsize * self.batch_size] * 2 if max_rowsize != -1 else [max_rowsize, max_rowsize]
         else:
             self.max_rowsize = [max_rowsize[0] * self.batch_size, max_rowsize[1] * self.batch_size]
@@ -3078,7 +3098,7 @@ class Pipe:
     Class to handle communication between the master process and the worker processes.
     """
 
-    def __init__(self, warning_ctl, shared_memory=False, max_rowsize=16):
+    def __init__(self, warning_ctl, shared_memory=False, max_rowsize=(-1, -1)):
         self.shared_memory = shared_memory
         self.eof = multiprocessing.Event()
         if self.shared_memory:
@@ -3139,7 +3159,10 @@ def _worker_loop(operations, pipe, worker_id):
     """
     Multiprocess worker process loop.
     """
-    # Ensure that the process does not hung when exiting
+    # Initialize C++ side signal handlers
+    cde.register_worker_handlers()
+
+    # Ensure that the process does not hang when exiting
     pipe.res_queue.cancel_join_thread()
 
     def _ignore_sigint():
@@ -3153,6 +3176,7 @@ def _worker_loop(operations, pipe, worker_id):
     # that the random results of each process are different.
     if get_seed() != 5489:
         set_seed(get_seed() + worker_id)
+
     while not _main_process_already_exit():
         _ignore_sigint()
 
@@ -3184,7 +3208,7 @@ class _MPWorker(multiprocessing.Process):
     Worker process for multiprocessing.
     """
 
-    def __init__(self, operations, warning_ctl, max_rowsize=16, worker_id=0):
+    def __init__(self, operations, warning_ctl, max_rowsize=(-1, -1), worker_id=0):
         shared_memory = get_enable_shared_mem()
         self.pipe = Pipe(warning_ctl, shared_memory=shared_memory, max_rowsize=max_rowsize)
         self.check_interval = get_multiprocessing_timeout_interval()
@@ -3216,14 +3240,6 @@ class _MPWorker(multiprocessing.Process):
                     logger.warning("Please `pip install py-spy` to get the stacks of the stuck process.")
             try:
                 res = self.pipe.master_receive()
-                # Because there is no need to copy when creating Tensors in the C++layer, it reduces the time
-                # from np.ndarray to C++Tensor creation. However, when using shared memory in multiple processes,
-                # the address of the shared memory will always be passed to subsequent nodes in the dataset pipeline,
-                # and the shared memory will also be written by the current node, causing dirty data to be accessed
-                # by subsequent nodes in the pipeline. So make a memory copy here to solve the problem of
-                # shared memory being contaminated.
-                if get_enable_shared_mem():
-                    res = copy.deepcopy(res)
             except queue.Empty:
                 continue
             if res is None:
@@ -3286,7 +3302,7 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
             self.origin_hook(ex_type, value, tb)
             self.mp_pool_exit_preprocess()
 
-    def __init__(self, op_name, num_parallel_workers, operations, max_rowsize=16):
+    def __init__(self, op_name, num_parallel_workers, operations, max_rowsize=(-1, -1)):
         super(_PythonMultiprocessing, self).__init__()
         self.op_name = op_name
         self.num_parallel_workers = num_parallel_workers
@@ -3302,7 +3318,7 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
 
         self.eot = None
         self.watch_dog = None
-        self.ppid = os.getpid()
+        self.ppid = None
         self.hook = None
         self.warning_ctl = None
         # cache thread (get_ident()) to worker_id mapping in Python layer
@@ -3327,10 +3343,10 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
                 if child_pid == 0:
                     break
         except OSError:
-            # waitpid may be failed for some reasons so we ignore this error
+            # waitpid may fail for some reason, so we ignore this error
             pass
 
-    # Dataset need watch_dog thread to monitoring fork multi-processing,
+    # Dataset need watch_dog thread to monitoring fork multiprocessing,
     # and thread can't be a member function otherwise python won't collect and release resources.
     @staticmethod
     def _watch_dog(eot, workers):
@@ -3363,6 +3379,8 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
                                 "main process will exit. If this is not an artificial operation, you can use "
                                 "ds.config.set_enable_watchdog(False) to block this error.")
                 os.kill(os.getpid(), signal.SIGTERM)
+            # sleep to release GIL
+            time.sleep(1)
 
         # release the workers
         del workers
@@ -3451,6 +3469,12 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         while _PythonMultiprocessing.is_process_alive(ppid):
             if quit_signal.is_set():
                 return
+
+            # independent dataset mode, the subprocess of GeneratorDataset / map / batch should exit when
+            # independent dataset process have exit
+            if os.getppid() != ppid:
+                break
+
             time.sleep(0.1)
 
         _PythonMultiprocessing._terminate_processes(workers)
@@ -3462,10 +3486,10 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         Launch Python multiprocessing pool.
 
         Args:
-            pop_id: ID for operation to have Python multiprocessing pool launched
+            op_id: ID for operation to have Python multiprocessing pool launched
 
         Returns:
-            Python multiprocssing pool is launched.
+            Python multiprocessing pool is launched.
         """
         self.python_threads_to_workers = {}
         self.op_id = op_id
@@ -3476,6 +3500,7 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
             logger.warning(message)
             self.terminate()
             self.reset()
+        self.ppid = os.getpid()
         self.create_pool()
 
     def create_pool(self):
@@ -3677,12 +3702,13 @@ class MapDataset(UnionBaseDataset):
             ``python_multiprocessing`` is set to True. If it is an int value, it represents ``input_columns`` and
             ``output_columns`` use this value as the unit to create shared memory. If it is a list, the first element
             represents the ``input_columns`` use this value as the unit to create shared memory, and the second element
-            represents ``output_columns`` use this value as the unit to create shared memory. Default: 16.
+            represents ``output_columns`` use this value as the unit to create shared memory. Default: ``None`` ,
+            allocate shared memory dynamically.
         offload (bool, optional): Flag to indicate whether offload is used. Default: ``None``.
     """
 
     def __init__(self, input_dataset, operations=None, input_columns=None, output_columns=None,
-                 num_parallel_workers=None, python_multiprocessing=False, cache=None, callbacks=None, max_rowsize=16,
+                 num_parallel_workers=None, python_multiprocessing=False, cache=None, callbacks=None, max_rowsize=None,
                  offload=None):
         super().__init__(children=input_dataset, num_parallel_workers=num_parallel_workers, cache=cache)
         self.operations = to_list(operations)
@@ -3708,7 +3734,9 @@ class MapDataset(UnionBaseDataset):
         self.process_pool = None
 
         self.callbacks = to_list(callbacks)
-        if isinstance(max_rowsize, int):
+        if max_rowsize is None:
+            self.max_rowsize = [-1, -1]
+        elif isinstance(max_rowsize, int):
             self.max_rowsize = [max_rowsize] * 2
         else:
             self.max_rowsize = max_rowsize

mindspore/dataset/engine/datasets_audio.py

@@ -63,7 +63,7 @@ class CMUArcticDataset(MappableDataset, AudioBaseDataset):
         shard_id (int, optional): The shard ID within `num_shards` . Default: ``None``, will use ``0``. This
             argument can only be specified when `num_shards` is also specified.
         cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
-            `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ .
+            `Single-Node Data Cache <https://www.mindspore.cn/docs/en/master/model_train/dataset/cache.html>`_ .
             Default: ``None``, which means no cache is used.
 
     Raises:
@@ -180,7 +180,7 @@ class GTZANDataset(MappableDataset, AudioBaseDataset):
         shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
             argument can only be specified when `num_shards` is also specified.
         cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
-            `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ .
+            `Single-Node Data Cache <https://www.mindspore.cn/docs/en/master/model_train/dataset/cache.html>`_ .
             Default: ``None`` , which means no cache is used.
 
     Raises:
@@ -298,7 +298,7 @@ class LibriTTSDataset(MappableDataset, AudioBaseDataset):
         shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
             argument can only be specified when `num_shards` is also specified.
         cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
-            `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ .
+            `Single-Node Data Cache <https://www.mindspore.cn/docs/en/master/model_train/dataset/cache.html>`_ .
             Default: ``None`` , which means no cache is used.
 
     Raises:
@@ -425,7 +425,7 @@ class LJSpeechDataset(MappableDataset, AudioBaseDataset):
         shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
             argument can only be specified when `num_shards` is also specified.
         cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
-            `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ .
+            `Single-Node Data Cache <https://www.mindspore.cn/docs/en/master/model_train/dataset/cache.html>`_ .
             Default: ``None`` , which means no cache is used.
 
     Raises:
@@ -548,7 +548,7 @@ class SpeechCommandsDataset(MappableDataset, AudioBaseDataset):
         shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` .
             This argument can only be specified when `num_shards` is also specified.
         cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
-            `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ .
+            `Single-Node Data Cache <https://www.mindspore.cn/docs/en/master/model_train/dataset/cache.html>`_ .
             Default: ``None`` , which means no cache is used.
 
     Raises:
@@ -661,7 +661,7 @@ class TedliumDataset(MappableDataset, AudioBaseDataset):
         shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
             argument can only be specified when `num_shards` is also specified.
         cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
-            `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ .
+            `Single-Node Data Cache <https://www.mindspore.cn/docs/en/master/model_train/dataset/cache.html>`_ .
             Default: ``None`` , which means no cache is used.
 
     Raises:
@@ -841,7 +841,7 @@ class YesNoDataset(MappableDataset, AudioBaseDataset):
         shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This argument can only
             be specified when `num_shards` is also specified.
         cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
-            `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ .
+            `Single-Node Data Cache <https://www.mindspore.cn/docs/en/master/model_train/dataset/cache.html>`_ .
             Default: ``None`` , which means no cache is used.
 
     Raises:

mindspore/dataset/engine/datasets_standard_format.py

@@ -77,7 +77,7 @@ class CSVDataset(SourceDataset, UnionBaseDataset):
         shard_id (int, optional): The shard ID within `num_shards` . Default: ``None``. This
             argument can only be specified when `num_shards` is also specified.
         cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
-            `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ .
+            `Single-Node Data Cache <https://www.mindspore.cn/docs/en/master/model_train/dataset/cache.html>`_ .
             Default: ``None``, which means no cache is used.
 
     Raises:
@@ -156,7 +156,7 @@ class MindDataset(MappableDataset, UnionBaseDataset):
         num_samples (int, optional): The number of samples to be included in the dataset.
             Default: ``None`` , all samples.
         cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
-            `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ .
+            `Single-Node Data Cache <https://www.mindspore.cn/docs/en/master/model_train/dataset/cache.html>`_ .
             Default: ``None`` , which means no cache is used.
 
     Raises:
@@ -166,6 +166,52 @@ class MindDataset(MappableDataset, UnionBaseDataset):
         RuntimeError: If `shard_id` is specified but `num_shards` is None.
         ValueError: If `shard_id` is not in range of [0, `num_shards` ).
 
+    Note:
+        - When sharding MindRecord (by configuring `num_shards` and `shard_id`), there are two strategies to implement
+          the data sharding logic. This API uses the strategy 2.
+
+        .. list-table:: Data sharding strategy 1
+            :widths: 50 50 50 50
+            :header-rows: 1
+
+            * - rank 0
+              - rank 1
+              - rank 2
+              - rank 3
+            * - 0
+              - 1
+              - 2
+              - 3
+            * - 4
+              - 5
+              - 6
+              - 7
+            * - 8
+              - 9
+              - 10
+              - 11
+
+        .. list-table:: Data sharding strategy 2
+            :widths: 50 50 50 50
+            :header-rows: 1
+
+            * - rank 0
+              - rank 1
+              - rank 2
+              - rank 3
+            * - 0
+              - 3
+              - 6
+              - 9
+            * - 1
+              - 4
+              - 7
+              - 10
+            * - 2
+              - 5
+              - 8
+              - 11
+
     Note:
         - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
           used in the dataset, and their effects when combined with parameter `sampler` are as follows.
@@ -307,7 +353,7 @@ class TFRecordDataset(SourceDataset, UnionBaseDataset):
             When `compression_type` is not ``None``, and `num_samples` or numRows (parsed from `schema` ) is provided,
             `shard_equal_rows` will be implied as ``True``.
         cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
-            `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ .
+            `Single-Node Data Cache <https://www.mindspore.cn/docs/en/master/model_train/dataset/cache.html>`_ .
             Default: ``None`` , which means no cache is used.
         compression_type (str, optional): The type of compression used for all files, must be either ``''``,
             ``'GZIP'``, or ``'ZLIB'``. Default: ``None`` , as in empty string. It is highly recommended to
@@ -383,6 +429,10 @@ class OBSMindDataset(GeneratorDataset):
 
     The columns of generated dataset depend on the source MindRecord files.
 
+    Note:
+        - This interface accesses the `/cache` directory for node synchronization and requires the user to ensure
+          access to the `/cache` directory.
+
     Args:
         dataset_files (list[str]): List of files in cloud storage to be read and file path is in
             the format of s3://bucketName/objectKey.