mindspore 2.3.0__cp39-none-any.whl → 2.3.0rc2__cp39-none-any.whl

mindspore/dataset/engine/datasets.py

@@ -71,8 +71,7 @@ from .queue import _SharedQueue, _Queue
 from .validators import check_batch, check_shuffle, check_map, check_filter, check_repeat, check_skip, check_zip, \
     check_rename, check_device_send, check_take, check_output_shape, check_project, \
     check_sync_wait, check_zip_dataset, check_add_column, check_concat, check_split, check_bucket_batch_by_length, \
-    check_save, check_tuple_iterator, check_dict_iterator, check_schema, check_to_device_send, check_padded_batch, \
-    check_total_batch
+    check_save, check_tuple_iterator, check_dict_iterator, check_schema, check_to_device_send, check_padded_batch
 from ..core.config import get_callback_timeout, _init_device_info, get_enable_shared_mem, get_num_parallel_workers, \
     get_enable_watchdog, get_seed, set_seed, get_debug_mode, get_multiprocessing_timeout_interval, _get_debug_hook_list
 from ..core.datatypes import mstype_to_detype
@@ -605,9 +604,8 @@ class Dataset:
 
                 - max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
                   allocation to copy data between processes, the total occupied shared memory will increase as
-                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set
-                  to -1, shared memory will be dynamically allocated with the actual size of data. This is only used if
-                  ``python_multiprocessing`` is set to True. If it is an int value, it represents
+                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
+                  used if python_multiprocessing is set to True. If it is an int value, it represents
                   ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
                   If it is a list, the first element represents the ``input_columns`` use this value as the unit to
                   create shared memory, and the second element represents ``output_columns`` use this value as the unit
@@ -899,9 +897,8 @@ class Dataset:
 
                 - max_rowsize (Union[int, list[int]], optional): Maximum size of row in MB that is used for shared
                   memory allocation to copy data between processes, the total occupied shared memory will increase as
-                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set
-                  to -1, shared memory will be dynamically allocated with the actual size of data. This is only used if
-                  ``python_multiprocessing`` is set to True. If it is an int value, it represents
+                  ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
+                  used if python_multiprocessing is set to True. If it is an int value, it represents
                   ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
                   If it is a list, the first element represents the ``input_columns`` use this value as the unit to
                   create shared memory, and the second element represents ``output_columns`` use this value as the unit
@@ -1571,34 +1568,32 @@ class Dataset:
     @check_tuple_iterator
     def create_tuple_iterator(self, columns=None, num_epochs=-1, output_numpy=False, do_copy=True):
         """
-        Create an iterator over the dataset that yields samples of type list, whose elements are
-        the data for each column.
+        Create an iterator over the dataset. The datatype retrieved back will be a list of `numpy.ndarray` .
+
+        To specify which columns to list and the order needed, use columns_list. If columns_list
+        is not provided, the order of the columns will remain unchanged.
 
         Args:
-            columns (list[str], optional): Specify the output columns and the order.
-                Default: ``None``, keep all the output columns and their original order.
-            num_epochs (int, optional): The number of epochs to iterate over the entire dataset.
-                Default: ``-1`` , the dataset can be iterated indefinitely.
-            output_numpy (bool, optional): Whether to keep the output data as NumPy ndarray, or
-                convert it to Tensor. Default: ``False`` .
-            do_copy (bool, optional): Whether to copy the data when converting output to Tensor,
-                or reuse the buffer for better performance, only works when `output_numpy` is ``False`` .
-                Default: ``True`` .
+            columns (list[str], optional): List of columns to be used to specify the order of columns.
+                Default: ``None``, means all columns.
+            num_epochs (int, optional): Maximum number of epochs that iterator can be iterated.
+                Default: ``-1``, iterator can be iterated infinite number of epochs.
+            output_numpy (bool, optional): Whether or not to output NumPy datatype.
+                If `output_numpy` is ``False``, iterator will output MSTensor. Default: ``False``.
+            do_copy (bool, optional): When output data type is :class:`mindspore.Tensor`,
+                use this param to select the conversion method, only take False for better performance.
+                Default: ``True``.
 
         Returns:
-            Iterator, a dataset iterator that yields samples of type list.
+            Iterator, a dataset iterator that returns data of type Tuple.
 
         Examples:
             >>> import mindspore.dataset as ds
-            >>>
-            >>> dataset = ds.GeneratorDataset([i for i in range(10)], "data")
-            >>> num_epochs = 3
-            >>> iterator = dataset.create_tuple_iterator(num_epochs=num_epochs)
-            >>> for epoch in range(num_epochs):
-            ...     for item in iterator:
-            ...         # output is of type tuple
-            ...         print(type(item))
-            ...         break
+            >>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
+            >>> iterator = dataset.create_tuple_iterator()
+            >>> for item in iterator:
+            ...     # item is a list
+            ...     print(type(item))
             ...     break
             <class 'list'>
         """
@@ -1612,32 +1607,27 @@ class Dataset:
     @check_dict_iterator
     def create_dict_iterator(self, num_epochs=-1, output_numpy=False, do_copy=True):
         """
-        Create an iterator over the dataset that yields samples of type dict,
-        while the key is the column name and the value is the data.
+        Create an iterator over the dataset. The data retrieved will be a dictionary datatype.
 
         Args:
-            num_epochs (int, optional): The number of epochs to iterate over the entire dataset.
-                Default: ``-1`` , the dataset can be iterated indefinitely.
-            output_numpy (bool, optional): Whether to keep the output data as NumPy ndarray, or
-                convert it to Tensor. Default: ``False`` .
-            do_copy (bool, optional): Whether to copy the data when converting output to Tensor,
-                or reuse the buffer for better performance, only works when `output_numpy` is ``False`` .
+            num_epochs (int, optional): Maximum number of epochs that iterator can be iterated.
+                Default: ``-1`` , iterator can be iterated infinite number of epochs.
+            output_numpy (bool, optional): Whether or not to output NumPy datatype,
+                if `output_numpy` is ``False``, iterator will output MSTensor. Default: ``False`` .
+            do_copy (bool, optional): When output data type is :class:`mindspore.Tensor`,
+                use this param to select the conversion method, only take False for better performance.
                 Default: ``True`` .
 
         Returns:
-            Iterator, a dataset iterator that yields samples of type dict.
+            Iterator, a dataset iterator that returns data of type Dict.
 
         Examples:
             >>> import mindspore.dataset as ds
-            >>>
-            >>> dataset = ds.GeneratorDataset([i for i in range(10)], "data")
-            >>> num_epochs = 3
-            >>> iterator = dataset.create_dict_iterator(num_epochs=num_epochs)
-            >>> for epoch in range(num_epochs):
-            ...     for item in iterator:
-            ...         # output is of type dict
-            ...         print(type(item))
-            ...         break
+            >>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
+            >>> iterator = dataset.create_dict_iterator()
+            >>> for item in iterator:
+            ...     # item is a dict
+            ...     print(type(item))
             ...     break
             <class 'dict'>
         """
@@ -2574,8 +2564,7 @@ def _check_shm_usage(num_worker, queue_size, in_rowsize, out_rowsize):
     when training in parallel mode.
     """
     threshold_ratio = 0.8
-    # Verify available size only when using static shared memory on Linux
-    if platform.system().lower() not in {"windows", "darwin"} and in_rowsize != -1 and out_rowsize != -1:
+    if platform.system().lower() not in {"windows", "darwin"}:
         device_num = _get_device_num()
         # In the cluster, _get_device_num indicates the number of the entire cluster. The maximum number of cards
         # on the ascend server is 8.
@@ -2623,9 +2612,8 @@ class BatchDataset(UnionBaseDataset):
             name as the input columns, i.e., the columns will be replaced.
         max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
             allocation to copy data between processes, the total occupied shared memory will increase as
-            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set to -1,
-            shared memory will be dynamically allocated with the actual size of data. This is only used if
-            ``python_multiprocessing`` is set to True. If it is an int value, it represents
+            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
+            used if python_multiprocessing is set to True. If it is an int value, it represents
             ``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
             If it is a list, the first element represents the ``input_columns`` use this value as the unit to
             create shared memory, and the second element represents ``output_columns`` use this value as the unit
@@ -2656,7 +2644,7 @@ class BatchDataset(UnionBaseDataset):
         self.python_multiprocessing = python_multiprocessing
         self.process_pool = None
         if isinstance(max_rowsize, int):
-            self.max_rowsize = [max_rowsize * self.batch_size] * 2 if max_rowsize != -1 else [max_rowsize, max_rowsize]
+            self.max_rowsize = [max_rowsize * self.batch_size] * 2
         else:
             self.max_rowsize = [max_rowsize[0] * self.batch_size, max_rowsize[1] * self.batch_size]
 
@@ -3188,8 +3176,7 @@ class _MPWorker(multiprocessing.Process):
         shared_memory = get_enable_shared_mem()
         self.pipe = Pipe(warning_ctl, shared_memory=shared_memory, max_rowsize=max_rowsize)
         self.check_interval = get_multiprocessing_timeout_interval()
-        super().__init__(target=worker_target(operations, worker_id), name="MapWorker" + str(worker_id),
-                         args=(self.pipe,), daemon=True)
+        super().__init__(target=worker_target(operations, worker_id), args=(self.pipe,), daemon=True)
 
     def execute(self, idx, *args):
         """Acquiring data from a worker in an infinite loop"""
@@ -3216,14 +3203,6 @@ class _MPWorker(multiprocessing.Process):
                     logger.warning("Please `pip install py-spy` to get the stacks of the stuck process.")
             try:
                 res = self.pipe.master_receive()
-                # Because there is no need to copy when creating Tensors in the C++layer, it reduces the time
-                # from np.ndarray to C++Tensor creation. However, when using shared memory in multiple processes,
-                # the address of the shared memory will always be passed to subsequent nodes in the dataset pipeline,
-                # and the shared memory will also be written by the current node, causing dirty data to be accessed
-                # by subsequent nodes in the pipeline. So make a memory copy here to solve the problem of
-                # shared memory being contaminated.
-                if get_enable_shared_mem():
-                    res = copy.deepcopy(res)
             except queue.Empty:
                 continue
             if res is None:
@@ -3584,16 +3563,16 @@ class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
         if platform.system().lower() != 'windows':
             self.eof = multiprocessing.Event()
             self.cleaning_process = multiprocessing.Process(target=self._clean_process,
-                                                            name="MapCleanProcess",
                                                             args=(self.ppid, self.workers, self.eof),
+                                                            name="OrphanCleaner",
                                                             daemon=True)
             self.cleaning_process.start()
 
             if get_enable_watchdog():
                 self.eot = threading.Event()
                 self.watch_dog = threading.Thread(target=self._watch_dog,
-                                                  name="MapWatchDog",
                                                   args=(self.eot, self.workers + [self.cleaning_process]),
+                                                  name="WatchDog",
                                                   daemon=True)
                 self.watch_dog.start()
 
@@ -3672,9 +3651,8 @@ class MapDataset(UnionBaseDataset):
         callbacks (DSCallback, list[DSCallback], optional): List of Dataset callbacks to be called. Default: ``None``.
         max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
             allocation to copy data between processes, the total occupied shared memory will increase as
-            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set to -1,
-            shared memory will be dynamically allocated with the actual size of data. This is only used if
-            ``python_multiprocessing`` is set to True. If it is an int value, it represents ``input_columns`` and
+            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
+            used if python_multiprocessing is set to True. If it is an int value, it represents ``input_columns`` and
             ``output_columns`` use this value as the unit to create shared memory. If it is a list, the first element
             represents the ``input_columns`` use this value as the unit to create shared memory, and the second element
             represents ``output_columns`` use this value as the unit to create shared memory. Default: 16.
@@ -4325,7 +4303,6 @@ class TransferDataset(Dataset):
         total_batch = 0
         if hasattr(self.children[0], "__total_batch__"):
             total_batch = self.children[0].__total_batch__
-            check_total_batch(total_batch)
         return cde.DataQueueNode(children[0], self.queue_name, self.device_type, self.device_id, self._send_epoch_end,
                                  total_batch, self._create_data_info_queue)
 

mindspore/dataset/engine/datasets_user_defined.py

@@ -19,7 +19,6 @@ After declaring the dataset object, you can further apply dataset operations
 (e.g. filter, skip, concat, map, batch) on it.
 """
 import builtins
-import copy
 import errno
 import math
 import os
@@ -116,28 +115,33 @@ def _cpp_sampler_fn_mp(sample_ids, sample_fn):
     return sample_fn.process(sample_ids)
 
 
-def _fill_worker_indices(workers, indices, idx_cursor, worker_to_quit):
+def _fill_worker_indices(workers, indices, idx):
     """
-    Worker index queue filler, fill worker index queue in round robin order or QUIT flag.
+    Worker index queue filler, fill worker index queue in round robin order.
     """
     num_worker = len(workers)
-    if idx_cursor < len(indices):
-        while idx_cursor < len(indices):
+    while idx < len(indices):
+        try:
+            workers[idx % num_worker].put(indices[idx])
+            idx += 1
+        except queue.Full:
+            break
+    return idx
+
+
+def _fill_worker_quit_flag(workers, worker_to_quit):
+    """
+    Worker index queue filler, fill worker index queue with QUIT flag.
+    """
+    num_worker = len(workers)
+    for i in range(num_worker):
+        # just put only one QUIT flag to the sub-thread / sub-process
+        if str(i) not in worker_to_quit:
             try:
-                workers[idx_cursor % num_worker].put(indices[idx_cursor])
-                idx_cursor += 1
+                workers[i].put("QUIT")
+                worker_to_quit[str(i)] = "QUIT"
             except queue.Full:
-                break
-    else:
-        for i in range(num_worker):
-            # just put only one QUIT flag to the sub-thread / sub-process
-            if str(i) not in worker_to_quit:
-                try:
-                    workers[i].put("QUIT")
-                    worker_to_quit[str(i)] = "QUIT"
-                except queue.Full:
-                    continue
-    return idx_cursor, worker_to_quit
+                continue
 
 
 def _convert_row(row):
@@ -217,11 +221,10 @@ class SamplerFn:
             # res_queue is used shared memory, so it' size is max_rowsize which is defined by user.
             _check_shm_usage(num_worker, queue_size, 0, max_rowsize)
         self.count = multiprocessing.Value('i', 0)
-        for worker_id in range(num_worker):
+        for _ in range(num_worker):
             if multi_process is True:
                 try:
-                    worker = _GeneratorWorkerMp(dataset, self.eof, max_rowsize, queue_size, self.ppid, self.count,
-                                                worker_id)
+                    worker = _GeneratorWorkerMp(dataset, self.eof, max_rowsize, queue_size, self.ppid, self.count)
                     worker.daemon = True
                     # When multi processes fork a subprocess, the lock of the main process is copied to the subprocess,
                     # which may cause deadlock. Therefore, the subprocess startup is performed in the initialization
@@ -240,18 +243,11 @@ class SamplerFn:
                 self.pids.append(worker.pid)
                 self.need_join = True
             else:
-                worker = _GeneratorWorkerMt(dataset, self.eof, worker_id)
+                worker = _GeneratorWorkerMt(dataset, self.eof)
                 worker.daemon = True
             self.workers.append(worker)
         self._launch_cleanup_worker(multi_process=multi_process)
 
-    def _interval_log(self, i, start_time, wait_count):
-        cost_time = int(time.time()) - start_time
-        if cost_time / self.check_interval >= wait_count:
-            wait_count += 1
-            self._log_stuck_warning(self.workers[i % self.num_worker], cost_time)
-        return wait_count
-
     def process(self, indices):
         """
         The main process, start the child process or child thread, and fill the index queue.
@@ -262,21 +258,19 @@ class SamplerFn:
             if not w.queue_empty():
                 # in failover reset scenario the QUIT flag should be pop first
                 while w.idx_queue.qsize() > 0:
-                    try:
-                        result = w.idx_queue.get(timeout=1)
-                        if result != "QUIT":
-                            raise Exception("The queue of the subprocess is not empty.")
-                    except queue.Empty:
-                        continue
+                    result = w.idx_queue.get()
+                    if result != "QUIT":
+                        raise Exception("The queue of the subprocess is not empty.")
             # Start all workers
             if not w.is_alive():
                 w.start()
 
         # Fill initial index queues
         idx_cursor = 0
+        idx_cursor = _fill_worker_indices(self.workers, indices, idx_cursor)
+
         # worker to quit
         worker_to_quit = {}
-        idx_cursor, worker_to_quit = _fill_worker_indices(self.workers, indices, idx_cursor, worker_to_quit)
 
         # Fetch results
         for i in range(len(indices)):
@@ -298,16 +292,11 @@ class SamplerFn:
                         self._stop_subprocess()
                         return
                     time.sleep(0.1)
-                    wait_count = self._interval_log(i, start_time, wait_count)
+                    cost_time = int(time.time()) - start_time
+                    if cost_time / self.check_interval >= wait_count:
+                        wait_count += 1
+                        self._log_stuck_warning(self.workers[i % self.num_worker], cost_time)
                 result = self.workers[i % self.num_worker].get()
-                # Because there is no need to copy when creating Tensors in the C++layer, it reduces the time
-                # from np.ndarray to C++Tensor creation. However, when using shared memory in multiple processes,
-                # the address of the shared memory will always be passed to subsequent nodes in the dataset pipeline,
-                # and the shared memory will also be written by the current node, causing dirty data to be accessed
-                # by subsequent nodes in the pipeline. So make a memory copy here to solve the problem of
-                # shared memory being contaminated.
-                if self.multi_process is True and get_enable_shared_mem():
-                    result = copy.deepcopy(result)
                 if isinstance(result, ExceptionHandler):
                     result.reraise()
             except queue.Empty:
@@ -319,9 +308,11 @@ class SamplerFn:
             if self.eof.is_set():
                 self._stop_subprocess()
                 return
-
-            idx_cursor, worker_to_quit = _fill_worker_indices(self.workers, indices, idx_cursor, worker_to_quit)
-
+            if idx_cursor < len(indices):
+                idx_cursor = _fill_worker_indices(self.workers, indices, idx_cursor)
+            else:
+                # send QUIT flag to workers
+                _fill_worker_quit_flag(self.workers, worker_to_quit)
             yield _convert_row(result)
 
     def _log_stuck_warning(self, worker, waiting_time):
@@ -370,7 +361,6 @@ class SamplerFn:
         if multi_process is True and platform.system().lower() != 'windows':
             _clean_worker_func = _PythonMultiprocessing._clean_process  # pylint: disable=W0212
             self.cleaning_process = multiprocessing.Process(target=_clean_worker_func,
-                                                            name="GeneratorCleanProcess",
                                                             args=(self.ppid, self.workers, self.eof))
             self.cleaning_process.daemon = True
             self.cleaning_process.start()
@@ -378,7 +368,6 @@ class SamplerFn:
             if get_enable_watchdog():
                 self.eot = threading.Event()
                 self.watch_dog = threading.Thread(target=_PythonMultiprocessing._watch_dog,  # pylint: disable=W0212
-                                                  name="GeneratorWatchDog",
                                                   args=(self.eot, self.workers + [self.cleaning_process]))
                 self.watch_dog.daemon = True
                 self.watch_dog.start()
@@ -393,7 +382,7 @@ class SamplerFn:
     def _stop_subprocess(self):
         """Only the main process can call join."""
         if self.need_join is True and self.ppid == os.getpid():
-            if hasattr(self, 'eof') and self.eof is not None:
+            if hasattr(self, 'eof') and self.eof is not None and not self.eof.is_set():
                 self.eof.set()
             # close the watch dog first
             self._abort_watchdog()
@@ -405,12 +394,10 @@ class SamplerFn:
                         del w.res_queue
                         del w.idx_queue
 
-                        # let the quit event notify the worker process to exit
-                        w.join(timeout=5)
-                        if w.is_alive():
-                            # if the worker process did not exit, it may hang, try to terminate it
-                            w.terminate()
-                            w.close()
+                        # close all the subprocess workers
+                        w.terminate()
+                        w.join()
+                        w.close()
                     except Exception:  # pylint: disable=W0703
                         # Block all errors when join
                         continue
@@ -442,15 +429,10 @@ class SamplerFn:
             self.workers = None
 
     def _abort_watchdog(self):
-        """Let watchdog quit."""
         if hasattr(self, 'eot') and self.eot is not None and not self.eot.is_set():
             self.eot.set()
         if hasattr(self, 'cleaning_process') and self.cleaning_process is not None:
-            # let the quit event notify the cleaning process to exit
-            self.cleaning_process.join(timeout=5)
-            if self.cleaning_process.is_alive():
-                # if the cleaning process did not exit, it may hang, try to terminate it
-                _PythonMultiprocessing._terminate_processes([self.cleaning_process])  # pylint: disable=W0212
+            _PythonMultiprocessing._terminate_processes([self.cleaning_process])  # pylint: disable=W0212
             del self.cleaning_process
         if hasattr(self, 'count'):
             del self.count
@@ -505,7 +487,7 @@ def _generator_worker_loop(dataset, idx_queue, result_queue, eof, is_multiproces
     if is_multiprocessing:
         result_queue.cancel_join_thread()  # Ensure that the process does not hung when exiting
         signal.signal(signal.SIGTERM, partial(_subprocess_handle, eof))
-    while not eof.is_set():
+    while True:
         _ignore_sigint(is_multiprocessing=is_multiprocessing)
 
         # Fetch index, block
@@ -540,7 +522,7 @@ def _generator_worker_loop(dataset, idx_queue, result_queue, eof, is_multiproces
         except Exception:  # pylint: disable=broad-except
             result = ExceptionHandler(where="in GeneratorDataset worker process")
         # Send data, block
-        while not eof.is_set():
+        while True:
             try:
                 result_queue.put(result, timeout=5)
             except queue.Full:
@@ -559,11 +541,11 @@ class _GeneratorWorkerMt(threading.Thread):
     Worker process for multi-thread Generator.
     """
 
-    def __init__(self, dataset, eof, worker_id):
+    def __init__(self, dataset, eof):
         self.idx_queue = queue.Queue(16)
         self.res_queue = queue.Queue(16)
         super().__init__(target=_generator_worker_loop, args=(dataset, self.idx_queue, self.res_queue, eof, False),
-                         name="GeneratorWorkerThread" + str(worker_id))
+                         name="GeneratorWorkerThread")
 
     def put(self, item):
         """
@@ -592,7 +574,7 @@ class _GeneratorWorkerMp(multiprocessing.Process):
     Worker process for multiprocess Generator.
     """
 
-    def __init__(self, dataset, eof, max_rowsize, queue_size, ppid, count, worker_id):
+    def __init__(self, dataset, eof, max_rowsize, queue_size, ppid, count):
         self.idx_queue = multiprocessing.Queue(queue_size)
         if get_enable_shared_mem():
             self.res_queue = _SharedQueue(queue_size, count, max_rowsize=max_rowsize)
@@ -600,7 +582,7 @@ class _GeneratorWorkerMp(multiprocessing.Process):
             self.res_queue = multiprocessing.Queue(queue_size)
         self.idx_queue.cancel_join_thread()  # Ensure that the process does not hung when exiting
         super().__init__(target=_generator_worker_loop, args=(dataset, self.idx_queue, self.res_queue, eof, True, ppid),
-                         name="GeneratorWorkerProcess" + str(worker_id))
+                         name="GeneratorWorkerProcess")
 
     def put(self, item):
         """
@@ -673,9 +655,8 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
             option could be beneficial if the Python operation is computational heavy. Default: ``True``.
         max_rowsize(int, optional): Maximum size of row in MB that is used for shared memory
             allocation to copy data between processes, the total occupied shared memory will increase as
-            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set to -1,
-            shared memory will be dynamically allocated with the actual size of data. This is only used if
-            ``python_multiprocessing`` is set to True. Default: 16.
+            ``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. This is only
+            used if python_multiprocessing is set to True. Default: 16.
 
     Raises:
         RuntimeError: If source raises an exception during execution.

mindspore/dataset/engine/datasets_vision.py

@@ -2799,8 +2799,8 @@ class ManifestDataset(MappableDataset, VisionBaseDataset):
 
     Manifest file contains a list of files included in a dataset, including basic file info such as File name and File
     ID, along with extended file metadata. Manifest is a data format file supported by Huawei Modelarts. For details,
-    see `Specifications for Importing the Manifest File <https://support.huaweicloud.com/intl/en-us/dataprepare-modelarts/
-    dataprepare-modelarts-0015.html>`_ .
+    see `Specifications for Importing the Manifest File <https://support.huaweicloud.com/engineers-modelarts/
+    modelarts_23_0009.html>`_ .
 
     .. code-block::