mindspore 2.4.10__cp39-cp39-win_amd64.whl → 2.6.0rc1__cp39-cp39-win_amd64.whl

mindspore/dataset/__init__.py

@@ -12,25 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-This module provides APIs to load and process various common datasets such as MNIST,
-CIFAR-10, CIFAR-100, VOC, COCO, ImageNet, CelebA, CLUE, etc. It also supports datasets
-in standard format, including MindRecord, TFRecord, Manifest, etc. Users can also define
-their own datasets with this module.
-
-Besides, this module provides APIs to sample data while loading.
-
-We can enable cache in most of the dataset with its key arguments 'cache'. Please notice that cache is not supported
-on Windows platform yet. Do not use it while loading and processing data on Windows. More introductions and limitations
-can refer `Single-Node Tensor Cache <https://www.mindspore.cn/docs/en/master/model_train/dataset/cache.html>`_ .
-
-Common imported modules in corresponding API examples are as follows:
-
-.. code-block::
-
-    import mindspore.dataset as ds
-    import mindspore.dataset.transforms as transforms
-    import mindspore.dataset.vision as vision
-
+At the heart of MindSpore data loading utility is the `mindspore.dataset` module.
+It is a `dataset engine <https://www.mindspore.cn/docs/en/master/design/data_engine.html>`_ based on pipline design.
+
+This module provides the following data loading methods to help users load datasets into MindSpore.
+
+- User defined dataset loading: allows users to define `Random-accessible(Map-style) dataset
+  <https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html#random-accessible-dataset>`_ or
+  `Iterable-style dataset <https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html#iterable-dataset>`_
+  to customize data reading and processing logic.
+- Standard format dataset loading: support loading dataset files in standard data formats, including
+  `MindRecord <https://www.mindspore.cn/tutorials/en/master/dataset/record.html>`_,
+  `TFRecord <https://tensorflow.google.cn/tutorials/load_data/tfrecord.md?hl=en>`_ .
+- Open source dataset loading: supports reading `open source datasets <#open-source>`_ ,
+  such as MNIST, CIFAR-10, CLUE, LJSpeech, etc.
+
+In addition, this module also provides data sampler, transformations, batching, as well as basic configurations
+such as random seed, parallelism setting and other features, to be used in conjunction with the dataset loading.
+
+- Data Sampler: Provides various common `sampler <#sampler-1>`_, such as RandomSampler, DistributedSampler, etc.
+- Data Transformations: Provides multiple `dataset operations <https://www.mindspore.cn/docs/en/master/api_python/
+  dataset/mindspore.dataset.GeneratorDataset.html#pre-processing-operation>`_ to perform data augmentation, batching.
+- Basic Configuration: Provides `pipeline configuration <#config>`_ for random seed setting, parallelism setting,
+  data recovery mode, etc.
 
 Descriptions of common dataset terms are as follows:
 
@@ -49,25 +53,49 @@ As shown in the above figure, the mindspore dataset module makes it easy for use
 pipelines and transform samples in the dataset in the most efficient (multi-process / multi-thread) manner.
 The specific steps are as follows:
 
-- Loading datasets: Users can easily load supported datasets using the `*Dataset` class, or load Python layer
-  customized datasets through `UDF Loader` + `GeneratorDataset` . At the same time, the loading class method can
-  accept a variety of parameters such as sampler, data slicing, and data shuffle;
-- Dataset operation: The user uses the dataset object method `.shuffle` / `.filter` / `.skip` / `.split` /
-  `.take` / ... to further shuffle, filter, skip, and obtain the maximum number of samples of datasets;
+- Loading datasets: Users can easily load supported datasets using the Dataset class
+  (`Standard-format Dataset
+  <https://www.mindspore.cn/docs/en/master/api_python/mindspore.dataset.loading.html#standard-format>`_,
+  `Vision Dataset <https://www.mindspore.cn/docs/en/master/api_python/mindspore.dataset.loading.html#vision>`_,
+  `NLP Dataset <https://www.mindspore.cn/docs/en/master/api_python/mindspore.dataset.loading.html#text>`_,
+  `Audio Dataset <https://www.mindspore.cn/docs/en/master/api_python/mindspore.dataset.loading.html#audio>`_,
+  or load Python layer customized datasets through
+  `User Defined Dataset
+  <https://www.mindspore.cn/docs/en/master/api_python/mindspore.dataset.loading.html#user-defined>`_,
+- Dataset operation: The user uses the dataset object method
+  `.shuffle <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/operation/
+  mindspore.dataset.Dataset.shuffle.html#mindspore.dataset.Dataset.shuffle>`_ /
+  `.filter <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/operation/
+  mindspore.dataset.Dataset.filter.html#mindspore.dataset.Dataset.filter>`_ /
+  `.skip <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/operation/
+  mindspore.dataset.Dataset.skip.html#mindspore.dataset.Dataset.skip>`_ /
+  `.split <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/operation/
+  mindspore.dataset.Dataset.split.html#mindspore.dataset.Dataset.split>`_ /
+  `.take <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/operation/
+  mindspore.dataset.Dataset.take.html#mindspore.dataset.Dataset.take>`_ /
+  … to further shuffle, filter, skip, and obtain the maximum number of samples of datasets.
 - Dataset sample transform operation: The user can add data transform operations
-  ( `vision transform <https://mindspore.cn/docs/en/master/api_python/mindspore.\
-  dataset.transforms.html#module-mindspore.dataset.vision>`_ ,
-  `NLP transform <https://mindspore.cn/docs/en/master/api_python/mindspore.\
-  dataset.transforms.html#module-mindspore.dataset.text>`_ ,
-  `audio transform <https://mindspore.cn/docs/en/master/api_python/mindspore.\
-  dataset.transforms.html#module-mindspore.dataset.audio>`_ ) to the map
-  operation to perform transformations. During data preprocessing, multiple map operations can be defined to
-  perform different transform operations to different fields. The data transform operation can also be a
-  user-defined transform `pyfunc` (Python function);
-- Batch: After the transformation of the samples, the user can use the batch operation to organize multiple samples
-  into batches, or use self-defined batch logic with the parameter `per_batch_map` applied;
-- Iterator: Finally, the user can use the dataset object method `create_dict_iterator` to create an
-  iterator, which can output the preprocessed data cyclically.
+  (`vision transform <https://www.mindspore.cn/docs/en/master/api_python/
+  mindspore.dataset.transforms.html#module-mindspore.dataset.vision>`_,
+  `nlp transform <https://www.mindspore.cn/docs/en/master/api_python/
+  mindspore.dataset.transforms.html#module-mindspore.dataset.text>`_,
+  `audio transform <https://www.mindspore.cn/docs/en/master/api_python/
+  mindspore.dataset.transforms.html#module-mindspore.dataset.audio>`_ )
+  to the `.map <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/operation/
+  mindspore.dataset.Dataset.map.html#mindspore.dataset.Dataset.map>`_ operation to perform transforms.
+  During data preprocessing, multiple map operations can be defined
+  to perform different transform operations to different fields.
+  The data transform operation can also be a user-defined Python function.
+- Batch: After the transforms of the samples, the user can use the
+  `.batch <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/batch/
+  mindspore.dataset.Dataset.batch.html#mindspore.dataset.Dataset.batch>`_ operation to organize multiple samples
+  into batches, or use self-defined batch logic with the parameter per_batch_map applied.
+- Iterator: Finally, user can use the method
+  `.create_dict_iterator <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/iterator/
+  mindspore.dataset.Dataset.create_dict_iterator.html#mindspore.dataset.Dataset.create_dict_iterator>`_ or
+  `.create_tuple_iterator <https://www.mindspore.cn/docs/en/master/api_python/dataset/dataset_method/iterator/
+  mindspore.dataset.Dataset.create_tuple_iterator.html#mindspore.dataset.Dataset.create_tuple_iterator>`_ to create
+  an iterator, which can output the preprocessed data cyclically.
 
 Quick start of Dataset Pipeline
 -------------------------------

mindspore/dataset/audio/__init__.py

@@ -26,12 +26,6 @@ Common imported modules in corresponding API examples are as follows:
     import mindspore.dataset.audio as audio
     from mindspore.dataset.audio import utils
 
-Alternative and equivalent imported audio module is as follows:
-
-.. code-block::
-
-    import mindspore.dataset.audio.transforms as audio
-
 Descriptions of common data processing terms are as follows:
 
 - TensorOperation, the base class of all data processing operations implemented in C++.
@@ -41,9 +35,9 @@ The data transform operation can be executed in the data processing pipeline or
 
 - Pipeline mode is generally used to process big datasets. Examples refer to
   `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/master/api_python/
-  mindspore.dataset.html#introduction-to-data-processing-pipeline>`_ .
+  mindspore.dataset.loading.html#introduction-to-data-processing-pipeline>`_ .
 - Eager mode is more like a function call to process data. Examples refer to
-  `Lightweight Data Processing <https://www.mindspore.cn/docs/en/master/model_train/dataset/eager.html>`_ .
+  `Lightweight Data Processing <https://www.mindspore.cn/tutorials/en/master/dataset/eager.html>`_ .
 """
 from __future__ import absolute_import
 

mindspore/dataset/audio/transforms.py

@@ -1423,10 +1423,6 @@ class FrequencyMasking(AudioTensorOperation):
     Tutorial Examples:
         - `Illustration of audio transforms
           <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/audio_gallery.html>`_
-
-    .. image:: frequency_masking_original.png
-
-    .. image:: frequency_masking.png
     """
 
     @check_masking
@@ -1500,10 +1496,10 @@ class GriffinLim(AudioTensorOperation):
     Args:
         n_fft (int, optional): Size of FFT. Default: ``400``.
         n_iter (int, optional): Number of iteration for phase recovery. Default: ``32``.
-        win_length (int, optional): Window size for GriffinLim. Default: ``None``, will be set to `n_fft` .
+        win_length (int, optional): Window size for Griffin-Lim. Default: ``None``, will be set to `n_fft` .
         hop_length (int, optional): Length of hop between STFT windows.
             Default: ``None``, will be set to `win_length // 2` .
-        window_type (WindowType, optional): Window type for GriffinLim, which can be ``WindowType.BARTLETT``,
+        window_type (WindowType, optional): Window type for Griffin-Lim, which can be ``WindowType.BARTLETT``,
             ``WindowType.BLACKMAN``, ``WindowType.HAMMING``, ``WindowType.HANN`` or ``WindowType.KAISER``.
             Default: ``WindowType.HANN``. Currently kaiser window is not supported on macOS.
         power (float, optional): Exponent for the magnitude spectrogram. Default: ``2.0``.
@@ -2075,7 +2071,7 @@ class Magphase(AudioTensorOperation):
         power (float): Power of the norm, which must be non-negative. Default: ``1.0``.
 
     Raises:
-        RuntimeError: If the shape of input audio waveform does not match (..., 2).
+        RuntimeError: If the shape of input audio waveform does not match :math:`(..., 2)`.
 
     Supported Platforms:
         ``CPU``
@@ -3336,10 +3332,6 @@ class TimeMasking(AudioTensorOperation):
     Tutorial Examples:
         - `Illustration of audio transforms
           <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/audio_gallery.html>`_
-
-    .. image:: time_masking_original.png
-
-    .. image:: time_masking.png
     """
 
     @check_masking
@@ -3405,12 +3397,6 @@ class TimeStretch(AudioTensorOperation):
     Tutorial Examples:
         - `Illustration of audio transforms
           <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/audio_gallery.html>`_
-
-    .. image:: time_stretch_rate1.5.png
-
-    .. image:: time_stretch_original.png
-
-    .. image:: time_stretch_rate0.8.png
     """
 
     @check_time_stretch

mindspore/dataset/callback/ds_callback.py

@@ -189,7 +189,8 @@ class WaitedDSCallback(Callback, DSCallback):
         >>> import mindspore.nn as nn
         >>> from mindspore.dataset import WaitedDSCallback
         >>>
-        >>> ms.set_context(mode=ms.GRAPH_MODE, device_target="CPU")
+        >>> ms.set_context(mode=ms.GRAPH_MODE)
+        >>> ms.set_device(device_target="CPU")
         >>>
         >>> # custom callback class for data synchronization in data pipeline
         >>> class MyWaitedCallback(WaitedDSCallback):

mindspore/dataset/core/config.py

@@ -30,7 +30,7 @@ import random
 import numpy
 import mindspore._c_dataengine as cde
 from mindspore import log as logger
-from mindspore.dataset.core.validator_helpers import replace_none, type_check
+from mindspore.dataset.core.validator_helpers import replace_none, type_check, check_valid_str
 from mindspore.dataset.debug import DebugHook, PrintMetaDataHook
 from mindspore.dataset.core.validator_helpers import check_independent_mode
 
@@ -51,7 +51,9 @@ __all__ = ['set_sending_batches', 'load', '_init_device_info',
            'set_fast_recovery', 'get_fast_recovery',
            'set_debug_mode', 'get_debug_mode',
            'set_error_samples_mode', 'get_error_samples_mode', 'ErrorSamplesMode',
-           'set_multiprocessing_timeout_interval', 'get_multiprocessing_timeout_interval']
+           'set_multiprocessing_timeout_interval', 'get_multiprocessing_timeout_interval',
+           'set_iterator_mode', 'get_iterator_mode',
+           'set_multiprocessing_start_method', 'get_multiprocessing_start_method']
 
 INT32_MAX = 2147483647
 UINT32_MAX = 4294967295
@@ -510,8 +512,8 @@ def set_enable_autotune(enable, filepath_prefix=None):
         RuntimeError: If `filepath_prefix` does not have write permission.
 
     Note:
-        - Saved parameter profiles can be loaded via the `mindspore.dataset.deserialize` interface to
-          directly obtain a data processing pipeline object configured with optimal parameters.
+        - Saved parameter profiles can be loaded via the :func:`mindspore.dataset.deserialize` interface to
+          return a data processing pipeline object of optimal parameters.
         - The parameter tuning process can be viewed by turning on INFO level logging.
 
     An example of the generated configuration file is as follows, the "remark" field describes whether or not data
@@ -755,8 +757,8 @@ def get_auto_offload():
 
 def set_enable_watchdog(enable):
     """
-    Set the default state of watchdog Python thread as enabled, the default state of watchdog Python thread is enabled.
-    Watchdog is a thread which cleans up hanging subprocesses.
+    Set the default state of watchdog Python thread as enabled. The watchdog Python thread
+    is responsible for cleaning up stuck or hanging subprocesses, and is enabled by default.
 
     Args:
         enable (bool): Whether to launch a watchdog Python thread.
@@ -1093,3 +1095,82 @@ def get_error_samples_mode():
         >>> error_samples_mode = ds.config.get_error_samples_mode()
     """
     return _CDE_TO_PYTHON_ERROR_SAMPLES_MODE.get(_config.get_error_samples_mode())
+
+
+def set_iterator_mode(do_copy=True, parallel_convert=False):
+    """
+    Select dataset iterator optimization strategy.
+
+    Args:
+        do_copy (bool): Whether dataset iterator creates a Tensor from numpy.ndarray without copy. Default: "True".
+        parallel_convert (bool): Whether dataset iterator starts a thread to organize Tensors to output.
+            Default: "False".
+
+    Raises:
+        TypeError: If `do_copy` is not a boolean data type.
+        TypeError: If `parallel_convert` is not a boolean data type.
+
+    Examples:
+        >>> import mindspore.dataset as ds
+        >>> ds.config.set_iterator_mode(do_copy=False, parallel_convert=False)
+    """
+    type_check(do_copy, (bool,), "do_copy")
+    type_check(parallel_convert, (bool,), "parallel_convert")
+    _config.set_iterator_mode(do_copy, parallel_convert)
+
+
+def get_iterator_mode():
+    """
+    Get dataset iterator mode indicate iterator optimization strategy.
+    If `set_iterator_mode` is never called before, `do_copy` default to "True", `parallel_convert` default to "False".
+
+    Returns:
+        dict, iterator mode dictionary contains the value of `do_copy` and `parallel_convert`.
+
+    Examples:
+        >>> import mindspore.dataset as ds
+        >>> do_copy = ds.config.get_iterator_mode()['do_copy']
+        >>> do_copy = ds.config.get_iterator_mode()['parallel_convert']
+    """
+    return _config.get_iterator_mode()
+
+
+def set_multiprocessing_start_method(start_method='fork'):
+    """
+    Set a global configuration to indicate how to start a subprocess for data preprocessing.
+    This setting will affect the way of starting subprocess in GeneratorDataset, map and batch operations.
+
+    Args:
+        start_method (str, optional): multiprocessing start method in the data preprocessing phase.
+            Default: ``'fork'`` . Optional values: ['fork', 'spawn'].
+
+    Raises:
+        TypeError: If `start_method` is not of type str.
+        ValueError: If `start_method` is neither ``fork`` nor ``spawn`` .
+
+    Examples:
+        >>> # Set a new global configuration value for multiprocessing start method.
+        >>> import mindspore.dataset as ds
+        >>> ds.config.set_multiprocessing_start_method("fork")
+    """
+    type_check(start_method, (str,), "start_method")
+    check_valid_str(start_method, ["fork", "spawn"], "start_method")
+    _config.set_multiprocessing_start_method(start_method)
+
+
+def get_multiprocessing_start_method():
+    """
+    Get the global configuration of multiprocessing start method.
+    If `set_multiprocessing_start_method` is never called before, the default start method is ``'fork'`` .
+
+    Returns:
+        str, multiprocessing start method.
+
+    Examples:
+        >>> # Get the global configuration of multiprocessing start method when main process gets data
+        >>> # from subprocesses.
+        >>> # default ``fork`` will be returned.
+        >>> import mindspore.dataset as ds
+        >>> multiprocessing_start_method = ds.config.get_multiprocessing_start_method()
+    """
+    return _config.get_multiprocessing_start_method()

mindspore/dataset/engine/cache_admin.py

@@ -25,14 +25,14 @@ import mindspore
 
 def main():
     """Entry point for cache service"""
-    cache_admin_dir = os.path.join(os.path.dirname(mindspore.__file__), "bin")
+    cache_admin_dir = os.path.join(os.path.dirname(mindspore.__file__), "utils/bin")
     os.chdir(cache_admin_dir)
-    cache_admin = os.path.join(cache_admin_dir, "cache_admin")
+    cache_admin = os.path.join(cache_admin_dir, "dataset-cache")
 
     if not os.path.exists(cache_admin):
         raise RuntimeError("Dataset cache is not supported on your mindspore version.")
 
-    cache_server = os.path.join(cache_admin_dir, "cache_server")
+    cache_server = os.path.join(cache_admin_dir, "dataset-cache-server")
     os.chmod(cache_admin, stat.S_IRWXU)
     os.chmod(cache_server, stat.S_IRWXU)
 

mindspore/dataset/engine/cache_client.py

@@ -27,7 +27,7 @@ class DatasetCache:
     A client to interface with tensor caching service.
 
     For details, please check
-    `Tutorial <https://www.mindspore.cn/docs/en/master/model_train/dataset/cache.html>`_ .
+    `Tutorial <https://www.mindspore.cn/tutorials/en/master/dataset/cache.html>`_ .
 
     Args:
         session_id (int): A user assigned session id for the current pipeline.
@@ -44,9 +44,10 @@ class DatasetCache:
         >>> import subprocess
         >>> import mindspore.dataset as ds
         >>>
-        >>> # Create a cache instance with command line `cache_admin --start` and create a session with `cache_admin -g`
-        >>> # After creating cache with a valid session, get session id with command `cache_admin --list_sessions`
-        >>> command = "cache_admin --list_sessions | tail -1 | awk -F ' ' '{{print $1;}}'"
+        >>> # Create a cache instance with command line `dataset-cache --start`
+        >>> # Create a session with `dataset-cache -g`
+        >>> # After creating cache with a valid session, get session id with command `dataset-cache --list_sessions`
+        >>> command = "dataset-cache --list_sessions | tail -1 | awk -F ' ' '{{print $1;}}'"
         >>> session_id = subprocess.getoutput(command).split('\n')[-1]
         >>> some_cache = ds.DatasetCache(session_id=int(session_id), size=0)
         >>>
@@ -93,7 +94,7 @@ class DatasetCache:
             >>> import mindspore.dataset as ds
             >>>
             >>> # In example above, we created cache with a valid session id
-            >>> command = "cache_admin --list_sessions | tail -1 | awk -F ' ' '{{print $1;}}'"
+            >>> command = "dataset-cache --list_sessions | tail -1 | awk -F ' ' '{{print $1;}}'"
             >>> id = subprocess.getoutput(command).split('\n')[-1]
             >>> some_cache = ds.DatasetCache(session_id=int(id), size=0)
             >>>