mindspore 2.4.10__cp310-none-any.whl → 2.5.0__cp310-none-any.whl

mindspore/dataset/__init__.py

@@ -12,25 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-This module provides APIs to load and process various common datasets such as MNIST,
-CIFAR-10, CIFAR-100, VOC, COCO, ImageNet, CelebA, CLUE, etc. It also supports datasets
-in standard format, including MindRecord, TFRecord, Manifest, etc. Users can also define
-their own datasets with this module.
-
-Besides, this module provides APIs to sample data while loading.
-
-We can enable cache in most of the dataset with its key arguments 'cache'. Please notice that cache is not supported
-on Windows platform yet. Do not use it while loading and processing data on Windows. More introductions and limitations
-can refer `Single-Node Tensor Cache <https://www.mindspore.cn/docs/en/master/model_train/dataset/cache.html>`_ .
-
-Common imported modules in corresponding API examples are as follows:
-
-.. code-block::
-
-    import mindspore.dataset as ds
-    import mindspore.dataset.transforms as transforms
-    import mindspore.dataset.vision as vision
-
+At the heart of MindSpore data loading utility is the `mindspore.dataset` module.
+It is a `dataset engine <https://www.mindspore.cn/docs/en/master/design/data_engine.html>`_ based on pipline design.
+
+This module provides the following data loading methods to help users load datasets into MindSpore.
+
+- User defined dataset loading: allows users to define `Random-accessible(Map-style) datasets
+  <https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html#random-accessible-dataset>`_ or
+  `Iterable-style dataset <https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html#iterable-dataset>`_
+  to customize data reading and processing logic.
+- Standard format dataset loading: support loading dataset files in standard data formats, including
+  `MindRecord <https://www.mindspore.cn/docs/en/master/model_train/dataset/record.html>`_,
+  `TFRecord <https://tensorflow.google.cn/tutorials/load_data/tfrecord.md?hl=en>`_ .
+- Open source dataset loading: supports reading `open source datasets <#open-source>`_ ,
+  such as MNIST, CIFAR-10, CLUE, LJSpeech, etc.
+
+In addition, this module also provides data sampler, transformations, batching, as well as basic configurations
+such as random seed, parallelism setting and other features, to be used in conjunction with the dataset loading.
+
+- Data Sampler: Provides various common `sampler <#sampler-1>`_, such as RandomSampler, DistributedSampler, etc.
+- Data Transformations: Provides multiple `dataset operations <https://www.mindspore.cn/docs/en/master/api_python/
+  dataset/mindspore.dataset.GeneratorDataset.html#pre-processing-operation>`_ to perform data augmentation, batching.
+- Basic Configuration: Provides `pipeline configuration <#config>`_ for random seed setting, parallelism setting,
+  data recovery mode, etc.
 
 Descriptions of common dataset terms are as follows:
 

mindspore/dataset/callback/ds_callback.py

@@ -189,7 +189,8 @@ class WaitedDSCallback(Callback, DSCallback):
         >>> import mindspore.nn as nn
         >>> from mindspore.dataset import WaitedDSCallback
         >>>
-        >>> ms.set_context(mode=ms.GRAPH_MODE, device_target="CPU")
+        >>> ms.set_context(mode=ms.GRAPH_MODE)
+        >>> ms.set_device(device_target="CPU")
         >>>
         >>> # custom callback class for data synchronization in data pipeline
         >>> class MyWaitedCallback(WaitedDSCallback):

mindspore/dataset/core/config.py

@@ -30,7 +30,7 @@ import random
 import numpy
 import mindspore._c_dataengine as cde
 from mindspore import log as logger
-from mindspore.dataset.core.validator_helpers import replace_none, type_check
+from mindspore.dataset.core.validator_helpers import replace_none, type_check, check_valid_str
 from mindspore.dataset.debug import DebugHook, PrintMetaDataHook
 from mindspore.dataset.core.validator_helpers import check_independent_mode
 
@@ -51,7 +51,9 @@ __all__ = ['set_sending_batches', 'load', '_init_device_info',
            'set_fast_recovery', 'get_fast_recovery',
            'set_debug_mode', 'get_debug_mode',
            'set_error_samples_mode', 'get_error_samples_mode', 'ErrorSamplesMode',
-           'set_multiprocessing_timeout_interval', 'get_multiprocessing_timeout_interval']
+           'set_multiprocessing_timeout_interval', 'get_multiprocessing_timeout_interval',
+           'set_iterator_mode', 'get_iterator_mode',
+           'set_multiprocessing_start_method', 'get_multiprocessing_start_method']
 
 INT32_MAX = 2147483647
 UINT32_MAX = 4294967295
@@ -510,7 +512,7 @@ def set_enable_autotune(enable, filepath_prefix=None):
         RuntimeError: If `filepath_prefix` does not have write permission.
 
     Note:
-        - Saved parameter profiles can be loaded via the `mindspore.dataset.deserialize` interface to
+        - Saved parameter profiles can be loaded via the :func:`mindspore.dataset.deserialize` interface to
           directly obtain a data processing pipeline object configured with optimal parameters.
         - The parameter tuning process can be viewed by turning on INFO level logging.
 
@@ -1093,3 +1095,82 @@ def get_error_samples_mode():
         >>> error_samples_mode = ds.config.get_error_samples_mode()
     """
     return _CDE_TO_PYTHON_ERROR_SAMPLES_MODE.get(_config.get_error_samples_mode())
+
+
+def set_iterator_mode(do_copy=True, parallel_convert=False):
+    """
+    Select dataset iterator optimization strategy.
+
+    Args:
+        do_copy (bool): Whether dataset iterator creates a Tensor from numpy.ndarray without copy. Default: "True".
+        parallel_convert (bool): Whether dataset iterator starts a thread to organize Tensors to output.
+            Default: "False".
+
+    Raises:
+        TypeError: If `do_copy` is not a boolean data type.
+        TypeError: If `parallel_convert` is not a boolean data type.
+
+    Examples:
+        >>> import mindspore.dataset as ds
+        >>> ds.config.set_iterator_mode(do_copy=False, parallel_convert=False)
+    """
+    type_check(do_copy, (bool,), "do_copy")
+    type_check(parallel_convert, (bool,), "parallel_convert")
+    _config.set_iterator_mode(do_copy, parallel_convert)
+
+
+def get_iterator_mode():
+    """
+    Get dataset iterator mode indicate iterator optimization strategy.
+    If `set_iterator_mode` is never called before, `do_copy` default to "True", `parallel_convert` default to "False".
+
+    Returns:
+        dict, iterator mode dictionary contains the value of `do_copy` and `parallel_convert`.
+
+    Examples:
+        >>> import mindspore.dataset as ds
+        >>> do_copy = ds.config.get_iterator_mode()['do_copy']
+        >>> do_copy = ds.config.get_iterator_mode()['parallel_convert']
+    """
+    return _config.get_iterator_mode()
+
+
+def set_multiprocessing_start_method(start_method='fork'):
+    """
+    Set a global configuration to indicate how to start a subprocess for data preprocessing.
+    This setting will affect the way of starting subprocess in GeneratorDataset, map and batch operations.
+
+    Args:
+        start_method (str, optional): multiprocessing start method in the data preprocessing phase.
+            Default: ``'fork'`` . Optional values: ['fork', 'spawn'].
+
+    Raises:
+        TypeError: If `start_method` is not of type str.
+        ValueError: If `start_method` is neither ``fork`` nor ``spawn`` .
+
+    Examples:
+        >>> # Set a new global configuration value for multiprocessing start method.
+        >>> import mindspore.dataset as ds
+        >>> ds.config.set_multiprocessing_start_method("fork")
+    """
+    type_check(start_method, (str,), "start_method")
+    check_valid_str(start_method, ["fork", "spawn"], "start_method")
+    _config.set_multiprocessing_start_method(start_method)
+
+
+def get_multiprocessing_start_method():
+    """
+    Get the global configuration of multiprocessing start method.
+    If `set_multiprocessing_start_method` is never called before, the default start method is ``'fork'`` .
+
+    Returns:
+        str, multiprocessing start method.
+
+    Examples:
+        >>> # Get the global configuration of multiprocessing start method when main process gets data
+        >>> # from subprocesses.
+        >>> # default ``fork`` will be returned.
+        >>> import mindspore.dataset as ds
+        >>> multiprocessing_start_method = ds.config.get_multiprocessing_start_method()
+    """
+    return _config.get_multiprocessing_start_method()

mindspore/dataset/engine/cache_admin.py

@@ -25,14 +25,14 @@ import mindspore
 
 def main():
     """Entry point for cache service"""
-    cache_admin_dir = os.path.join(os.path.dirname(mindspore.__file__), "bin")
+    cache_admin_dir = os.path.join(os.path.dirname(mindspore.__file__), "utils/bin")
     os.chdir(cache_admin_dir)
-    cache_admin = os.path.join(cache_admin_dir, "cache_admin")
+    cache_admin = os.path.join(cache_admin_dir, "dataset-cache")
 
     if not os.path.exists(cache_admin):
         raise RuntimeError("Dataset cache is not supported on your mindspore version.")
 
-    cache_server = os.path.join(cache_admin_dir, "cache_server")
+    cache_server = os.path.join(cache_admin_dir, "dataset-cache-server")
     os.chmod(cache_admin, stat.S_IRWXU)
     os.chmod(cache_server, stat.S_IRWXU)
 

mindspore/dataset/engine/cache_client.py

@@ -44,9 +44,10 @@ class DatasetCache:
         >>> import subprocess
         >>> import mindspore.dataset as ds
         >>>
-        >>> # Create a cache instance with command line `cache_admin --start` and create a session with `cache_admin -g`
-        >>> # After creating cache with a valid session, get session id with command `cache_admin --list_sessions`
-        >>> command = "cache_admin --list_sessions | tail -1 | awk -F ' ' '{{print $1;}}'"
+        >>> # Create a cache instance with command line `dataset-cache --start`
+        >>> # Create a session with `dataset-cache -g`
+        >>> # After creating cache with a valid session, get session id with command `dataset-cache --list_sessions`
+        >>> command = "dataset-cache --list_sessions | tail -1 | awk -F ' ' '{{print $1;}}'"
         >>> session_id = subprocess.getoutput(command).split('\n')[-1]
         >>> some_cache = ds.DatasetCache(session_id=int(session_id), size=0)
         >>>
@@ -93,7 +94,7 @@ class DatasetCache:
             >>> import mindspore.dataset as ds
             >>>
             >>> # In example above, we created cache with a valid session id
-            >>> command = "cache_admin --list_sessions | tail -1 | awk -F ' ' '{{print $1;}}'"
+            >>> command = "dataset-cache --list_sessions | tail -1 | awk -F ' ' '{{print $1;}}'"
             >>> id = subprocess.getoutput(command).split('\n')[-1]
             >>> some_cache = ds.DatasetCache(session_id=int(id), size=0)
             >>>