mindspore 1.10.0__cp39-cp39-win_amd64.whl → 2.0.0rc1__cp39-cp39-win_amd64.whl

mindspore/dataset/core/config.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2022 Huawei Technologies Co., Ltd
+# Copyright 2019-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,14 +23,15 @@ Common imported modules in corresponding API examples are as follows:
     import mindspore.dataset as ds
 """
 from __future__ import absolute_import
-
+from enum import IntEnum
 import os
 import platform
 import random
 import numpy
 import mindspore._c_dataengine as cde
 from mindspore import log as logger
-from mindspore.dataset.core.validator_helpers import replace_none
+from mindspore.dataset.core.validator_helpers import replace_none, type_check
+from mindspore.dataset.debug import DebugHook, PrintMetaDataHook
 
 __all__ = ['set_sending_batches', 'load', '_init_device_info',
            'set_seed', 'get_seed',
@@ -45,12 +46,16 @@ __all__ = ['set_sending_batches', 'load', '_init_device_info',
            'set_autotune_interval', 'get_autotune_interval',
            'set_auto_offload', 'get_auto_offload',
            'set_enable_watchdog', 'get_enable_watchdog',
+           'set_fast_recovery', 'get_fast_recovery',
+           'set_debug_mode', 'get_debug_mode',
+           'set_error_samples_mode', 'get_error_samples_mode', 'ErrorSamplesMode',
            'set_multiprocessing_timeout_interval', 'get_multiprocessing_timeout_interval']
 
 INT32_MAX = 2147483647
 UINT32_MAX = 4294967295
 
 _config = cde.GlobalContext.config_manager()
+_debug_context = {}
 
 
 def _init_device_info():
@@ -133,7 +138,7 @@ def get_seed():
     """
     Get random number seed. If the seed has been set, then will
     return the set value, otherwise it will return the default seed value
-    which equals to std::mt19937::default_seed.
+    which equals to `std::mt19937::default_seed <http://www.cplusplus.com/reference/random/mt19937/>`_ .
 
     Returns:
         int, random number seed.
@@ -151,11 +156,12 @@ def set_prefetch_size(size):
     Set the queue capacity of the thread in pipeline.
 
     Args:
-        size (int): The length of the cache queue.
+        size (int): The length of the cache queue. The `size` must be greater than 0, otherwise the queue capacity of
+            the thread is invalid.
 
     Raises:
         TypeError: If `size` is not of type int.
-        ValueError: If `size` <= 0 or `size` > INT32_MAX(2147483647).
+        ValueError: If `size` is not a positive number.
 
     Note:
         Since total memory used for prefetch can grow very large with high number of workers,
@@ -204,7 +210,7 @@ def set_num_parallel_workers(num):
 
     Examples:
         >>> # Set a new global configuration value for the number of parallel workers.
-        >>> # Now parallel dataset operators will run with 8 workers.
+        >>> # Now parallel dataset operations will run with 8 workers.
         >>> ds.config.set_num_parallel_workers(8)
     """
     if not isinstance(num, int) or isinstance(num, bool):
@@ -233,7 +239,8 @@ def get_num_parallel_workers():
 
 def set_numa_enable(numa_enable):
     """
-    Set the default state of numa enabled. If numa_enable is True, need to ensure numa library is installed.
+    Set the default state of numa enabled. If `numa_enable` is True, need to
+    ensure `numa library <http://rpmfind.net/linux/rpm2html/search.php?query=libnuma-devel>`_ is installed.
 
     Args:
         numa_enable (bool): Whether to use numa bind feature.
@@ -243,7 +250,7 @@ def set_numa_enable(numa_enable):
 
     Examples:
         >>> # Set a new global configuration value for the state of numa enabled.
-        >>> # Now parallel dataset operators will run with numa bind function
+        >>> # Now parallel dataset operations will run with numa bind function
         >>> ds.config.set_numa_enable(True)
     """
     if not isinstance(numa_enable, bool):
@@ -374,14 +381,15 @@ def get_auto_num_workers():
 
 def set_callback_timeout(timeout):
     """
-    Set the default timeout (in seconds) for DSWaitedCallback.
+    Set the default timeout (in seconds) for :class:`mindspore.dataset.WaitedDSCallback` .
 
     Args:
-        timeout (int): Timeout (in seconds) to be used to end the wait in DSWaitedCallback in case of a deadlock.
+        timeout (int): Timeout (in seconds) to be used to end the wait in :class:`mindspore.dataset.WaitedDSCallback`
+            in case of a deadlock. The `timeout` must be greater than 0.
 
     Raises:
         TypeError: If `timeout` is not type int.
-        ValueError: If `timeout` <= 0 or `timeout` > INT32_MAX(2147483647).
+        ValueError: If `timeout` is not a positive number.
 
     Examples:
         >>> # Set a new global configuration value for the timeout value.
@@ -396,10 +404,11 @@ def set_callback_timeout(timeout):
 
 def get_callback_timeout():
     """
-    Get the default timeout for WaitedDSCallback.
+    Get the default timeout for :class:`mindspore.dataset.WaitedDSCallback` .
 
     Returns:
-        int, Timeout (in seconds) to be used to end the wait in DSWaitedCallback in case of a deadlock.
+        int, Timeout (in seconds) to be used to end the wait in :class:`mindspore.dataset.WaitedDSCallback` in case of
+        a deadlock.
 
     Examples:
         >>> # Get the global configuration of callback timeout.
@@ -612,13 +621,13 @@ def get_enable_shared_mem():
 def set_enable_shared_mem(enable):
     """
     Set the default state of shared memory flag. If shared_mem_enable is True, will use shared memory queues
-    to pass data to processes that are created for operators that set python_multiprocessing=True.
+    to pass data to processes that are created for operations that set python_multiprocessing=True.
 
     Note:
         `set_enable_shared_mem` is not supported on Windows and MacOS platforms yet.
 
     Args:
-        enable (bool): Whether to use shared memory in operators when python_multiprocessing=True.
+        enable (bool): Whether to use shared memory in operations when python_multiprocessing=True.
 
     Raises:
         TypeError: If `enable` is not a boolean data type.
@@ -760,7 +769,7 @@ def get_multiprocessing_timeout_interval():
 
     Returns:
         int, interval (in seconds) for multiprocessing/multithreading timeout when main process/thread gets data from
-        subprocesses/child threads (default is 300s).
+        subprocesses/child threads. Default: 300s.
 
     Examples:
         >>> # Get the global configuration of multiprocessing/multithreading timeout when main process/thread gets data
@@ -799,3 +808,223 @@ def get_dynamic_shape():
         >>> is_dynamic_shape = ds.config.get_dynamic_shape()
     """
     return _config.get_dynamic_shape()
+
+
+def set_fast_recovery(fast_recovery):
+    """
+    Set whether dataset pipeline should recover in fast mode during failover
+    (In fast mode, random augmentations may not get same results as before the failure occurred).
+
+    Args:
+        fast_recovery (bool): Whether the dataset pipeline recovers in fast mode. System default: True.
+
+    Raises:
+        TypeError: If `fast_recovery` is not a boolean data type.
+
+    Examples:
+        >>> ds.config.set_fast_recovery(False)
+    """
+    if not isinstance(fast_recovery, bool):
+        raise TypeError("fast_recovery must be a boolean dtype.")
+    _config.set_fast_recovery(fast_recovery)
+
+
+def get_fast_recovery():
+    """
+    Get whether the fast recovery mode is enabled for the current dataset pipeline.
+
+    Returns:
+        bool, whether the dataset recovers fast in failover reset.
+
+    Examples:
+        >>> is_fast_recovery = ds.config.get_fast_recovery()
+    """
+    return _config.get_fast_recovery()
+
+
+def set_debug_mode(debug_mode_flag: bool, debug_hook_list: list = None):
+    """
+    Set the debug_mode flag of the dataset pipeline. When enabled, the dataset pipeline is run synchronously and
+    sequentially with a single thread.
+
+    Note:
+        - When debug_mode is enabled, if set_seed has not yet been issued, MindData will internally set the seed to 1
+          so that debug mode execution of the dataset pipeline can produce deterministic results.
+        - When debug_mode is enabled, many configuration settings are ignored, including the following noteworthy
+          settings:
+            - auto_offload (False is used.)
+            - enable_autotune (False is used.)
+            - error_samples_mode (ErrorSamplesMode.RETURN is used.)
+            - num_parallel_workers (Value 1 is used.)
+        - If both debug_mode is enabled and a dataset pipeline has Map operation with offload set, then offloading is
+          ignored.
+        - If both debug_mode is enabled and a dataset pipeline has Map operation or Batch operation with
+          python_multiprocessing=True, then Python multiprocessing is ignored.
+        - If both debug_mode is enabled and a dataset pipeline has GeneratorDataset with
+          python_multiprocessing=True (the default value), then Python multiprocessing is ignored.
+        - If both debug_mode is enabled and a dataset operation has cache set, then the cache is dropped.
+        - If both debug_mode and profiling are enabled, then dataset profiling is ignored.
+
+    Args:
+        debug_mode_flag (bool): Whether dataset pipeline debug mode is enabled, which forces the pipeline
+            to run synchronously and sequentially.
+        debug_hook_list (list[DebugHook]): a list of debug hook objects to be inserted before and after each
+            transform operation in map operation. Default: None, which means to use `[PrintMetaDataHook]`,
+            which prints shape/size/type of each input/output data of each transformation.
+
+    Raises:
+        TypeError: If `debug_mode_flag` is not a boolean data type.
+        TypeError: If `debug_hook_list` is not a list type.
+        TypeError: If any item in `debug_hook_list` is not DebugHook type.
+
+    Examples:
+        1. Enable dataset pipeline debug mode and use default debug hook.
+        >>> # Print shape and type of input/output data of each transform op in map operator.
+        >>> ds.config.set_debug_mode(True)
+
+        2. Enable dataset pipeline debug mode and use pre-defined debug hook provided by MindData.
+        >>> import mindspore.dataset.debug as debug
+        >>>
+        >>> ds.config.set_debug_mode(True, debug_hook_list=[debug.PrintDataHook()])
+
+        3. Enable dataset pipeline debug mode and use user-defined debug hook. It must define a
+        class inherited from DebugHook.
+        >>> import mindspore.dataset.debug as debug
+        >>>
+        >>> class CustomizedHook(debug.DebugHook):
+        ...     def __init__(self):
+        ...         super().__init__()
+        ...
+        ...     def compute(self, *args):
+        ...         # Add your debugging code here.
+        ...         return args
+        >>>
+        >>> ds.config.set_debug_mode(True, debug_hook_list=[CustomizedHook()])
+
+        4. Enable dataset pipeline debug mode and use user-defined debug hook and insert by users manually.
+        >>> ds.config.set_debug_mode(True)
+        >>> dataset = ds.ImageFolderDataset(dataset_dir="/path/to/image_folder_dataset_directory")
+        >>> # the debug hook is added after `Decode` operation.
+        >>> dataset = dataset.map([vision.Decode(), CustomizedHook(), vision.CenterCrop(100)])
+    """
+    if not isinstance(debug_mode_flag, bool):
+        raise TypeError("debug_mode_flag isn't of type boolean.")
+    if not debug_hook_list:
+        debug_hook_list = [PrintMetaDataHook()]
+    if not isinstance(debug_hook_list, list):
+        raise TypeError("debug_hook_list is not a list.")
+    for debug_func in debug_hook_list:
+        if not isinstance(debug_func, DebugHook):
+            raise TypeError("All items in debug_hook_list must be of type DebugHook.")
+    if debug_mode_flag:
+        logger.warning("Dataset pipeline debug mode is enabled. Performance will be impacted because the pipeline"
+                       " will be running in a single thread.")
+    if debug_hook_list:
+        _debug_context["debug_hook_list"] = debug_hook_list
+
+    _config.set_debug_mode(debug_mode_flag)
+
+
+def get_debug_mode():
+    """
+    Get the debug_mode flag of the dataset pipeline
+
+    Returns:
+        bool, whether dataset pipeline debug mode is enabled
+
+    Examples:
+        >>> debug_mode = ds.config.get_debug_mode()
+    """
+    return _config.get_debug_mode()
+
+
+def _get_debug_hook_list():
+    """
+    INTERNAL USE ONLY!
+    Get value of debug_hook_list.
+
+    Returns:
+        list, the debug hook objects to be inserted in map operation to debug inputs/outputs of each transform.
+    """
+    return _debug_context.get("debug_hook_list")
+
+
+class ErrorSamplesMode(IntEnum):
+    """
+    An enumeration for `error_samples_mode` .
+
+    Possible enumeration values are: ErrorSamplesMode.RETURN, ErrorSamplesMode.REPLACE, ErrorSamplesMode.SKIP.
+
+    - ErrorSamplesMode.RETURN: means erroneous sample results in error raised and returned.
+    - ErrorSamplesMode.REPLACE: means erroneous sample is replaced with an internally determined sample.
+    - ErrorSamplesMode.SKIP: means erroneous sample is skipped.
+    """
+
+    RETURN = 0
+    REPLACE = 1
+    SKIP = 2
+
+
+# Convert ErrorSamplesMode from Python enum format to CDE enum format
+_PYTHON_TO_CDE_ERROR_SAMPLES_MODE = {
+    ErrorSamplesMode.RETURN: cde.ErrorSamplesMode.DE_ERROR_SAMPLES_MODE_RETURN,
+    ErrorSamplesMode.REPLACE: cde.ErrorSamplesMode.DE_ERROR_SAMPLES_MODE_REPLACE,
+    ErrorSamplesMode.SKIP: cde.ErrorSamplesMode.DE_ERROR_SAMPLES_MODE_SKIP
+}
+
+# Convert ErrorSamplesMode from CDE int format to Python enum format
+_CDE_TO_PYTHON_ERROR_SAMPLES_MODE = {
+    0: ErrorSamplesMode.RETURN,
+    1: ErrorSamplesMode.REPLACE,
+    2: ErrorSamplesMode.SKIP
+}
+
+
+def set_error_samples_mode(error_samples_mode):
+    """
+    Set the method in which erroneous samples should be processed in a dataset pipeline.
+
+    Note:
+        - This error samples feature is only applicable to the Map operation in a dataset pipeline.
+        - For 'ErrorSamplesMode.REPLACE' mode, a cache of other samples will be used.
+        - If 'ErrorSamplesMode.SKIP' mode is used in a distributed setting, beware to manually ensure the
+          number of valid samples are the same for each shard (otherwise one may encounter hangs).
+          One technique is to manually concat a dataset of all valid samples plus a
+          take operation for the number of skipped erroneous samples.
+
+    Args:
+        error_samples_mode (ErrorSamplesMode): The method in which erroneous samples should be processed in a dataset
+            pipeline. It can be any of [ErrorSamplesMode.RETURN, ErrorSamplesMode.REPLACE, ErrorSamplesMode.SKIP].
+            System default: ErrorSamplesMode.RETURN.
+
+            - ErrorSamplesMode.RETURN: means erroneous sample results in error raised and returned.
+
+            - ErrorSamplesMode.REPLACE: means erroneous sample is replaced with a correct sample.
+
+            - ErrorSamplesMode.SKIP: means erroneous sample is skipped.
+
+    Raises:
+        TypeError: If `error_samples_mode` is not of type ErrorSamplesMode.
+
+    Examples:
+        >>> ds.config.set_error_samples_mode(ds.config.ErrorSamplesMode.SKIP)
+    """
+    type_check(error_samples_mode, (ErrorSamplesMode,), "error_samples_mode")
+    _config.set_error_samples_mode(_PYTHON_TO_CDE_ERROR_SAMPLES_MODE.get(error_samples_mode))
+
+
+def get_error_samples_mode():
+    """
+    Get the current configuration for strategy for processing erroneous samples in a dataset pipeline.
+
+    Returns:
+        ErrorSamplesMode, The method in which erroneous samples should be processed in a dataset pipeline.
+
+        - ErrorSamplesMode.RETURN: means erroneous sample results in error raised and returned.
+        - ErrorSamplesMode.REPLACE: means erroneous sample is replaced with an internally determined sample.
+        - ErrorSamplesMode.SKIP: means erroneous sample is skipped.
+
+    Examples:
+        >>> error_samples_mode = ds.config.get_error_samples_mode()
+    """
+    return _CDE_TO_PYTHON_ERROR_SAMPLES_MODE.get(_config.get_error_samples_mode())

mindspore/dataset/core/py_util_helpers.py

@@ -53,9 +53,10 @@ class ExceptionHandler:
 
     def reraise(self):
         """Reraise the caught exception in the main thread/process"""
-        # Error message like: "Caught ValueError in GeneratorDataset worker process. Original Traceback:".
-        err_msg = "Caught {} {}.\nOriginal {}".format(
-            self.except_type.__name__, self.where, self.except_msg)
+        # Find the last traceback which is more useful to user.
+        index = [i for i in range(len(self.except_msg)) if self.except_msg.startswith('Traceback', i)]
+        err_msg = "{}".format(self.except_msg[index[-1]:]).strip()
+
         if self.except_type == KeyError:
             # As KeyError will call its repr() function automatically, which makes stack info hard to read.
             err_msg = KeyErrorParse(err_msg)

mindspore/dataset/core/validator_helpers.py

@@ -746,9 +746,9 @@ def check_gnn_list_or_ndarray(param, param_name, data_type=int):
         elif data_type == str:
             data_type = np.str_
 
-        if param.dtype.type != data_type:
+        if param.dtype != data_type:
             raise TypeError("Each member in {0} should be of type {1}. Got {2}.".format(
-                param_name, data_type, param.dtype.type))
+                param_name, data_type, param.dtype))
 
 
 def check_tensor_op(param, param_name):
@@ -760,7 +760,7 @@ def check_tensor_op(param, param_name):
 def check_c_tensor_op(param, param_name):
     """check whether param is a tensor op or a callable Python function but not a py_transform"""
     if callable(param) and str(param).find("py_transform") >= 0:
-        raise TypeError("{0} is a py_transform op which is not allow to use.".format(param_name))
+        raise TypeError("{0} is a py_transform op which is not allowed to use.".format(param_name))
     if not isinstance(param, cde.TensorOp) and not callable(param) and not getattr(param, 'parse', None):
         raise TypeError("{0} is neither a c_transform op (TensorOperation) nor a callable pyfunc.".format(param_name))
 
@@ -781,19 +781,19 @@ def check_dataset_num_shards_shard_id(num_shards, shard_id):
 
 
 def deprecator_factory(version, old_module, new_module, substitute_name=None, substitute_module=None):
-    """Decorator factory function for deprecated operator to log deprecation warning message.
+    """Decorator factory function for deprecated operation to log deprecation warning message.
 
     Args:
-        version (str): Version that the operator is deprecated.
-        old_module (str): Old module for deprecated operator.
-        new_module (str): New module for deprecated operator.
-        substitute_name (str, optional): The substitute name for deprecated operator.
-        substitute_module (str, optional): The substitute module for deprecated operator.
+        version (str): Version that the operation is deprecated.
+        old_module (str): Old module for deprecated operation.
+        new_module (str): New module for deprecated operation.
+        substitute_name (str, optional): The substitute name for deprecated operation.
+        substitute_module (str, optional): The substitute module for deprecated operation.
     """
 
     def decorator(op):
         def wrapper(*args, **kwargs):
-            # Get operator class name for operator class which applies decorator to __init__()
+            # Get operation class name for operation class which applies decorator to __init__()
             name = str(op).split()[1].split(".")[0]
             # Build message
             message = f"'{name}' from " + f"{old_module}" + f" is deprecated from version " f"{version}" + \

mindspore/{parallel/nn/layers.py → dataset/debug/__init__.py}

@@ -1,4 +1,4 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,12 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ============================================================================
-"""
-The basic layer of the Transformer Networks. This is an experimental interface that is subject to
-change or deletion.
-"""
+# ==============================================================================
+"""Init file for dataset pipeline debug mode"""
 from __future__ import absolute_import
-from mindspore.nn.transformer.layers import FixedSparseAttention
 
-__all__ = ["FixedSparseAttention"]
+from mindspore.dataset.debug.debug_hook import DebugHook
+from mindspore.dataset.debug.pre_defined_hook import PrintMetaDataHook, PrintDataHook
+
+__all__ = ["DebugHook", "PrintMetaDataHook", "PrintDataHook"]

mindspore/dataset/debug/debug_hook.py

@@ -0,0 +1,65 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""
+This module defines the class for minddata pipeline debugger.
+class DebugHook is not exposed to users as an external API.
+"""
+
+from abc import ABC, abstractmethod
+from mindspore import log as logger
+
+
+class DebugHook(ABC):
+    """
+    The base class for Dataset Pipeline Python Debugger hook. All user defined hook behaviors
+    must inherit this base class.
+
+    To debug the input and output data of map operation in dataset pipeline, users can add
+    breakpoint to or single stepping in this class. They can also see the type and shape of
+    the data from the log being printed.
+
+    Args:
+        prev_op_name (str, optional): name of the operation before current debugging point.
+    """
+    def __init__(self, prev_op_name=None):
+        self.prev_op_name = prev_op_name
+
+    def __call__(self, *args):
+        # log op name
+        if self.prev_op_name:
+            log_message = "Debugging the output of the operation [{}].".format(self.prev_op_name)
+        else:
+            log_message = "Debugging the input of the first operation."
+        logger.info(log_message)
+
+        ######################## NOTE ########################
+        # Add a breakpoint to the following line to inspect
+        # input and output of each transform.
+        ######################################################
+        self.compute(args)
+        return args
+
+    @abstractmethod
+    def compute(self, *args):
+        """
+        Defines the debug behaviour to be performed. This method must be overridden by all subclasses.
+        """
+        raise RuntimeError("compute() is not overridden in subclass of class DebugHook.")
+
+    def set_previous_op_name(self, prev_op_name):
+        """
+        Set prev_op_name.
+        """
+        self.prev_op_name = prev_op_name

mindspore/dataset/debug/pre_defined_hook.py

@@ -0,0 +1,67 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""
+This module defines the subclass of DebugHook for minddata pipeline debugger.
+All these class are pre-defined for users for basic debugging purpose.
+"""
+
+import collections
+import numpy as np
+from PIL import Image
+from mindspore import log as logger
+from mindspore.dataset.debug.debug_hook import DebugHook
+
+
+class PrintMetaDataHook(DebugHook):
+    """
+    Debug hook used for MindData debug mode to print type and shape of data.
+    """
+    def __init__(self):
+        super().__init__()
+
+    def compute(self, *args):
+        for col_idx, col in enumerate(*args):
+            log_message = "Column {}. ".format(col_idx)
+            # log type
+            log_message += "The type is [{}].".format(type(col))
+
+            # log shape/size
+            if isinstance(col, np.ndarray):
+                log_message += " The shape is [{}].".format(col.shape)
+            elif isinstance(col, Image.Image):
+                log_message += " The shape is [{}].".format(col.size)
+            elif isinstance(col, collections.abc.Sized):
+                log_message += " The size is [{}].".format(len(col))
+            logger.info(log_message)
+        return args
+
+
+class PrintDataHook(DebugHook):
+    """
+    Debug hook used for MindData debug mode to print data.
+    """
+    def __init__(self):
+        super().__init__()
+
+    def compute(self, *args):
+        for col_idx, col in enumerate(*args):
+            log_message = "Column {}. ".format(col_idx)
+            if isinstance(col, Image.Image):
+                data = np.asarray(col)
+                log_message += "The data is [{}].".format(data)
+            else:
+                log_message += "The data is [{}].".format(col)
+            logger.info(log_message)
+        return args

mindspore/dataset/engine/__init__.py

@@ -50,6 +50,7 @@ __all__ = ["Caltech101Dataset",        # Vision
            "FashionMnistDataset",      # Vision
            "FlickrDataset",            # Vision
            "Flowers102Dataset",        # Vision
+           "Food101Dataset",           # Vision
            "ImageFolderDataset",       # Vision
            "KITTIDataset",             # Vision
            "KMnistDataset",            # Vision
@@ -62,10 +63,12 @@ __all__ = ["Caltech101Dataset",        # Vision
            "Places365Dataset",         # Vision
            "QMnistDataset",            # Vision
            "RandomDataset",            # Vision
+           "RenderedSST2Dataset",      # Vision
            "SBDataset",                # Vision
            "SBUDataset",               # Vision
            "SemeionDataset",           # Vision
            "STL10Dataset",             # Vision
+           "SUN397Dataset",            # Vision
            "SVHNDataset",              # Vision
            "USPSDataset",              # Vision
            "VOCDataset",               # Vision
@@ -83,6 +86,7 @@ __all__ = ["Caltech101Dataset",        # Vision
            "PennTreebankDataset",      # Text
            "SogouNewsDataset",         # Text
            "SQuADDataset",             # Text
+           "SST2Dataset",              # Text
            "TextFileDataset",          # Text
            "UDPOSDataset",             # Text
            "WikiTextDataset",          # Text
@@ -102,10 +106,10 @@ __all__ = ["Caltech101Dataset",        # Vision
            "GeneratorDataset",         # User Defined
            "NumpySlicesDataset",       # User Defined
            "PaddedDataset",            # User Defined
-           "GraphData",                # Graph Data
+           "GraphData",                # Graph
            "Graph",                    # Graph
-           "InMemoryGraphDataset",     # InMemoryGraphDataset
-           "ArgoverseDataset",         # ArgoverseDataset
+           "InMemoryGraphDataset",     # Graph
+           "ArgoverseDataset",         # Graph
            "DistributedSampler",       # Sampler
            "RandomSampler",            # Sampler
            "SequentialSampler",        # Sampler

mindspore/dataset/engine/cache_client.py

@@ -27,18 +27,18 @@ class DatasetCache:
     A client to interface with tensor caching service.
 
     For details, please check `Tutorial <https://www.mindspore.cn/
-    tutorials/experts/en/r1.10/dataset/cache.html>`_.
+    tutorials/experts/en/r2.0/dataset/cache.html>`_ .
 
     Args:
         session_id (int): A user assigned session id for the current pipeline.
-        size (int, optional): Size of the memory set aside for the row caching (default=0, which means unlimited,
-            note that it might bring in the risk of running out of memory on the machine).
-        spilling (bool, optional): Whether or not spilling to disk if out of memory (default=False).
-        hostname (str, optional): Host name (default=None, use default hostname '127.0.0.1').
-        port (int, optional): Port to connect to server (default=None, use default port 50052).
-        num_connections (int, optional): Number of tcp/ip connections (default=None, use default value 12).
-        prefetch_size (int, optional): The size of the cache queue between operations
-            (default=None, use default value 20).
+        size (int, optional): Size of the memory set aside for the row caching. Default: 0, which means unlimited,
+            note that it might bring in the risk of running out of memory on the machine.
+        spilling (bool, optional): Whether or not spilling to disk if out of memory. Default: False.
+        hostname (str, optional): Host name. Default: None, use default hostname '127.0.0.1'.
+        port (int, optional): Port to connect to server. Default: None, use default port 50052.
+        num_connections (int, optional): Number of tcp/ip connections. Default: None, use default value 12.
+        prefetch_size (int, optional): The size of the cache queue between operations.
+            Default: None, use default value 20.
 
     Examples:
             >>> import mindspore.dataset as ds