mindspore 2.0.0a0__cp37-none-any.whl → 2.0.0rc1__cp37-none-any.whl

mindspore/dataset/audio/utils.py

@@ -223,6 +223,16 @@ def create_dct(n_mfcc, n_mels, norm=NormMode.NONE):
     Returns:
         numpy.ndarray, the transformation matrix, to be right-multiplied to row-wise data of size (n_mels, n_mfcc).
 
+    Raises:
+        TypeError: If `n_mfcc` is not of type int.
+        ValueError: If `n_mfcc` is not positive.
+        TypeError: If `n_mels` is not of type int.
+        ValueError: If `n_mels` is not positive.
+        TypeError: If `n_mels` is not of type :class:`mindspore.dataset.audio.NormMode` .
+
+    Supported Platforms:
+        ``CPU``
+
     Examples:
         >>> from mindspore.dataset.audio import create_dct, NormMode
         >>>
@@ -257,15 +267,31 @@ def linear_fbanks(n_freqs, f_min, f_max, n_filter, sample_rate):
     Creates a linear triangular filterbank.
 
     Args:
-        n_freqs (int): Number of frequency.
-        f_min (float): Minimum of frequency in Hz.
-        f_max (float): Maximum of frequency in Hz.
+        n_freqs (int): Number of frequencies to highlight/apply.
+        f_min (float): Minimum frequency in Hz.
+        f_max (float): Maximum frequency in Hz.
         n_filter (int): Number of (linear) triangular filter.
-        sample_rate (int): Sample rate.
+        sample_rate (int): Sample rate of the waveform.
 
     Returns:
         numpy.ndarray, the linear triangular filterbank.
 
+    Raises:
+        TypeError: If `n_freqs` is not of type int.
+        ValueError: If `n_freqs` is negative.
+        TypeError: If `f_min` is not of type float.
+        ValueError: If `f_min` is negative.
+        TypeError: If `f_max` is not of type float.
+        ValueError: If `f_max` is negative.
+        ValueError: If `f_min` is larger than `f_max`.
+        TypeError: If `n_filter` is not of type int.
+        ValueError: If `n_filter` is not positive.
+        TypeError: If `sample_rate` is not of type int.
+        ValueError: If `sample_rate` is not positive.
+
+    Supported Platforms:
+        ``CPU``
+
     Examples:
         >>> from mindspore.dataset.audio import linear_fbanks
         >>>
@@ -310,6 +336,23 @@ def melscale_fbanks(n_freqs, f_min, f_max, n_mels, sample_rate, norm=NormType.NO
     Returns:
         numpy.ndarray, the frequency transformation matrix with shape ( `n_freqs` , `n_mels` ).
 
+    Raises:
+        TypeError: If `n_freqs` is not of type int.
+        ValueError: If `n_freqs` is a negative number.
+        TypeError: If `f_min` is not of type float.
+        ValueError: If `f_min` is greater than `f_max` .
+        TypeError: If `f_max` is not of type float.
+        ValueError: If `f_max` is a negative number.
+        TypeError: If `n_mels` is not of type int.
+        ValueError: If `n_mels` is not positive.
+        TypeError: If `sample_rate` is not of type int.
+        ValueError: If `sample_rate` is not positive.
+        TypeError: If `norm` is not of type :class:`mindspore.dataset.audio.NormType` .
+        TypeError: If `mel_type` is not of type :class:`mindspore.dataset.audio.MelType` .
+
+    Supported Platforms:
+        ``CPU``
+
     Examples:
         >>> from mindspore.dataset.audio import melscale_fbanks
         >>>

mindspore/dataset/audio/validators.py

@@ -23,7 +23,7 @@ from mindspore.dataset.core.validator_helpers import check_float32, check_float3
     check_int32_not_zero, check_list_same_size, check_non_negative_float32, check_non_negative_int32, \
     check_pos_float32, check_pos_int32, check_value, INT32_MAX, parse_user_args, type_check
 from mindspore.dataset.audio.utils import BorderType, DensityFunction, FadeShape, GainType, \
-    Interpolation, MelType, Modulation, NormType, ResampleMethod, ScaleType, WindowType
+    Interpolation, MelType, Modulation, NormMode, NormType, ResampleMethod, ScaleType, WindowType
 
 
 def check_amplitude_to_db(method):
@@ -366,6 +366,35 @@ def check_inverse_mel_scale(method):
     return new_method
 
 
+def check_inverse_spectrogram(method):
+    """Wrapper method to check the parameters of InverseSpectrogram."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        [length, n_fft, win_length, hop_length, pad, window, normalized, center, \
+         pad_mode, onesided], _ = parse_user_args(method, *args, **kwargs)
+        if length is not None:
+            check_non_negative_int32(length, "length")
+        check_pos_int32(n_fft, "n_fft")
+        type_check(window, (WindowType,), "window")
+        type_check(normalized, (bool,), "normalized")
+        type_check(center, (bool,), "center")
+        type_check(pad_mode, (BorderType,), "pad_mode")
+        type_check(onesided, (bool,), "onesided")
+        check_non_negative_int32(pad, "pad")
+        if hop_length is not None:
+            check_pos_int32(hop_length, "hop_length")
+        if win_length is not None:
+            check_pos_int32(win_length, "win_length")
+            if win_length > n_fft:
+                raise ValueError(
+                    "Input win_length should be no more than n_fft, but got win_length: {0} and n_fft: {1}.".format(
+                        win_length, n_fft))
+        return method(self, *args, **kwargs)
+
+    return new_method
+
+
 def check_lfilter(method):
     """Wrapper method to check the parameters of LFilter."""
 
@@ -925,6 +954,33 @@ def check_phase_vocoder(method):
     return new_method
 
 
+def check_pitch_shift(method):
+    """Wrapper method to check the parameters of PitchShift."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        [sample_rate, n_steps, bins_per_octave, n_fft, win_length, hop_length, window], _ = parse_user_args(
+            method, *args, **kwargs)
+
+        check_non_negative_int32(sample_rate, "sample_rate")
+        check_int32(n_steps, "n_steps")
+        check_int32_not_zero(bins_per_octave, "bins_per_octave")
+        check_pos_int32(n_fft, "n_fft")
+        type_check(window, (WindowType,), "window")
+
+        if win_length is not None:
+            check_pos_int32(win_length, "win_length")
+            if win_length > n_fft:
+                raise ValueError(
+                    "Input win_length should be no more than n_fft, but got win_length: {0} and n_fft: {1}.".format(
+                        win_length, n_fft))
+        if hop_length is not None:
+            check_pos_int32(hop_length, "hop_length")
+        return method(self, *args, **kwargs)
+
+    return new_method
+
+
 def check_resample(method):
     """Wrapper method to check the parameters of Resample."""
 
@@ -948,3 +1004,169 @@ def check_resample(method):
         return method(self, *args, **kwargs)
 
     return new_method
+
+
+def check_lfcc(method):
+    """Wrapper method to check the parameters of LFCC."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        [sample_rate, n_filter, n_lfcc, f_min, f_max, dct_type, norm, log_lf, speckwargs], _ = parse_user_args(
+            method, *args, **kwargs)
+        type_check(sample_rate, (int,), "sample_rate")
+        check_non_negative_int32(sample_rate, "sample_rate")
+        type_check(n_filter, (int,), "n_filter")
+        check_pos_int32(n_filter, "n_filter")
+        type_check(n_lfcc, (int,), "n_lfcc")
+        check_pos_int32(n_lfcc, "n_lfcc")
+        type_check(log_lf, (bool,), "log_lf")
+        type_check(norm, (NormMode,), "norm")
+        type_check(f_min, (int, float), "f_min")
+        check_non_negative_float32(f_min, "f_min")
+        if f_max is not None:
+            type_check(f_max, (int, float), "f_max")
+            check_non_negative_float32(f_max, "f_max")
+            if f_min > f_max:
+                raise ValueError(
+                    "f_max should be greater than or equal to f_min, but got f_min: {0} and f_max: {1}.".format(
+                        f_min, f_max))
+        else:
+            if f_min >= sample_rate // 2:
+                raise ValueError(
+                    "Input sample_rate // 2 should be greater than f_min when f_max is set to None, but got f_min: {0} "
+                    "and sample_rate: {1}.".format(f_min, sample_rate))
+        if dct_type != 2:
+            raise ValueError("Input dct_type must be 2, but got : {0}.".format(dct_type))
+        if speckwargs is not None:
+            type_check(speckwargs, (dict,), "speckwargs")
+            window = speckwargs["window"]
+            pad_mode = speckwargs["pad_mode"]
+            n_fft = speckwargs["n_fft"]
+            win_length = speckwargs["win_length"]
+            pad = speckwargs["pad"]
+            power = speckwargs["power"]
+            type_check(window, (WindowType,), "window")
+            type_check(pad_mode, (BorderType,), "pad_mode")
+            type_check(pad, (int,), "pad")
+            check_non_negative_int32(pad, "pad")
+            type_check(power, (float,), "power")
+            check_non_negative_float32(power, "power")
+            if n_fft < n_lfcc:
+                raise ValueError(
+                    "n_fft should be greater than or equal to n_lfcc, but got n_fft: {0} and n_lfcc: {1}.".format(
+                        n_fft, n_lfcc))
+            if win_length > n_fft:
+                raise ValueError(
+                    "win_length must be less than or equal to n_fft, but got win_length: {0} and n_fft: {1}.".format(
+                        win_length, n_fft))
+        return method(self, *args, **kwargs)
+
+    return new_method
+
+
+def check_mfcc(method):
+    """Wrapper method to check the parameters of MFCC."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        [sample_rate, n_mfcc, dct_type, norm, log_mels, melkwargs], _ = parse_user_args(method, *args, **kwargs)
+        check_non_negative_int32(sample_rate, "sample_rate")
+        type_check(log_mels, (bool,), "log_mels")
+        type_check(norm, (NormMode,), "norm")
+        check_non_negative_int32(n_mfcc, "n_mfcc")
+        if dct_type != 2:
+            raise ValueError("Input dct_type must be 2, but got : {0}.".format(dct_type))
+
+        if melkwargs is not None:
+            type_check(melkwargs, (dict,), "melkwargs")
+            n_fft = melkwargs["n_fft"]
+            win_length = melkwargs["win_length"]
+            hop_length = melkwargs["hop_length"]
+            f_min = melkwargs["f_min"]
+            f_max = melkwargs["f_max"]
+            pad = melkwargs["pad"]
+            power = melkwargs["power"]
+            normalized = melkwargs["normalized"]
+            center = melkwargs["center"]
+            onesided = melkwargs["onesided"]
+            window = melkwargs["window"]
+            pad_mode = melkwargs["pad_mode"]
+            norm_mel = melkwargs["norm"]
+            mel_scale = melkwargs["mel_scale"]
+            n_mels = melkwargs["n_mels"]
+
+            check_pos_int32(n_fft, "n_fft")
+            check_mel_scale_n_mels(n_mels)
+            check_mel_scale_freq(f_min, f_max, sample_rate)
+            check_mel_scale_norm(norm_mel)
+            check_mel_scale_mel_type(mel_scale)
+            check_power(power)
+            type_check(window, (WindowType,), "window")
+            type_check(normalized, (bool,), "normalized")
+            type_check(center, (bool,), "center")
+            type_check(pad_mode, (BorderType,), "pad_mode")
+            type_check(onesided, (bool,), "onesided")
+            check_non_negative_int32(pad, "pad")
+            if hop_length is not None:
+                check_pos_int32(hop_length, "hop_length")
+            if f_max is not None:
+                check_non_negative_float32(f_max, "f_max")
+            if win_length is not None:
+                check_non_negative_int32(win_length, "win_length")
+            if n_mels < n_mfcc:
+                raise ValueError("Input n_mels should be greater than or equal to n_mfcc, but got n_mfcc: {0} and " \
+                                 "n_mels: {1}.".format(n_mfcc, n_mels))
+
+        return method(self, *args, **kwargs)
+
+    return new_method
+
+
+def check_mel_spectrogram_freq(f_min, f_max, sample_rate):
+    """Wrapper method to check the parameters of f_min and f_max."""
+    type_check(f_min, (float,), "f_min")
+
+    if f_max is not None:
+        check_non_negative_float32(f_max, "f_max")
+        if f_min > f_max:
+            raise ValueError("f_max should be greater than or equal to f_min, but got f_min: {0} and f_max: {1}."
+                             .format(f_min, f_max))
+    else:
+        if f_min >= sample_rate // 2:
+            raise ValueError(
+                "MelSpectrogram: sample_rate // 2 should be greater than f_min when f_max is set to None, "
+                "but got f_min: {0}.".format(f_min))
+
+
+def check_mel_spectrogram(method):
+    """Wrapper method to check the parameters of MelSpectrogram."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        [sample_rate, n_fft, win_length, hop_length, f_min, f_max, pad, n_mels, window, power, normalized, center, \
+         pad_mode, onesided, norm, mel_scale], _ = parse_user_args(method, *args, **kwargs)
+        check_non_negative_int32(sample_rate, "sample_rate")
+        check_pos_int32(n_fft, "n_fft")
+        check_non_negative_int32(n_mels, "n_mels")
+        check_mel_spectrogram_freq(f_min, f_max, sample_rate)
+        check_mel_scale_norm(norm)
+        check_mel_scale_mel_type(mel_scale)
+        check_pos_float32(power, "power")
+        type_check(window, (WindowType,), "window")
+        type_check(normalized, (bool,), "normalized")
+        type_check(center, (bool,), "center")
+        type_check(pad_mode, (BorderType,), "pad_mode")
+        type_check(onesided, (bool,), "onesided")
+        check_non_negative_int32(pad, "pad")
+        if hop_length is not None:
+            check_pos_int32(hop_length, "hop_length")
+        if win_length is not None:
+            check_pos_int32(win_length, "win_length")
+            if win_length > n_fft:
+                raise ValueError(
+                    "Input win_length should be no more than n_fft, but got win_length: {0} and n_fft: {1}.".format(
+                        win_length, n_fft))
+
+        return method(self, *args, **kwargs)
+
+    return new_method

mindspore/dataset/callback/ds_callback.py

@@ -131,7 +131,7 @@ class WaitedDSCallback(Callback, DSCallback):
     r"""
     Abstract base class used to build dataset callback classes that are synchronized with the training callback class
     `mindspore.train.Callback \
-    <https://www.mindspore.cn/docs/en/r2.0.0-alpha/api_python/train/
+    <https://www.mindspore.cn/docs/en/r2.0/api_python/train/
     mindspore.train.Callback.html#mindspore.train.Callback>`_ .
 
     It can be used to execute a custom callback method before a step or an epoch, such as
@@ -142,7 +142,7 @@ class WaitedDSCallback(Callback, DSCallback):
     `device_number` , `list_callback` , `cur_epoch_num` , `cur_step_num` , `dataset_sink_mode` ,
     `net_outputs` , etc., see
     `mindspore.train.Callback \
-    <https://www.mindspore.cn/docs/en/r2.0.0-alpha/api_python/train/
+    <https://www.mindspore.cn/docs/en/r2.0/api_python/train/
     mindspore.train.Callback.html#mindspore.train.Callback>`_ .
 
     Users can obtain the dataset pipeline context through `ds_run_context` , including

mindspore/dataset/core/config.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2022 Huawei Technologies Co., Ltd
+# Copyright 2019-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,14 +23,15 @@ Common imported modules in corresponding API examples are as follows:
     import mindspore.dataset as ds
 """
 from __future__ import absolute_import
-
+from enum import IntEnum
 import os
 import platform
 import random
 import numpy
 import mindspore._c_dataengine as cde
 from mindspore import log as logger
-from mindspore.dataset.core.validator_helpers import replace_none
+from mindspore.dataset.core.validator_helpers import replace_none, type_check
+from mindspore.dataset.debug import DebugHook, PrintMetaDataHook
 
 __all__ = ['set_sending_batches', 'load', '_init_device_info',
            'set_seed', 'get_seed',
@@ -46,12 +47,15 @@ __all__ = ['set_sending_batches', 'load', '_init_device_info',
            'set_auto_offload', 'get_auto_offload',
            'set_enable_watchdog', 'get_enable_watchdog',
            'set_fast_recovery', 'get_fast_recovery',
+           'set_debug_mode', 'get_debug_mode',
+           'set_error_samples_mode', 'get_error_samples_mode', 'ErrorSamplesMode',
            'set_multiprocessing_timeout_interval', 'get_multiprocessing_timeout_interval']
 
 INT32_MAX = 2147483647
 UINT32_MAX = 4294967295
 
 _config = cde.GlobalContext.config_manager()
+_debug_context = {}
 
 
 def _init_device_info():
@@ -134,7 +138,7 @@ def get_seed():
     """
     Get random number seed. If the seed has been set, then will
     return the set value, otherwise it will return the default seed value
-    which equals to std::mt19937::default_seed.
+    which equals to `std::mt19937::default_seed <http://www.cplusplus.com/reference/random/mt19937/>`_ .
 
     Returns:
         int, random number seed.
@@ -152,11 +156,12 @@ def set_prefetch_size(size):
     Set the queue capacity of the thread in pipeline.
 
     Args:
-        size (int): The length of the cache queue.
+        size (int): The length of the cache queue. The `size` must be greater than 0, otherwise the queue capacity of
+            the thread is invalid.
 
     Raises:
         TypeError: If `size` is not of type int.
-        ValueError: If `size` <= 0 or `size` > INT32_MAX(2147483647).
+        ValueError: If `size` is not a positive number.
 
     Note:
         Since total memory used for prefetch can grow very large with high number of workers,
@@ -234,7 +239,8 @@ def get_num_parallel_workers():
 
 def set_numa_enable(numa_enable):
     """
-    Set the default state of numa enabled. If numa_enable is True, need to ensure numa library is installed.
+    Set the default state of numa enabled. If `numa_enable` is True, need to
+    ensure `numa library <http://rpmfind.net/linux/rpm2html/search.php?query=libnuma-devel>`_ is installed.
 
     Args:
         numa_enable (bool): Whether to use numa bind feature.
@@ -375,14 +381,15 @@ def get_auto_num_workers():
 
 def set_callback_timeout(timeout):
     """
-    Set the default timeout (in seconds) for DSWaitedCallback.
+    Set the default timeout (in seconds) for :class:`mindspore.dataset.WaitedDSCallback` .
 
     Args:
-        timeout (int): Timeout (in seconds) to be used to end the wait in DSWaitedCallback in case of a deadlock.
+        timeout (int): Timeout (in seconds) to be used to end the wait in :class:`mindspore.dataset.WaitedDSCallback`
+            in case of a deadlock. The `timeout` must be greater than 0.
 
     Raises:
         TypeError: If `timeout` is not type int.
-        ValueError: If `timeout` <= 0 or `timeout` > INT32_MAX(2147483647).
+        ValueError: If `timeout` is not a positive number.
 
     Examples:
         >>> # Set a new global configuration value for the timeout value.
@@ -397,10 +404,11 @@ def set_callback_timeout(timeout):
 
 def get_callback_timeout():
     """
-    Get the default timeout for WaitedDSCallback.
+    Get the default timeout for :class:`mindspore.dataset.WaitedDSCallback` .
 
     Returns:
-        int, Timeout (in seconds) to be used to end the wait in DSWaitedCallback in case of a deadlock.
+        int, Timeout (in seconds) to be used to end the wait in :class:`mindspore.dataset.WaitedDSCallback` in case of
+        a deadlock.
 
     Examples:
         >>> # Get the global configuration of callback timeout.
@@ -805,10 +813,10 @@ def get_dynamic_shape():
 def set_fast_recovery(fast_recovery):
     """
     Set whether dataset pipeline should recover in fast mode during failover
-    (yet with slightly different random augmentations).
+    (In fast mode, random augmentations may not get same results as before the failure occurred).
 
     Args:
-        fast_recovery (bool): Whether the dataset pipeline recovers in fast mode.
+        fast_recovery (bool): Whether the dataset pipeline recovers in fast mode. System default: True.
 
     Raises:
         TypeError: If `fast_recovery` is not a boolean data type.
@@ -832,3 +840,191 @@ def get_fast_recovery():
         >>> is_fast_recovery = ds.config.get_fast_recovery()
     """
     return _config.get_fast_recovery()
+
+
+def set_debug_mode(debug_mode_flag: bool, debug_hook_list: list = None):
+    """
+    Set the debug_mode flag of the dataset pipeline. When enabled, the dataset pipeline is run synchronously and
+    sequentially with a single thread.
+
+    Note:
+        - When debug_mode is enabled, if set_seed has not yet been issued, MindData will internally set the seed to 1
+          so that debug mode execution of the dataset pipeline can produce deterministic results.
+        - When debug_mode is enabled, many configuration settings are ignored, including the following noteworthy
+          settings:
+            - auto_offload (False is used.)
+            - enable_autotune (False is used.)
+            - error_samples_mode (ErrorSamplesMode.RETURN is used.)
+            - num_parallel_workers (Value 1 is used.)
+        - If both debug_mode is enabled and a dataset pipeline has Map operation with offload set, then offloading is
+          ignored.
+        - If both debug_mode is enabled and a dataset pipeline has Map operation or Batch operation with
+          python_multiprocessing=True, then Python multiprocessing is ignored.
+        - If both debug_mode is enabled and a dataset pipeline has GeneratorDataset with
+          python_multiprocessing=True (the default value), then Python multiprocessing is ignored.
+        - If both debug_mode is enabled and a dataset operation has cache set, then the cache is dropped.
+        - If both debug_mode and profiling are enabled, then dataset profiling is ignored.
+
+    Args:
+        debug_mode_flag (bool): Whether dataset pipeline debug mode is enabled, which forces the pipeline
+            to run synchronously and sequentially.
+        debug_hook_list (list[DebugHook]): a list of debug hook objects to be inserted before and after each
+            transform operation in map operation. Default: None, which means to use `[PrintMetaDataHook]`,
+            which prints shape/size/type of each input/output data of each transformation.
+
+    Raises:
+        TypeError: If `debug_mode_flag` is not a boolean data type.
+        TypeError: If `debug_hook_list` is not a list type.
+        TypeError: If any item in `debug_hook_list` is not DebugHook type.
+
+    Examples:
+        1. Enable dataset pipeline debug mode and use default debug hook.
+        >>> # Print shape and type of input/output data of each transform op in map operator.
+        >>> ds.config.set_debug_mode(True)
+
+        2. Enable dataset pipeline debug mode and use pre-defined debug hook provided by MindData.
+        >>> import mindspore.dataset.debug as debug
+        >>>
+        >>> ds.config.set_debug_mode(True, debug_hook_list=[debug.PrintDataHook()])
+
+        3. Enable dataset pipeline debug mode and use user-defined debug hook. It must define a
+        class inherited from DebugHook.
+        >>> import mindspore.dataset.debug as debug
+        >>>
+        >>> class CustomizedHook(debug.DebugHook):
+        ...     def __init__(self):
+        ...         super().__init__()
+        ...
+        ...     def compute(self, *args):
+        ...         # Add your debugging code here.
+        ...         return args
+        >>>
+        >>> ds.config.set_debug_mode(True, debug_hook_list=[CustomizedHook()])
+
+        4. Enable dataset pipeline debug mode and use user-defined debug hook and insert by users manually.
+        >>> ds.config.set_debug_mode(True)
+        >>> dataset = ds.ImageFolderDataset(dataset_dir="/path/to/image_folder_dataset_directory")
+        >>> # the debug hook is added after `Decode` operation.
+        >>> dataset = dataset.map([vision.Decode(), CustomizedHook(), vision.CenterCrop(100)])
+    """
+    if not isinstance(debug_mode_flag, bool):
+        raise TypeError("debug_mode_flag isn't of type boolean.")
+    if not debug_hook_list:
+        debug_hook_list = [PrintMetaDataHook()]
+    if not isinstance(debug_hook_list, list):
+        raise TypeError("debug_hook_list is not a list.")
+    for debug_func in debug_hook_list:
+        if not isinstance(debug_func, DebugHook):
+            raise TypeError("All items in debug_hook_list must be of type DebugHook.")
+    if debug_mode_flag:
+        logger.warning("Dataset pipeline debug mode is enabled. Performance will be impacted because the pipeline"
+                       " will be running in a single thread.")
+    if debug_hook_list:
+        _debug_context["debug_hook_list"] = debug_hook_list
+
+    _config.set_debug_mode(debug_mode_flag)
+
+
+def get_debug_mode():
+    """
+    Get the debug_mode flag of the dataset pipeline
+
+    Returns:
+        bool, whether dataset pipeline debug mode is enabled
+
+    Examples:
+        >>> debug_mode = ds.config.get_debug_mode()
+    """
+    return _config.get_debug_mode()
+
+
+def _get_debug_hook_list():
+    """
+    INTERNAL USE ONLY!
+    Get value of debug_hook_list.
+
+    Returns:
+        list, the debug hook objects to be inserted in map operation to debug inputs/outputs of each transform.
+    """
+    return _debug_context.get("debug_hook_list")
+
+
+class ErrorSamplesMode(IntEnum):
+    """
+    An enumeration for `error_samples_mode` .
+
+    Possible enumeration values are: ErrorSamplesMode.RETURN, ErrorSamplesMode.REPLACE, ErrorSamplesMode.SKIP.
+
+    - ErrorSamplesMode.RETURN: means erroneous sample results in error raised and returned.
+    - ErrorSamplesMode.REPLACE: means erroneous sample is replaced with an internally determined sample.
+    - ErrorSamplesMode.SKIP: means erroneous sample is skipped.
+    """
+
+    RETURN = 0
+    REPLACE = 1
+    SKIP = 2
+
+
+# Convert ErrorSamplesMode from Python enum format to CDE enum format
+_PYTHON_TO_CDE_ERROR_SAMPLES_MODE = {
+    ErrorSamplesMode.RETURN: cde.ErrorSamplesMode.DE_ERROR_SAMPLES_MODE_RETURN,
+    ErrorSamplesMode.REPLACE: cde.ErrorSamplesMode.DE_ERROR_SAMPLES_MODE_REPLACE,
+    ErrorSamplesMode.SKIP: cde.ErrorSamplesMode.DE_ERROR_SAMPLES_MODE_SKIP
+}
+
+# Convert ErrorSamplesMode from CDE int format to Python enum format
+_CDE_TO_PYTHON_ERROR_SAMPLES_MODE = {
+    0: ErrorSamplesMode.RETURN,
+    1: ErrorSamplesMode.REPLACE,
+    2: ErrorSamplesMode.SKIP
+}
+
+
+def set_error_samples_mode(error_samples_mode):
+    """
+    Set the method in which erroneous samples should be processed in a dataset pipeline.
+
+    Note:
+        - This error samples feature is only applicable to the Map operation in a dataset pipeline.
+        - For 'ErrorSamplesMode.REPLACE' mode, a cache of other samples will be used.
+        - If 'ErrorSamplesMode.SKIP' mode is used in a distributed setting, beware to manually ensure the
+          number of valid samples are the same for each shard (otherwise one may encounter hangs).
+          One technique is to manually concat a dataset of all valid samples plus a
+          take operation for the number of skipped erroneous samples.
+
+    Args:
+        error_samples_mode (ErrorSamplesMode): The method in which erroneous samples should be processed in a dataset
+            pipeline. It can be any of [ErrorSamplesMode.RETURN, ErrorSamplesMode.REPLACE, ErrorSamplesMode.SKIP].
+            System default: ErrorSamplesMode.RETURN.
+
+            - ErrorSamplesMode.RETURN: means erroneous sample results in error raised and returned.
+
+            - ErrorSamplesMode.REPLACE: means erroneous sample is replaced with a correct sample.
+
+            - ErrorSamplesMode.SKIP: means erroneous sample is skipped.
+
+    Raises:
+        TypeError: If `error_samples_mode` is not of type ErrorSamplesMode.
+
+    Examples:
+        >>> ds.config.set_error_samples_mode(ds.config.ErrorSamplesMode.SKIP)
+    """
+    type_check(error_samples_mode, (ErrorSamplesMode,), "error_samples_mode")
+    _config.set_error_samples_mode(_PYTHON_TO_CDE_ERROR_SAMPLES_MODE.get(error_samples_mode))
+
+
+def get_error_samples_mode():
+    """
+    Get the current configuration for strategy for processing erroneous samples in a dataset pipeline.
+
+    Returns:
+        ErrorSamplesMode, The method in which erroneous samples should be processed in a dataset pipeline.
+
+        - ErrorSamplesMode.RETURN: means erroneous sample results in error raised and returned.
+        - ErrorSamplesMode.REPLACE: means erroneous sample is replaced with an internally determined sample.
+        - ErrorSamplesMode.SKIP: means erroneous sample is skipped.
+
+    Examples:
+        >>> error_samples_mode = ds.config.get_error_samples_mode()
+    """
+    return _CDE_TO_PYTHON_ERROR_SAMPLES_MODE.get(_config.get_error_samples_mode())

mindspore/dataset/core/validator_helpers.py

@@ -746,9 +746,9 @@ def check_gnn_list_or_ndarray(param, param_name, data_type=int):
         elif data_type == str:
             data_type = np.str_
 
-        if param.dtype.type != data_type:
+        if param.dtype != data_type:
             raise TypeError("Each member in {0} should be of type {1}. Got {2}.".format(
-                param_name, data_type, param.dtype.type))
+                param_name, data_type, param.dtype))
 
 
 def check_tensor_op(param, param_name):