mindspore 2.0.0a0__cp37-none-any.whl → 2.0.0rc1__cp37-none-any.whl

mindspore/dataset/engine/graphdata.py

@@ -80,7 +80,7 @@ class GraphData:
 
     About how to load raw graph dataset into MindSpore please
     refer to `Loading Graph Dataset <https://www.mindspore.cn/tutorials/en/
-    r2.0.0-alpha/advanced/dataset/augment_graph_data.html>`_ .
+    r2.0/advanced/dataset/augment_graph_data.html>`_ .
 
     Args:
         dataset_file (str): One of file names in the dataset.
@@ -202,6 +202,13 @@ class GraphData:
         Returns:
             numpy.ndarray, array of nodes.
 
+        Examples:
+            >>> from mindspore.dataset import GraphData
+            >>>
+            >>> g = ds.GraphData("/path/to/testdata", 1)
+            >>> edges = g.get_all_edges(0)
+            >>> nodes = g.get_nodes_from_edges(edges)
+
         Raises:
             TypeError: If `edge_list` is not list or ndarray.
         """
@@ -492,6 +499,12 @@ class GraphData:
         Returns:
             dict, meta information of the graph. The key is node_type, edge_type, node_num, edge_num,
             node_feature_type and edge_feature_type.
+
+        Examples:
+        >>> from mindspore.dataset import GraphData
+        >>>
+        >>> g = ds.GraphData("/path/to/testdata", 2)
+        >>> graph_info = g.graph_info()
         """
         if self._working_mode == 'server':
             raise Exception("This method is not supported when working mode is server.")
@@ -1286,17 +1299,29 @@ class InMemoryGraphDataset(GeneratorDataset):
             Default: 'graph'.
         num_samples (int, optional): The number of samples to be included in the dataset. Default: None, all samples.
         num_parallel_workers (int, optional): Number of subprocesses used to fetch the dataset in parallel. Default: 1.
-        shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
-            Default: None, expected order behavior shown in the table below.
+        shuffle (bool, optional): Whether or not to perform shuffle on the dataset. This parameter can only be
+            specified when the implemented dataset has a random access attribute ( `__getitem__` ). Default: None.
         num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
             When this argument is specified, `num_samples` reflects the max
             sample number of per shard.
         shard_id (int, optional): The shard ID within `num_shards` . Default: None. This argument must be specified only
-            when num_shards is also specified.
+            when `num_shards` is also specified.
         python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker process. This
             option could be beneficial if the Python operation is computational heavy. Default: True.
         max_rowsize(int, optional): Maximum size of row in MB that is used for shared memory allocation to copy
-            data between processes.  This is only used if python_multiprocessing is set to True. Default: 6 MB.
+            data between processes. This is only used if python_multiprocessing is set to True. Default: 6 MB.
+
+    Raises:
+        TypeError: If `data_dir` is not of type str.
+        TypeError: If `save_dir` is not of type str.
+        TypeError: If `num_parallel_workers` is not of type int.
+        TypeError: If `shuffle` is not of type bool.
+        TypeError: If `python_multiprocessing` is not of type bool.
+        TypeError: If `perf_mode` is not of type bool.
+        RuntimeError: If `data_dir` is not valid or does not exit.
+        RuntimeError: If `num_shards` is specified but `shard_id` is None.
+        RuntimeError: If `shard_id` is specified but `num_shards` is None.
+        ValueError: If `num_parallel_workers` exceeds the max thread numbers.
 
     Examples:
         >>> from mindspore.dataset import InMemoryGraphDataset, Graph
@@ -1385,19 +1410,28 @@ class ArgoverseDataset(InMemoryGraphDataset):
     Args:
         data_dir (str): directory for loading dataset, here contains origin format data and will be loaded in
             `process` method.
-        column_names (Union[str, list[str]], optional): single column name or list of column names of the dataset,
-            num of column name should be equal to num of item in return data when implement method like `__getitem__` ,
-            recommend to specify it with
+        column_names (Union[str, list[str]], optional): single column name or list of column names of the dataset.
+            Default: "graph". Num of column name should be equal to num of item in return data when implement method
+            like `__getitem__`, recommend to specify it with
             `column_names=["edge_index", "x", "y", "cluster", "valid_len", "time_step_len"]` like the following example.
         num_parallel_workers (int, optional): Number of subprocesses used to fetch the dataset in parallel. Default: 1.
-        shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
-            Default: None, expected order behavior shown in the table below.
+        shuffle (bool, optional): Whether or not to perform shuffle on the dataset. This parameter can only be
+            specified when the implemented dataset has a random access attribute ( `__getitem__` ). Default: None.
         python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker process. This
             option could be beneficial if the Python operation is computational heavy. Default: True.
         perf_mode(bool, optional): mode for obtaining higher performance when iterate created dataset(will call
             `__getitem__` method in this process). Default True, will save all the data in graph
             (like edge index, node feature and graph feature) into graph feature.
 
+    Raises:
+        TypeError: If `data_dir` is not of type str.
+        TypeError: If `num_parallel_workers` is not of type int.
+        TypeError: If `shuffle` is not of type bool.
+        TypeError: If `python_multiprocessing` is not of type bool.
+        TypeError: If `perf_mode` is not of type bool.
+        RuntimeError: If `data_dir` is not valid or does not exit.
+        ValueError: If `num_parallel_workers` exceeds the max thread numbers.
+
     Examples:
         >>> from mindspore.dataset import ArgoverseDataset
         >>>
@@ -1407,6 +1441,37 @@ class ArgoverseDataset(InMemoryGraphDataset):
         ...                                                "time_step_len"])
         >>> for item in graph_dataset.create_dict_iterator(output_numpy=True, num_epochs=1):
         ...     pass
+
+    About Argoverse Dataset:
+
+    Argverse is the first dataset containing high-precision maps, which contains 290KM high-precision map data with
+    geometric shape and semantic information.
+
+    You can unzip the dataset files into the following structure and read by MindSpore's API:
+
+    .. code-block::
+
+        .
+        └── argoverse_dataset_dir
+            ├── train
+            │    ├──...
+            ├── val
+            │    └──...
+            ├── test
+            │    └──...
+
+    Citation:
+
+    .. code-block::
+
+        @inproceedings{Argoverse,
+        author     = {Ming-Fang Chang and John W Lambert and Patsorn Sangkloy and Jagjeet Singh
+                   and Slawomir Bak and Andrew Hartnett and De Wang and Peter Carr
+                   and Simon Lucey and Deva Ramanan and James Hays},
+        title      = {Argoverse: 3D Tracking and Forecasting with Rich Maps},
+        booktitle  = {Conference on Computer Vision and Pattern Recognition (CVPR)},
+        year       = {2019}
+        }
     """
 
     def __init__(self, data_dir, column_names="graph", num_parallel_workers=1, shuffle=None,

mindspore/dataset/engine/iterators.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2022 Huawei Technologies Co., Ltd
+# Copyright 2019-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Built-in iterators.
-"""
+"""Built-in iterators"""
 from abc import abstractmethod
+from copy import deepcopy
 import json
 import os
 import signal
@@ -22,8 +22,9 @@ import weakref
 import numpy as np
 
 import mindspore._c_dataengine as cde
-from mindspore.common.tensor import Tensor
+from mindspore.common.tensor import Tensor, np_types
 import mindspore.dataset.engine.offload as offload
+from mindspore.dataset.core.config import get_debug_mode
 
 from mindspore import log as logger
 
@@ -75,7 +76,10 @@ class Iterator:
 
         self._runtime_context = cde.PythonRuntimeContext()
         self._runtime_context.Init()
-        consumer = cde.PythonIteratorConsumer(num_epochs)
+        if get_debug_mode():
+            consumer = cde.PythonPullBasedIteratorConsumer(num_epochs)
+        else:
+            consumer = cde.PythonIteratorConsumer(num_epochs)
         consumer.Init(self.ir_tree)
         self._runtime_context.AssignConsumer(consumer)
         self._iterator = self._runtime_context.GetConsumer()
@@ -179,12 +183,48 @@ class Iterator:
         """
         self._iterator.Reset(step, epoch)
 
+    def __convert_python(self, obj, to_numpy):
+        """
+        Attempts to recursively convert a python object to Numpy array(s) or tensor(s).
+
+        Args:
+            obj (any): the python object to be converted
+            to_numpy (bool): If True, convert primitive types to NumPy array. If False, convert to Tensor.
+                             (return the obj if type isn't supported)
+        """
+        if isinstance(obj, (int, float, bool, str, np.ndarray, np.str_, np.bytes_, *np_types)):
+            # error out if array is of unsupported type
+            if isinstance(obj, np.ndarray) and obj.dtype not in np_types and obj.dtype.kind not in ('U', 'S'):
+                new_line = '\n'
+                raise TypeError("A NumPy array of unsupported type detected: {}."
+                                "\nSupported types are: {}.".format(
+                                    obj.dtype, new_line.join(map(str, (*np_types, np.str_, np.bytes_)))))
+            if to_numpy:
+                return np.array(obj, copy=self._do_copy)
+            if self._do_copy:
+                return Tensor(np.asarray(obj))
+            return Tensor.from_numpy(np.asarray(obj))
+        if isinstance(obj, dict):
+            return {key: self.__convert_python(val, to_numpy) for key, val in obj.items()}
+        if isinstance(obj, tuple):
+            return tuple([self.__convert_python(item, to_numpy) for item in obj])
+        if isinstance(obj, list):
+            return [self.__convert_python(item, to_numpy) for item in obj]
+        # if we can't convert it to Tensor, return the object as is
+        if self._do_copy:
+            return deepcopy(obj)
+        return obj
+
     def _transform_md_to_output(self, t):
         if self._output_numpy:
+            if t.type().is_python():
+                return self.__convert_python(t.as_python(), True)
             return t.as_array()
         return self._transform_md_to_tensor(t)
 
     def _transform_md_to_tensor(self, t):
+        if t.type().is_python():
+            return self.__convert_python(t.as_python(), False)
         array = t.as_array()
         if self._do_copy:
             return Tensor(array)

mindspore/dataset/engine/offload.py

@@ -23,7 +23,7 @@ import mindspore.ops as ops
 import mindspore.nn as nn
 import mindspore.ops.composite as C
 from mindspore.ops import operations as P
-from mindspore.ops.primitive import constexpr
+from mindspore.ops.primitive import _primexpr
 from mindspore import log as logger
 
 
@@ -88,7 +88,7 @@ def apply_offload_iterators(data, offload_model):
             data[i] = Tensor(data[i], dtype=mstype.float32)
             non_tensor_idxs.append(i)
 
-    data = offload_model(data)
+    data = offload_model(*data)
     data = list(data)
     for idx in non_tensor_idxs:
         data[idx] = data[idx].asnumpy()
@@ -96,7 +96,7 @@ def apply_offload_iterators(data, offload_model):
     return data
 
 
-@constexpr
+@_primexpr
 def check_input_dims(x_shape, required_dim, offload_op_name):
     """
     Check if input has the required number of dimensions for the operation.
@@ -136,7 +136,7 @@ class ApplyPreTransform(nn.Cell):
         for data_col in x:
             data.append(data_col)
 
-        data = self.transform(data)
+        data = self.transform(*data)
         data = self.model(*data)
 
         return data
@@ -183,9 +183,10 @@ class RandomHorizontalFlip(nn.Cell):
         flip_rand_factor = self.reshape(C.repeat_elements(flip_rand_factor, rep=(h*w*c)), (bs, h, w, c))
 
         x_flip = self.h_flip(x)
-        x = self.mul(x_flip, flip_rand_factor) + self.mul((1 - flip_rand_factor), x)
-
-        return x
+        operation = self.mul(x_flip, flip_rand_factor) + self.mul((1 - flip_rand_factor), x)
+        # ops.depend is added to find the RandomHorizontalFlip operator in IR files.
+        depend = ops.depend(operation, "RandomHorizontalFlip")
+        return depend
 
 
 class RandomVerticalFlip(nn.Cell):
@@ -216,9 +217,10 @@ class RandomVerticalFlip(nn.Cell):
         flip_rand_factor = self.reshape(C.repeat_elements(flip_rand_factor, rep=(h*w*c)), (bs, h, w, c))
 
         x_flip = self.h_flip(x)
-        x = self.mul(x_flip, flip_rand_factor) + self.mul((1 - flip_rand_factor), x)
-
-        return x
+        operation = self.mul(x_flip, flip_rand_factor) + self.mul((1 - flip_rand_factor), x)
+        # ops.depend is added to find the RandomVerticalFlip operator in IR files.
+        depend = ops.depend(operation, "RandomVerticalFlip")
+        return depend
 
 
 class GenerateRandBatch(nn.Cell):
@@ -362,9 +364,10 @@ class RandomColorAdjust(nn.Cell):
         x_rgb = x_rgb + C.repeat_elements(self.expand_dims((v - c), 1), 3, 1)
 
         x_rgb = self.transpose(x, (0, 2, 3, 1)) * 255.0
-        x_rgb = ops.clip_by_value(x, 0.0, 255.0)
-
-        return x_rgb
+        operation = ops.clip_by_value(x, 0.0, 255.0)
+        # ops.depend is added to find the RandomColorAdjust operator in IR files.
+        depend = ops.depend(operation, "RandomColorAdjust")
+        return depend
 
 
 class RandomSharpness(nn.Cell):
@@ -415,8 +418,10 @@ class RandomSharpness(nn.Cell):
         x_sharp = self.transpose(x_sharp, (0, 2, 3, 1))
 
         x = self.mul(x, degree_rand_factor) + self.mul((1 - degree_rand_factor), x_sharp)
-        x = ops.clip_by_value(x, 0.0, 255.0)
-        return x
+        operation = ops.clip_by_value(x, 0.0, 255.0)
+        # ops.depend is added to find the RandomSharpness operator in IR files.
+        depend = ops.depend(operation, "RandomSharpness")
+        return depend
 
 
 class Rescale(nn.Cell):
@@ -436,9 +441,10 @@ class Rescale(nn.Cell):
     def construct(self, x):
 
         x = self.cast(x, mstype.float32)
-        x = x * self.rescale + self.shift
-
-        return x
+        operation = x * self.rescale + self.shift
+        # ops.depend is added to find the Rescale operator in IR files.
+        depend = ops.depend(operation, "Rescale")
+        return depend
 
 
 class HwcToChw(nn.Cell):
@@ -454,7 +460,10 @@ class HwcToChw(nn.Cell):
     def construct(self, x):
         x_shape = self.shape(x)
         check_input_dims(x_shape, 4, 'HwcToChw')
-        return self.trans(x, (0, 3, 1, 2))
+        operation = self.trans(x, (0, 3, 1, 2))
+        # ops.depend is added to find the HwcToChw operator in IR files.
+        depend = ops.depend(operation, "HwcToChw")
+        return depend
 
 
 class Normalize(nn.Cell):
@@ -462,8 +471,11 @@ class Normalize(nn.Cell):
     Applies Normalize transform on given input tensors.
     """
 
-    def __init__(self, mean, std):
+    def __init__(self, mean, std, is_hwc=True):
         super(Normalize, self).__init__()
+        if is_hwc is False:
+            mean = np.expand_dims(np.array(mean), (1, 2))
+            std = np.expand_dims(np.array(std), (1, 2))
         self.mean = Tensor(mean, mstype.float32)
         self.std = Tensor(std, mstype.float32)
         self.sub = P.Sub()
@@ -473,8 +485,10 @@ class Normalize(nn.Cell):
     def construct(self, x):
         x = self.cast(x, mstype.float32)
         x = self.sub(x, self.mean)
-        x = self.div(x, self.std)
-        return x
+        operation = self.div(x, self.std)
+        # ops.depend is added to find the Normalize operator in IR files.
+        depend = ops.depend(operation, "Normalize")
+        return depend
 
 
 class TypeCast(nn.Cell):
@@ -489,8 +503,10 @@ class TypeCast(nn.Cell):
         self.data_type = mstype.typing.str_to_type(data_type_str)
 
     def construct(self, x):
-
-        return self.cast(x, self.data_type)
+        operation = self.cast(x, self.data_type)
+        # ops.depend is added to find the TypeCast operator in IR files.
+        depend = ops.depend(operation, "TypeCast")
+        return depend
 
 
 class OffloadModel():
@@ -503,7 +519,7 @@ class OffloadModel():
 op_to_model = {
     "HWC2CHW": OffloadModel(HwcToChw),
     "HwcToChw": OffloadModel(HwcToChw),
-    "Normalize": OffloadModel(Normalize, ["mean", "std"]),
+    "Normalize": OffloadModel(Normalize, ["mean", "std", "is_hwc"]),
     "RandomColorAdjust": OffloadModel(RandomColorAdjust, ["brightness", "contrast", "saturation", "hue"]),
     "RandomHorizontalFlip": OffloadModel(RandomHorizontalFlip, ["prob"]),
     "RandomSharpness": OffloadModel(RandomSharpness, ["degrees"]),
@@ -572,7 +588,11 @@ class GetOffloadModel(nn.Cell):
                     self.transform_list.append(GetModelFromJson2Col(node, ds_col_idxs))
             self.transform_list.reverse()
 
-    def construct(self, x):
+    def construct(self, *x):
+        data = []
+        for d in x:
+            data.append(d)
+
         for transform in self.transform_list:
-            x = transform(x)
-        return x
+            data = transform(data)
+        return data

mindspore/dataset/engine/validators.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2022 Huawei Technologies Co., Ltd
+# Copyright 2019-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -587,9 +587,10 @@ def check_tfrecorddataset(method):
 
         compression_type = param_dict.get('compression_type')
         if compression_type is not None and compression_type not in ['', 'ZLIB', 'GZIP']:
-            raise ValueError("Input compression_type can only be either '' (no compression), 'ZLIB', or 'GZIP', \
-                             but got '" + str(compression_type) + "'.")
-        if compression_type is not None and compression_type in ['ZLIB', 'GZIP']:
+            raise ValueError("Input compression_type can only be either '' (no compression), 'ZLIB', or 'GZIP', " +
+                             "but got '" + str(compression_type) + "'.")
+        if compression_type is not None and compression_type in ['ZLIB', 'GZIP'] and \
+            param_dict.get('num_samples') is not None:
             if param_dict.get('num_shards') is not None and ((isinstance(dataset_files, str) and \
                 param_dict.get('num_shards') > 1) or (isinstance(dataset_files, list) and \
                 len(dataset_files) < param_dict.get('num_shards'))):
@@ -598,9 +599,6 @@ def check_tfrecorddataset(method):
                 raise ValueError("When compression_type is provided, the number of dataset files cannot be less " +
                                  "than num_shards, but the actual number of files is " + str(num_files) +
                                  " and actual num_shards is " + str(act_num_shard) + ".")
-            if param_dict.get('num_samples') is None or param_dict.get('num_samples') <= 0:
-                raise ValueError("When compression_type is provided, num_samples must be provided and > 0, but got " +
-                                 str(param_dict.get('num_samples')) + " for num_samples.")
             if param_dict.get('shard_equal_rows') is None or not param_dict.get('shard_equal_rows'):
                 logger.warning("If compression_type is set, shard_equal_rows will be ignored.")
 
@@ -998,7 +996,7 @@ def check_dict_iterator(method):
 
     @wraps(method)
     def new_method(self, *args, **kwargs):
-        [num_epochs, _], param_dict = parse_user_args(method, *args, **kwargs)
+        [num_epochs, _, _], param_dict = parse_user_args(method, *args, **kwargs)
         nreq_param_bool = ['output_numpy']
         validate_dataset_param_value(nreq_param_bool, param_dict, bool)
         if num_epochs is not None:
@@ -1166,6 +1164,34 @@ def check_random_dataset(method):
     return new_method
 
 
+def check_rendered_sst2_dataset(method):
+    """A wrapper that wraps a parameter checker around the original Dataset(RenderedSST2Dataset)."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        _, param_dict = parse_user_args(method, *args, **kwargs)
+
+        nreq_param_int = ['num_samples', 'num_parallel_workers', 'num_shards', 'shard_id']
+        nreq_param_bool = ['shuffle', 'decode']
+
+        dataset_dir = param_dict.get('dataset_dir')
+        usage = param_dict.get('usage')
+        check_dir(dataset_dir)
+        if usage is not None:
+            check_valid_str(usage, ['val', 'all', 'train', 'test'])
+
+        validate_dataset_param_value(nreq_param_int, param_dict, int)
+        validate_dataset_param_value(nreq_param_bool, param_dict, bool)
+        check_sampler_shuffle_shard_options(param_dict)
+
+        cache = param_dict.get('cache')
+        check_cache_option(cache)
+
+        return method(self, *args, **kwargs)
+
+    return new_method
+
+
 def check_pad_info(key, val):
     """check the key and value pair of pad_info in batch"""
     type_check(key, (str,), "key in pad_info")
@@ -2377,6 +2403,36 @@ def check_flickr_dataset(method):
     return new_method
 
 
+def check_food101_dataset(method):
+    """A wrapper that wraps a parameter checker around the Food101Dataset."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        _, param_dict = parse_user_args(method, *args, **kwargs)
+
+        nreq_param_int = ['num_samples', 'num_parallel_workers', 'num_shards', 'shard_id']
+        nreq_param_bool = ['decode', 'shuffle']
+
+        dataset_dir = param_dict.get('dataset_dir')
+        check_dir(dataset_dir)
+
+        usage = param_dict.get('usage')
+        if usage is not None:
+            check_valid_str(usage, ["train", "test", "all"], "usage")
+
+        validate_dataset_param_value(nreq_param_int, param_dict, int)
+        validate_dataset_param_value(nreq_param_bool, param_dict, bool)
+
+        check_sampler_shuffle_shard_options(param_dict)
+
+        cache = param_dict.get('cache')
+        check_cache_option(cache)
+
+        return method(self, *args, **kwargs)
+
+    return new_method
+
+
 def check_sb_dataset(method):
     """A wrapper that wraps a parameter checker around the original Semantic Boundaries Dataset."""
 
@@ -2789,6 +2845,34 @@ def check_svhn_dataset(method):
     return new_method
 
 
+def check_sst2_dataset(method):
+    """A wrapper that wraps a parameter checker around the original SST2 Dataset."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        _, param_dict = parse_user_args(method, *args, **kwargs)
+
+        nreq_param_int = ['num_samples', 'num_parallel_workers', 'num_shards', 'shard_id']
+
+        dataset_dir = param_dict.get('dataset_dir')
+        check_dir(dataset_dir)
+
+        usage = param_dict.get('usage')
+        if usage is not None:
+            check_valid_str(usage, ["train", "test", "dev"], "usage")
+
+        validate_dataset_param_value(nreq_param_int, param_dict, int)
+
+        check_sampler_shuffle_shard_options(param_dict)
+
+        cache = param_dict.get('cache')
+        check_cache_option(cache)
+
+        return method(self, *args, **kwargs)
+
+    return new_method
+
+
 def check_stl10_dataset(method):
     """A wrapper that wraps a parameter checker around the original Dataset(STL10Dataset)."""
 
@@ -2835,6 +2919,31 @@ def check_stl10_dataset(method):
     return new_method
 
 
+def check_sun397_dataset(method):
+    """A wrapper that wraps a parameter checker around the original Dataset(SUN397Dataset)."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        _, param_dict = parse_user_args(method, *args, **kwargs)
+
+        nreq_param_int = ['num_samples', 'num_parallel_workers', 'num_shards', 'shard_id']
+        nreq_param_bool = ['shuffle', 'decode']
+
+        dataset_dir = param_dict.get('dataset_dir')
+        check_dir(dataset_dir)
+
+        validate_dataset_param_value(nreq_param_int, param_dict, int)
+        validate_dataset_param_value(nreq_param_bool, param_dict, bool)
+        check_sampler_shuffle_shard_options(param_dict)
+
+        cache = param_dict.get('cache')
+        check_cache_option(cache)
+
+        return method(self, *args, **kwargs)
+
+    return new_method
+
+
 def check_yahoo_answers_dataset(method):
     """A wrapper that wraps a parameter checker around the original YahooAnswers Dataset."""
 

mindspore/dataset/text/__init__.py

@@ -25,8 +25,7 @@ Common imported modules in corresponding API examples are as follows:
     import mindspore.dataset.text as text
 
 See `Text Transforms
-<https://www.mindspore.cn/tutorials/en/r2.0.0-alpha/beginner/transforms.html#text-transforms>`_ tutorial
-for more details.
+<https://www.mindspore.cn/tutorials/en/r2.0/beginner/transforms.html#text-transforms>`_ tutorial for more details.
 
 Descriptions of common data processing terms are as follows:
 
@@ -36,7 +35,7 @@ Descriptions of common data processing terms are as follows:
 The data transform operation can be executed in the data processing pipeline or in the eager mode:
 
 - Pipeline mode is generally used to process datasets. For examples, please refer to
-  `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/r2.0.0-alpha/api_python/
+  `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/r2.0/api_python/
   mindspore.dataset.html#introduction-to-data-processing-pipeline>`_ .
 - Eager mode is generally used for scattered samples. Examples of text preprocessing are as follows:
 
@@ -66,13 +65,15 @@ The data transform operation can be executed in the data processing pipeline or
       # ids to token
       tokens_from_ids = vocab.ids_to_tokens([15, 3, 7])
       print("token to id: {}".format(tokens_from_ids), flush=True)
+
+Note: In eager mode, non-NumPy input is implicitly converted to NumPy format and sent to MindSpore.
 """
 import platform
 
 from . import transforms
 from . import utils
-from .transforms import JiebaTokenizer, Lookup, Ngram, PythonTokenizer, SentencePieceTokenizer, SlidingWindow, \
-    ToNumber, ToVectors, TruncateSequencePair, UnicodeCharTokenizer, WordpieceTokenizer
+from .transforms import AddToken, JiebaTokenizer, Lookup, Ngram, PythonTokenizer, SentencePieceTokenizer, \
+    SlidingWindow, ToNumber, ToVectors, Truncate, TruncateSequencePair, UnicodeCharTokenizer, WordpieceTokenizer
 from .utils import CharNGram, FastText, GloVe, JiebaMode, NormalizeForm, SentencePieceModel, SentencePieceVocab, \
     SPieceTokenizerLoadType, SPieceTokenizerOutType, Vectors, Vocab, to_bytes, to_str