mindspore 2.0.0rc1__cp38-cp38-manylinux1_x86_64.whl → 2.2.0__cp38-cp38-manylinux1_x86_64.whl

mindspore/dataset/engine/iterators.py

@@ -76,11 +76,18 @@ class Iterator:
 
         self._runtime_context = cde.PythonRuntimeContext()
         self._runtime_context.Init()
+        if dataset.get_init_step() == 0:
+            init_step = 0
+            dataset_size = -1
+        else:
+            init_step = dataset.get_init_step()
+            dataset_size = dataset.get_dataset_size()
         if get_debug_mode():
             consumer = cde.PythonPullBasedIteratorConsumer(num_epochs)
+            consumer.Init(self.ir_tree)
         else:
             consumer = cde.PythonIteratorConsumer(num_epochs)
-        consumer.Init(self.ir_tree)
+            consumer.Init(self.ir_tree, init_step, dataset_size)
         self._runtime_context.AssignConsumer(consumer)
         self._iterator = self._runtime_context.GetConsumer()
         self._output_numpy = output_numpy
@@ -173,15 +180,15 @@ class Iterator:
             self._col_names = self.__ori_dataset.get_col_names()
         return self._col_names
 
-    def _reset(self, step, epoch):
+    def _reset(self, step, dataset_size):
         """
         Reset the iterator to the given step number and epoch number.
 
         Args:
             step (int): Global step number
-            epoch (int): Global epoch number
+            dataset_size (int): The number of steps that one epoch has.
         """
-        self._iterator.Reset(step, epoch)
+        self._iterator.Reset(step, dataset_size)
 
     def __convert_python(self, obj, to_numpy):
         """

mindspore/dataset/engine/obs/obs_mindrecord_dataset.py

@@ -24,6 +24,7 @@ from multiprocessing.managers import SyncManager
 import os
 import queue
 import random
+import stat
 import sys
 import time
 
@@ -115,6 +116,9 @@ def _wait_remove_datset(num_shards, shard_id, epoch_num):
     with open(sync_file, 'w') as f:
         f.write('ok')
 
+    if os.path.exists(sync_file):
+        os.chmod(sync_file, stat.S_IRUSR | stat.S_IWUSR)
+
     while True:
         if os.path.exists(sync_dir) and not os.listdir(sync_dir):
             break
@@ -170,7 +174,7 @@ def _download_work(shard_id, current_idx, local_path, cache, q):
         used_disk = get_used_disk_per()
         while used_disk > float(config.DISK_THRESHOLD):
             logger.info("[{} FUNCTION] Used disk space is {}%, and the disk threshold is {}%.".format(
-                sys._getframe().f_code.co_name, used_disk*100,  # pylint: disable=W0212
+                sys._getframe().f_code.co_name, used_disk * 100,  # pylint: disable=W0212
                 float(config.DISK_THRESHOLD)*100))
             retry_cnt = 0
             has_deleted = _delete_candidate_datasets(
@@ -267,7 +271,7 @@ def sync_wait_for_dataset(rank_id, rank_size, current_epoch):
     Wait util the dataset files required by all devices are downloaded.
 
     Note:
-        It should be used together with `mindspore.dataset.OBSMindDataset` and
+        It should be used together with :class:`mindspore.dataset.OBSMindDataset` and
         be called before each epoch.
 
     Args:
@@ -489,7 +493,7 @@ class MindRecordFromOBS:
         return len(self)
 
     def get_col_names(self):
-        """ Get column names of Mindrecord format dataset."""
+        """ Get column names of MindRecord format dataset."""
 
         from ..datasets_standard_format import MindDataset
 

mindspore/dataset/engine/obs/util.py

@@ -19,6 +19,7 @@ This dataset module provides internal utility function for OBSMindDataset API.
 import fcntl
 import os
 import shutil
+import stat
 import sys
 import sqlite3
 import time
@@ -272,6 +273,8 @@ def exclusive_lock(func):
                     time.sleep(config.RETRY_DELTA_TIME)
                 finally:
                     fcntl.flock(fd, fcntl.LOCK_UN)
+        if os.path.exists(lock_file):
+            os.chmod(lock_file, stat.S_IRUSR | stat.S_IWUSR)
         return result
     return wrapped_func
 

mindspore/dataset/engine/offload.py

@@ -180,7 +180,7 @@ class RandomHorizontalFlip(nn.Cell):
 
         flip_rand_factor = Tensor(np.random.uniform(size=(bs, 1)), dtype=mstype.float32)
         flip_rand_factor = self.cast((self.prob > flip_rand_factor), mstype.float32)
-        flip_rand_factor = self.reshape(C.repeat_elements(flip_rand_factor, rep=(h*w*c)), (bs, h, w, c))
+        flip_rand_factor = self.reshape(C.repeat_elements(flip_rand_factor, rep=(h * w * c)), (bs, h, w, c))
 
         x_flip = self.h_flip(x)
         operation = self.mul(x_flip, flip_rand_factor) + self.mul((1 - flip_rand_factor), x)
@@ -214,7 +214,7 @@ class RandomVerticalFlip(nn.Cell):
 
         flip_rand_factor = Tensor(np.random.uniform(size=(bs, 1)), dtype=mstype.float32)
         flip_rand_factor = self.cast((self.prob > flip_rand_factor), mstype.float32)
-        flip_rand_factor = self.reshape(C.repeat_elements(flip_rand_factor, rep=(h*w*c)), (bs, h, w, c))
+        flip_rand_factor = self.reshape(C.repeat_elements(flip_rand_factor, rep=(h * w * c)), (bs, h, w, c))
 
         x_flip = self.h_flip(x)
         operation = self.mul(x_flip, flip_rand_factor) + self.mul((1 - flip_rand_factor), x)
@@ -241,7 +241,7 @@ class GenerateRandBatch(nn.Cell):
         rand_factor = degree_min + (degree_max - degree_min)*rand_factor
         degree_factor = degree_min * self.ones((bs, 1), mstype.float32)
         rand_factor = (check_rand * degree_factor) + (~check_rand * rand_factor)
-        rand_factor = self.reshape(C.repeat_elements(rand_factor, rep=(h*w*c)), (bs, h, w, c))
+        rand_factor = self.reshape(C.repeat_elements(rand_factor, rep=(h * w * c)), (bs, h, w, c))
 
         return rand_factor
 
@@ -306,7 +306,7 @@ class RandomColorAdjust(nn.Cell):
 
         x_gray = 0.2989 * r_ + 0.587 * g_ + 0.114 * b_
         x_gray_mean = self.expand_dims(self.mean(x_gray, (1, 2)) + 0.5, -1)
-        x_gray_mean = self.reshape(C.repeat_elements(x_gray_mean, rep=(h*w*c)), (bs, h, w, c))
+        x_gray_mean = self.reshape(C.repeat_elements(x_gray_mean, rep=(h * w * c)), (bs, h, w, c))
         x_gray = C.repeat_elements(self.expand_dims(x_gray, -1), rep=c, axis=-1)
 
         # Apply brightness
@@ -341,7 +341,7 @@ class RandomColorAdjust(nn.Cell):
         hue_rand_factor = self.hue_min + (self.hue_max - self.hue_min)*hue_rand_factor
         degree_factor = self.hue_min * self.ones((bs, 1), mstype.float32)
         hue_rand_factor = (self.check_rand_hue * degree_factor) + (~self.check_rand_hue * hue_rand_factor)
-        hue_rand_factor = self.reshape(C.repeat_elements(hue_rand_factor, rep=(h*w)), (bs, h, w))
+        hue_rand_factor = self.reshape(C.repeat_elements(hue_rand_factor, rep=(h * w)), (bs, h, w))
         hue = hue + (hue_rand_factor * 360.0)
 
         # Convert tensor from hsv to rgb
@@ -412,7 +412,7 @@ class RandomSharpness(nn.Cell):
         degree_rand_factor = self.degree_min + (self.degree_max - self.degree_min)*degree_rand_factor
         degree_factor = self.degree_min * self.ones((bs, 1), mstype.float32)
         degree_rand_factor = (self.check_rand * degree_factor) + (~self.check_rand * degree_rand_factor)
-        degree_rand_factor = self.reshape(C.repeat_elements(degree_rand_factor, rep=(h*w*c)), (bs, h, w, c))
+        degree_rand_factor = self.reshape(C.repeat_elements(degree_rand_factor, rep=(h * w * c)), (bs, h, w, c))
 
         x_sharp = self.filter(self.transpose(x, (0, 3, 1, 2)), self.weight)
         x_sharp = self.transpose(x_sharp, (0, 2, 3, 1))

mindspore/dataset/engine/queue.py

@@ -18,11 +18,13 @@ between multiple processes in Python.  It has same API as multiprocessing.queue
 but it will pass large data through shared memory.
 """
 
-import multiprocessing.queues
+import errno
 import multiprocessing
-import types
 import queue
+import types
+
 import numpy as np
+
 from mindspore import log as logger
 from ..transforms.py_transforms_util import ExceptionHandler
 
@@ -45,7 +47,7 @@ class _SharedQueue(multiprocessing.queues.Queue):
 
         # change max_rowsize in MB into bytes
         self.seg_size = max_rowsize * 1024 * 1024
-        ##pipe can hold up to 65,636 bytes at a time
+        # pipe can hold up to 65,636 bytes at a time
         # there is less benefit for small data. To small data it can be slower as we need to pass 100 bytes of metadata
         # and then access the shared memory.
         self.min_shared_mem = 10000
@@ -59,19 +61,16 @@ class _SharedQueue(multiprocessing.queues.Queue):
         self.count = count
         self.print_error = True
 
-        try:
-            for _ in range(self.num_seg):
+        for _ in range(self.num_seg):
+            try:
                 a = multiprocessing.Array("b", self.seg_size)
+            except OSError as e:
+                if e.errno == errno.ENOMEM:
+                    raise RuntimeError("Failed to allocate shared memory for {0} elements of {1}MB: {2}"
+                                       .format(self.num_seg, self.seg_size / 1024 / 1024, e))
+                raise
+            else:
                 self.shm_list.append(a)
-        except Exception:
-            raise RuntimeError(
-                "_SharedQueue: Error allocating "
-                + str(self.seg_size / 1024 / 1024)
-                + "MB, "
-                + str(self.num_seg)
-                + " elements."
-                + " This might be caused by insufficient shm, and the recommended shm size is at least 5 GB."
-            )
 
     def put_until(self, data, timeout=None, exit_signal=None):
         """Put data into the queue. Block until timeout is reached or exit_signal is set."""
@@ -187,6 +186,7 @@ class _SharedQueue(multiprocessing.queues.Queue):
         shm_list_len = len(self.shm_list)
         for idx in range(shm_list_len):
             del self.shm_list[shm_list_len - idx - 1]
+        self.shm_list.clear()
         del self.shm_list
 
         self.close()
@@ -195,6 +195,7 @@ class _SharedQueue(multiprocessing.queues.Queue):
 
 class _Queue(multiprocessing.queues.Queue):
     """Specialized multiprocessing Queue that supports interrupted operations."""
+
     def __init__(self, size):
         super().__init__(size, ctx=multiprocessing.get_context())
 

mindspore/dataset/engine/samplers.py

@@ -110,10 +110,15 @@ class BuiltinSampler:
 
         Args:
             sampler (Sampler): Object used to choose samples from the dataset. Only builtin
-                samplers(DistributedSampler, PKSampler, RandomSampler, SequentialSampler,
-                SubsetRandomSampler, WeightedRandomSampler) are supported.
+                samplers(:class:`mindspore.dataset.DistributedSampler` ,
+                :class:`mindspore.dataset.PKSampler`,
+                :class:`mindspore.dataset.RandomSampler`,
+                :class:`mindspore.dataset.SequentialSampler`,
+                :class:`mindspore.dataset.SubsetRandomSampler`,
+                :class:`mindspore.dataset.WeightedRandomSampler` ) are supported.
 
         Examples:
+            >>> import mindspore.dataset as ds
             >>> sampler = ds.SequentialSampler(start_index=0, num_samples=3)
             >>> sampler.add_child(ds.RandomSampler(num_samples=4))
             >>> dataset = ds.Cifar10Dataset(cifar10_dataset_dir, sampler=sampler)
@@ -130,6 +135,7 @@ class BuiltinSampler:
             Sampler, The child sampler of given sampler.
 
         Examples:
+            >>> import mindspore.dataset as ds
             >>> sampler = ds.SequentialSampler(start_index=0, num_samples=3)
             >>> sampler.add_child(ds.RandomSampler(num_samples=2))
             >>> child_sampler = sampler.get_child()
@@ -160,8 +166,8 @@ class BuiltinSampler:
 
     def get_num_samples(self):
         """
-        Get num_samples value of the current sampler instance.
-        This parameter can be optionally passed in when defining the Sampler. Default: None.
+        Get `num_samples` value of the current sampler instance.
+        This parameter can be optionally passed in when defining the Sampler. Default: ``None``.
         This method will return the num_samples value.
         If the current sampler has child samplers,
         it will continue to access the child samplers and process the obtained value according to certain rules.
@@ -205,6 +211,7 @@ class BuiltinSampler:
             int, the number of samples, or None.
 
         Examples:
+            >>> import mindspore.dataset as ds
             >>> sampler = ds.SequentialSampler(start_index=0, num_samples=3)
             >>> num_samplers = sampler.get_num_samples()
         """
@@ -232,6 +239,7 @@ class Sampler(BuiltinSampler):
     dataset_size and num_samples will be set by dataset once a dataset iterator is created.
 
     Examples:
+        >>> import mindspore.dataset as ds
         >>> class ReverseSampler(ds.Sampler):
         ...     def __iter__(self):
         ...         for i in range(self.dataset_size - 1, -1, -1):
@@ -329,12 +337,13 @@ class DistributedSampler(BuiltinSampler):
     Args:
         num_shards (int): Number of shards to divide the dataset into.
         shard_id (int): Shard ID of the current shard, which should within the range of [0, `num_shards` - 1].
-        shuffle (bool, optional): If True, the indices are shuffled, otherwise it will not be shuffled. Default: True.
-        num_samples (int, optional): The number of samples to draw. Default: None, which means sample all elements.
+        shuffle (bool, optional): If True, the indices are shuffled, otherwise it will not be shuffled.
+            Default: ``True``.
+        num_samples (int, optional): The number of samples to draw. Default: ``None``, which means sample all elements.
         offset(int, optional): The starting shard ID where the elements in the dataset are sent to, which
             should be no more than `num_shards` . This parameter is only valid when a ConcatDataset takes
-            a DistributedSampler as its sampler. It will affect the number of samples of per shard.
-            Default: -1, which means each shard has the same number of samples.
+            a :class:`mindspore.dataset.DistributedSampler` as its sampler. It will affect the number of
+            samples of per shard. Default: ``-1``, which means each shard has the same number of samples.
 
     Raises:
         TypeError: If `num_shards` is not of type int.
@@ -348,6 +357,7 @@ class DistributedSampler(BuiltinSampler):
         RuntimeError: If `offset` is greater than `num_shards` .
 
     Examples:
+        >>> import mindspore.dataset as ds
         >>> # creates a distributed sampler with 10 shards in total. This shard is shard 5.
         >>> sampler = ds.DistributedSampler(10, 5)
         >>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir,
@@ -432,22 +442,22 @@ class PKSampler(BuiltinSampler):
 
     Args:
         num_val (int): Number of elements to sample for each class.
-        num_class (int, optional): Number of classes to sample. Default: None, sample all classes.
+        num_class (int, optional): Number of classes to sample. Default: ``None`` , sample all classes.
             The parameter does not support to specify currently.
-        shuffle (bool, optional): If True, the class IDs are shuffled, otherwise it will not be
-            shuffled. Default: False.
-        class_column (str, optional): Name of column with class labels for MindDataset. Default: 'label'.
-        num_samples (int, optional): The number of samples to draw. Default: None, which means sample all elements.
+        shuffle (bool, optional): Whether to shuffle the class IDs. Default: ``False``.
+        class_column (str, optional): Name of column with class labels for MindDataset. Default: ``'label'``.
+        num_samples (int, optional): The number of samples to draw. Default: ``None`` , which means sample all elements.
 
     Raises:
         TypeError: If `shuffle` is not of type bool.
         TypeError: If `class_column` is not of type str.
         TypeError: If `num_samples` is not of type int.
-        NotImplementedError: If `num_class` is not None.
+        NotImplementedError: If `num_class` is not ``None``.
         RuntimeError: If `num_val` is not a positive value.
         ValueError: If `num_samples` is a negative value.
 
     Examples:
+        >>> import mindspore.dataset as ds
         >>> # creates a PKSampler that will get 3 samples from every class.
         >>> sampler = ds.PKSampler(3)
         >>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir,
@@ -519,8 +529,8 @@ class RandomSampler(BuiltinSampler):
     Samples the elements randomly.
 
     Args:
-        replacement (bool, optional): If True, put the sample ID back for the next draw. Default: False.
-        num_samples (int, optional): Number of elements to sample. Default: None, which means sample all elements.
+        replacement (bool, optional): If True, put the sample ID back for the next draw. Default: ``False``.
+        num_samples (int, optional): Number of elements to sample. Default: ``None`` , which means sample all elements.
 
     Raises:
         TypeError: If `replacement` is not of type bool.
@@ -528,6 +538,7 @@ class RandomSampler(BuiltinSampler):
         ValueError: If `num_samples` is a negative value.
 
     Examples:
+        >>> import mindspore.dataset as ds
         >>> # creates a RandomSampler
         >>> sampler = ds.RandomSampler()
         >>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir,
@@ -584,8 +595,8 @@ class SequentialSampler(BuiltinSampler):
     Samples the dataset elements sequentially that is equivalent to not using a sampler.
 
     Args:
-        start_index (int, optional): Index to start sampling at. Default: None, start at first ID.
-        num_samples (int, optional): Number of elements to sample. Default: None, which means sample all elements.
+        start_index (int, optional): Index to start sampling at. Default: ``None`` , start at first ID.
+        num_samples (int, optional): Number of elements to sample. Default: ``None`` , which means sample all elements.
 
     Raises:
         TypeError: If `start_index` is not of type int.
@@ -594,6 +605,7 @@ class SequentialSampler(BuiltinSampler):
         ValueError: If `num_samples` is a negative value.
 
     Examples:
+        >>> import mindspore.dataset as ds
         >>> # creates a SequentialSampler
         >>> sampler = ds.SequentialSampler()
         >>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir,
@@ -653,7 +665,7 @@ class SubsetSampler(BuiltinSampler):
 
     Args:
         indices (Iterable): A sequence of indices (Any iterable Python object but string).
-        num_samples (int, optional): Number of elements to sample. Default: None, which means sample all elements.
+        num_samples (int, optional): Number of elements to sample. Default: ``None`` , which means sample all elements.
 
     Raises:
         TypeError: If elements of `indices` are not of type number.
@@ -661,6 +673,7 @@ class SubsetSampler(BuiltinSampler):
         ValueError: If `num_samples` is a negative value.
 
     Examples:
+        >>> import mindspore.dataset as ds
         >>> indices = [0, 1, 2, 3, 4, 5]
         >>>
         >>> # creates a SubsetSampler, will sample from the provided indices
@@ -741,7 +754,7 @@ class SubsetRandomSampler(SubsetSampler):
 
     Args:
         indices (Iterable): A sequence of indices (Any iterable Python object but string).
-        num_samples (int, optional): Number of elements to sample. Default: None, which means sample all elements.
+        num_samples (int, optional): Number of elements to sample. Default: ``None`` , which means sample all elements.
 
     Raises:
         TypeError: If elements of `indices` are not of type number.
@@ -749,6 +762,7 @@ class SubsetRandomSampler(SubsetSampler):
         ValueError: If `num_samples` is a negative value.
 
     Examples:
+        >>> import mindspore.dataset as ds
         >>> indices = [0, 1, 2, 3, 7, 88, 119]
         >>>
         >>> # create a SubsetRandomSampler, will sample from the provided indices
@@ -786,9 +800,10 @@ class IterSampler(Sampler):
 
     Args:
         sampler (iterable object): an user defined iterable object.
-        num_samples (int, optional): Number of elements to sample. Default: None, which means sample all elements.
+        num_samples (int, optional): Number of elements to sample. Default: ``None`` , which means sample all elements.
 
     Examples:
+        >>> import mindspore.dataset as ds
         >>> class MySampler:
         ...     def __iter__(self):
         ...         for i in range(99, -1, -1):
@@ -817,8 +832,8 @@ class WeightedRandomSampler(BuiltinSampler):
 
     Args:
         weights (list[float, int]): A sequence of weights, not necessarily summing up to 1.
-        num_samples (int, optional): Number of elements to sample. Default: None, which means sample all elements.
-        replacement (bool): If True, put the sample ID back for the next draw. Default: True.
+        num_samples (int, optional): Number of elements to sample. Default: ``None`` , which means sample all elements.
+        replacement (bool): If ``True``, put the sample ID back for the next draw. Default: ``True``.
 
     Raises:
         TypeError: If elements of `weights` are not of type number.
@@ -828,6 +843,7 @@ class WeightedRandomSampler(BuiltinSampler):
         ValueError: If `num_samples` is a negative value.
 
     Examples:
+        >>> import mindspore.dataset as ds
         >>> weights = [0.9, 0.01, 0.4, 0.8, 0.1, 0.1, 0.3]
         >>>
         >>> # creates a WeightedRandomSampler that will sample 4 elements without replacement

mindspore/dataset/engine/serializer_deserializer.py

@@ -38,7 +38,7 @@ def serialize(dataset, json_filepath=""):
 
     Args:
         dataset (Dataset): The starting node.
-        json_filepath (str): The filepath where a serialized JSON file will be generated. Default: ''.
+        json_filepath (str): The filepath where a serialized JSON file will be generated. Default: ``''``.
 
     Returns:
        Dict, the dictionary contains the serialized dataset graph.
@@ -47,6 +47,10 @@ def serialize(dataset, json_filepath=""):
         OSError: Cannot open a file.
 
     Examples:
+        >>> import mindspore.dataset as ds
+        >>> import mindspore.dataset.transforms as transforms
+        >>>
+        >>> mnist_dataset_dir = "/path/to/mnist_dataset_directory"
         >>> dataset = ds.MnistDataset(mnist_dataset_dir, num_samples=100)
         >>> one_hot_encode = transforms.OneHot(10)  # num_classes is input argument
         >>> dataset = dataset.map(operations=one_hot_encode, input_columns="label")
@@ -62,9 +66,9 @@ def deserialize(input_dict=None, json_filepath=None):
     Construct dataset pipeline from a JSON file produced by dataset serialize function.
 
     Args:
-        input_dict (dict): A Python dictionary containing a serialized dataset graph. Default: None.
+        input_dict (dict): A Python dictionary containing a serialized dataset graph. Default: ``None``.
         json_filepath (str): A path to the JSON file containing dataset graph.
-            User can obtain this file by calling API `mindspore.dataset.serialize()` . Default: None.
+            User can obtain this file by calling API `mindspore.dataset.serialize()` . Default: ``None``.
 
     Returns:
         de.Dataset or None if error occurs.
@@ -73,13 +77,19 @@ def deserialize(input_dict=None, json_filepath=None):
         OSError: Can not open the JSON file.
 
     Examples:
+        >>> import mindspore.dataset as ds
+        >>> import mindspore.dataset.transforms as transforms
+        >>>
+        >>> mnist_dataset_dir = "/path/to/mnist_dataset_directory"
         >>> dataset = ds.MnistDataset(mnist_dataset_dir, num_samples=100)
         >>> one_hot_encode = transforms.OneHot(10)  # num_classes is input argument
         >>> dataset = dataset.map(operations=one_hot_encode, input_columns="label")
         >>> dataset = dataset.batch(batch_size=10, drop_remainder=True)
+        >>>
         >>> # Case 1: to/from JSON file
         >>> serialized_data = ds.serialize(dataset, json_filepath="/path/to/mnist_dataset_pipeline.json")
         >>> deserialized_dataset = ds.deserialize(json_filepath="/path/to/mnist_dataset_pipeline.json")
+        >>>
         >>> # Case 2: to/from Python dictionary
         >>> serialized_data = ds.serialize(dataset)
         >>> deserialized_dataset = ds.deserialize(input_dict=serialized_data)
@@ -109,9 +119,13 @@ def show(dataset, indentation=2):
     Args:
         dataset (Dataset): The starting node.
         indentation (int, optional): The indentation used by the JSON print.
-            Do not indent if indentation is None. Default: 2.
+            Do not indent if indentation is None. Default: ``2``, indent 2 space.
 
     Examples:
+        >>> import mindspore.dataset as ds
+        >>> import mindspore.dataset.transforms as transforms
+        >>>
+        >>> mnist_dataset_dir = "/path/to/mnist_dataset_directory"
         >>> dataset = ds.MnistDataset(mnist_dataset_dir, num_samples=100)
         >>> one_hot_encode = transforms.OneHot(10)
         >>> dataset = dataset.map(operations=one_hot_encode, input_columns="label")
@@ -135,8 +149,10 @@ def compare(pipeline1, pipeline2):
         Whether pipeline1 is equal to pipeline2.
 
     Examples:
-        >>> pipeline1 = ds.MnistDataset(mnist_dataset_dir, num_samples=100)
-        >>> pipeline2 = ds.Cifar10Dataset(cifar10_dataset_dir, num_samples=100)
+        >>> import mindspore.dataset as ds
+        >>>
+        >>> pipeline1 = ds.MnistDataset("/path/to/mnist_dataset_directory", num_samples=100)
+        >>> pipeline2 = ds.Cifar10Dataset("/path/to/cifar10_dataset_directory", num_samples=100)
         >>> res = ds.compare(pipeline1, pipeline2)
     """