mindspore 1.10.0__cp37-none-any.whl → 2.0.0rc1__cp37-none-any.whl

mindspore/dataset/engine/offload.py

@@ -19,10 +19,11 @@ import numpy as np
 
 import mindspore.common.dtype as mstype
 from mindspore.common.tensor import Tensor
+import mindspore.ops as ops
 import mindspore.nn as nn
 import mindspore.ops.composite as C
 from mindspore.ops import operations as P
-from mindspore.ops.primitive import constexpr
+from mindspore.ops.primitive import _primexpr
 from mindspore import log as logger
 
 
@@ -87,7 +88,7 @@ def apply_offload_iterators(data, offload_model):
             data[i] = Tensor(data[i], dtype=mstype.float32)
             non_tensor_idxs.append(i)
 
-    data = offload_model(data)
+    data = offload_model(*data)
     data = list(data)
     for idx in non_tensor_idxs:
         data[idx] = data[idx].asnumpy()
@@ -95,7 +96,7 @@ def apply_offload_iterators(data, offload_model):
     return data
 
 
-@constexpr
+@_primexpr
 def check_input_dims(x_shape, required_dim, offload_op_name):
     """
     Check if input has the required number of dimensions for the operation.
@@ -135,7 +136,7 @@ class ApplyPreTransform(nn.Cell):
         for data_col in x:
             data.append(data_col)
 
-        data = self.transform(data)
+        data = self.transform(*data)
         data = self.model(*data)
 
         return data
@@ -182,9 +183,10 @@ class RandomHorizontalFlip(nn.Cell):
         flip_rand_factor = self.reshape(C.repeat_elements(flip_rand_factor, rep=(h*w*c)), (bs, h, w, c))
 
         x_flip = self.h_flip(x)
-        x = self.mul(x_flip, flip_rand_factor) + self.mul((1 - flip_rand_factor), x)
-
-        return x
+        operation = self.mul(x_flip, flip_rand_factor) + self.mul((1 - flip_rand_factor), x)
+        # ops.depend is added to find the RandomHorizontalFlip operator in IR files.
+        depend = ops.depend(operation, "RandomHorizontalFlip")
+        return depend
 
 
 class RandomVerticalFlip(nn.Cell):
@@ -215,9 +217,10 @@ class RandomVerticalFlip(nn.Cell):
         flip_rand_factor = self.reshape(C.repeat_elements(flip_rand_factor, rep=(h*w*c)), (bs, h, w, c))
 
         x_flip = self.h_flip(x)
-        x = self.mul(x_flip, flip_rand_factor) + self.mul((1 - flip_rand_factor), x)
-
-        return x
+        operation = self.mul(x_flip, flip_rand_factor) + self.mul((1 - flip_rand_factor), x)
+        # ops.depend is added to find the RandomVerticalFlip operator in IR files.
+        depend = ops.depend(operation, "RandomVerticalFlip")
+        return depend
 
 
 class GenerateRandBatch(nn.Cell):
@@ -308,15 +311,15 @@ class RandomColorAdjust(nn.Cell):
 
         # Apply brightness
         x = self.mul(x, br_rand_factor)
-        x = C.clip_by_value(x, 0.0, 255.0)
+        x = ops.clip_by_value(x, 0.0, 255.0)
 
         # Apply contrast
         x = self.mul(x, cont_rand_factor) + self.mul((1 - cont_rand_factor), x_gray_mean)
-        x = C.clip_by_value(x, 0.0, 255.0)
+        x = ops.clip_by_value(x, 0.0, 255.0)
 
         # Apply saturation
         x = self.mul(x, sat_rand_factor) + self.mul((1 - sat_rand_factor), x_gray)
-        x = C.clip_by_value(x, 0.0, 255.0)
+        x = ops.clip_by_value(x, 0.0, 255.0)
 
         # Apply Hue Transform
         # Convert tensor from rgb to hsv
@@ -361,9 +364,10 @@ class RandomColorAdjust(nn.Cell):
         x_rgb = x_rgb + C.repeat_elements(self.expand_dims((v - c), 1), 3, 1)
 
         x_rgb = self.transpose(x, (0, 2, 3, 1)) * 255.0
-        x_rgb = C.clip_by_value(x, 0.0, 255.0)
-
-        return x_rgb
+        operation = ops.clip_by_value(x, 0.0, 255.0)
+        # ops.depend is added to find the RandomColorAdjust operator in IR files.
+        depend = ops.depend(operation, "RandomColorAdjust")
+        return depend
 
 
 class RandomSharpness(nn.Cell):
@@ -414,8 +418,10 @@ class RandomSharpness(nn.Cell):
         x_sharp = self.transpose(x_sharp, (0, 2, 3, 1))
 
         x = self.mul(x, degree_rand_factor) + self.mul((1 - degree_rand_factor), x_sharp)
-        x = C.clip_by_value(x, 0.0, 255.0)
-        return x
+        operation = ops.clip_by_value(x, 0.0, 255.0)
+        # ops.depend is added to find the RandomSharpness operator in IR files.
+        depend = ops.depend(operation, "RandomSharpness")
+        return depend
 
 
 class Rescale(nn.Cell):
@@ -435,9 +441,10 @@ class Rescale(nn.Cell):
     def construct(self, x):
 
         x = self.cast(x, mstype.float32)
-        x = x * self.rescale + self.shift
-
-        return x
+        operation = x * self.rescale + self.shift
+        # ops.depend is added to find the Rescale operator in IR files.
+        depend = ops.depend(operation, "Rescale")
+        return depend
 
 
 class HwcToChw(nn.Cell):
@@ -453,7 +460,10 @@ class HwcToChw(nn.Cell):
     def construct(self, x):
         x_shape = self.shape(x)
         check_input_dims(x_shape, 4, 'HwcToChw')
-        return self.trans(x, (0, 3, 1, 2))
+        operation = self.trans(x, (0, 3, 1, 2))
+        # ops.depend is added to find the HwcToChw operator in IR files.
+        depend = ops.depend(operation, "HwcToChw")
+        return depend
 
 
 class Normalize(nn.Cell):
@@ -461,8 +471,11 @@ class Normalize(nn.Cell):
     Applies Normalize transform on given input tensors.
     """
 
-    def __init__(self, mean, std):
+    def __init__(self, mean, std, is_hwc=True):
         super(Normalize, self).__init__()
+        if is_hwc is False:
+            mean = np.expand_dims(np.array(mean), (1, 2))
+            std = np.expand_dims(np.array(std), (1, 2))
         self.mean = Tensor(mean, mstype.float32)
         self.std = Tensor(std, mstype.float32)
         self.sub = P.Sub()
@@ -472,8 +485,10 @@ class Normalize(nn.Cell):
     def construct(self, x):
         x = self.cast(x, mstype.float32)
         x = self.sub(x, self.mean)
-        x = self.div(x, self.std)
-        return x
+        operation = self.div(x, self.std)
+        # ops.depend is added to find the Normalize operator in IR files.
+        depend = ops.depend(operation, "Normalize")
+        return depend
 
 
 class TypeCast(nn.Cell):
@@ -488,8 +503,10 @@ class TypeCast(nn.Cell):
         self.data_type = mstype.typing.str_to_type(data_type_str)
 
     def construct(self, x):
-
-        return self.cast(x, self.data_type)
+        operation = self.cast(x, self.data_type)
+        # ops.depend is added to find the TypeCast operator in IR files.
+        depend = ops.depend(operation, "TypeCast")
+        return depend
 
 
 class OffloadModel():
@@ -502,7 +519,7 @@ class OffloadModel():
 op_to_model = {
     "HWC2CHW": OffloadModel(HwcToChw),
     "HwcToChw": OffloadModel(HwcToChw),
-    "Normalize": OffloadModel(Normalize, ["mean", "std"]),
+    "Normalize": OffloadModel(Normalize, ["mean", "std", "is_hwc"]),
     "RandomColorAdjust": OffloadModel(RandomColorAdjust, ["brightness", "contrast", "saturation", "hue"]),
     "RandomHorizontalFlip": OffloadModel(RandomHorizontalFlip, ["prob"]),
     "RandomSharpness": OffloadModel(RandomSharpness, ["degrees"]),
@@ -571,7 +588,11 @@ class GetOffloadModel(nn.Cell):
                     self.transform_list.append(GetModelFromJson2Col(node, ds_col_idxs))
             self.transform_list.reverse()
 
-    def construct(self, x):
+    def construct(self, *x):
+        data = []
+        for d in x:
+            data.append(d)
+
         for transform in self.transform_list:
-            x = transform(x)
-        return x
+            data = transform(data)
+        return data

mindspore/dataset/engine/samplers.py

@@ -161,7 +161,7 @@ class BuiltinSampler:
     def get_num_samples(self):
         """
         Get num_samples value of the current sampler instance.
-        This parameter can be optionally passed in when defining the Sampler (default is None).
+        This parameter can be optionally passed in when defining the Sampler. Default: None.
         This method will return the num_samples value.
         If the current sampler has child samplers,
         it will continue to access the child samplers and process the obtained value according to certain rules.
@@ -328,13 +328,13 @@ class DistributedSampler(BuiltinSampler):
 
     Args:
         num_shards (int): Number of shards to divide the dataset into.
-        shard_id (int): Shard ID of the current shard, which should within the range of [0, `num_shards`-1].
-        shuffle (bool, optional): If True, the indices are shuffled, otherwise it will not be shuffled(default=True).
-        num_samples (int, optional): The number of samples to draw (default=None, which means sample all elements).
+        shard_id (int): Shard ID of the current shard, which should within the range of [0, `num_shards` - 1].
+        shuffle (bool, optional): If True, the indices are shuffled, otherwise it will not be shuffled. Default: True.
+        num_samples (int, optional): The number of samples to draw. Default: None, which means sample all elements.
         offset(int, optional): The starting shard ID where the elements in the dataset are sent to, which
-            should be no more than `num_shards`. This parameter is only valid when a ConcatDataset takes
-            a DistributedSampler as its sampler. It will affect the number of samples of per shard
-            (default=-1, which means each shard has the same number of samples).
+            should be no more than `num_shards` . This parameter is only valid when a ConcatDataset takes
+            a DistributedSampler as its sampler. It will affect the number of samples of per shard.
+            Default: -1, which means each shard has the same number of samples.
 
     Raises:
         TypeError: If `num_shards` is not of type int.
@@ -344,8 +344,8 @@ class DistributedSampler(BuiltinSampler):
         TypeError: If `offset` is not of type int.
         ValueError: If `num_samples` is a negative value.
         RuntimeError: If `num_shards` is not a positive value.
-        RuntimeError: If `shard_id` is smaller than 0 or equal to `num_shards` or larger than `num_shards`.
-        RuntimeError: If `offset` is greater than `num_shards`.
+        RuntimeError: If `shard_id` is smaller than 0 or equal to `num_shards` or larger than `num_shards` .
+        RuntimeError: If `offset` is greater than `num_shards` .
 
     Examples:
         >>> # creates a distributed sampler with 10 shards in total. This shard is shard 5.
@@ -378,7 +378,10 @@ class DistributedSampler(BuiltinSampler):
         self.num_shards = num_shards
         self.shard_id = shard_id
         self.shuffle = shuffle
-        self.seed = 0
+        # get seed in distributed scenario
+        # Example 1. if user set seeds by ds.config.set_seed(4321), then seed 4321 is used
+        # Example 2. if user does not set the seed, then existing or default seed (like 5489) is used
+        self.seed = ds.config.get_seed()
         self.offset = offset
         super().__init__(num_samples)
 
@@ -429,12 +432,12 @@ class PKSampler(BuiltinSampler):
 
     Args:
         num_val (int): Number of elements to sample for each class.
-        num_class (int, optional): Number of classes to sample (default=None, sample all classes).
+        num_class (int, optional): Number of classes to sample. Default: None, sample all classes.
             The parameter does not support to specify currently.
         shuffle (bool, optional): If True, the class IDs are shuffled, otherwise it will not be
-            shuffled (default=False).
-        class_column (str, optional): Name of column with class labels for MindDataset (default='label').
-        num_samples (int, optional): The number of samples to draw (default=None, which means sample all elements).
+            shuffled. Default: False.
+        class_column (str, optional): Name of column with class labels for MindDataset. Default: 'label'.
+        num_samples (int, optional): The number of samples to draw. Default: None, which means sample all elements.
 
     Raises:
         TypeError: If `shuffle` is not of type bool.
@@ -516,8 +519,8 @@ class RandomSampler(BuiltinSampler):
     Samples the elements randomly.
 
     Args:
-        replacement (bool, optional): If True, put the sample ID back for the next draw (default=False).
-        num_samples (int, optional): Number of elements to sample (default=None, which means sample all elements).
+        replacement (bool, optional): If True, put the sample ID back for the next draw. Default: False.
+        num_samples (int, optional): Number of elements to sample. Default: None, which means sample all elements.
 
     Raises:
         TypeError: If `replacement` is not of type bool.
@@ -581,8 +584,8 @@ class SequentialSampler(BuiltinSampler):
     Samples the dataset elements sequentially that is equivalent to not using a sampler.
 
     Args:
-        start_index (int, optional): Index to start sampling at. (default=None, start at first ID)
-        num_samples (int, optional): Number of elements to sample (default=None, which means sample all elements).
+        start_index (int, optional): Index to start sampling at. Default: None, start at first ID.
+        num_samples (int, optional): Number of elements to sample. Default: None, which means sample all elements.
 
     Raises:
         TypeError: If `start_index` is not of type int.
@@ -650,7 +653,7 @@ class SubsetSampler(BuiltinSampler):
 
     Args:
         indices (Iterable): A sequence of indices (Any iterable Python object but string).
-        num_samples (int, optional): Number of elements to sample (default=None, which means sample all elements).
+        num_samples (int, optional): Number of elements to sample. Default: None, which means sample all elements.
 
     Raises:
         TypeError: If elements of `indices` are not of type number.
@@ -738,7 +741,7 @@ class SubsetRandomSampler(SubsetSampler):
 
     Args:
         indices (Iterable): A sequence of indices (Any iterable Python object but string).
-        num_samples (int, optional): Number of elements to sample (default=None, which means sample all elements).
+        num_samples (int, optional): Number of elements to sample. Default: None, which means sample all elements.
 
     Raises:
         TypeError: If elements of `indices` are not of type number.
@@ -778,12 +781,12 @@ class IterSampler(Sampler):
     User provided an iterable object without inheriting from our Sampler class.
 
     Note:
-        This class exists to allow handshake logic between dataset operators and user defined samplers.
+        This class exists to allow handshake logic between dataset operations and user defined samplers.
         By constructing this object we avoid the user having to inherit from our Sampler class.
 
     Args:
         sampler (iterable object): an user defined iterable object.
-        num_samples (int, optional): Number of elements to sample (default=None, which means sample all elements).
+        num_samples (int, optional): Number of elements to sample. Default: None, which means sample all elements.
 
     Examples:
         >>> class MySampler:
@@ -814,8 +817,8 @@ class WeightedRandomSampler(BuiltinSampler):
 
     Args:
         weights (list[float, int]): A sequence of weights, not necessarily summing up to 1.
-        num_samples (int, optional): Number of elements to sample (default=None, which means sample all elements).
-        replacement (bool): If True, put the sample ID back for the next draw (default=True).
+        num_samples (int, optional): Number of elements to sample. Default: None, which means sample all elements.
+        replacement (bool): If True, put the sample ID back for the next draw. Default: True.
 
     Raises:
         TypeError: If elements of `weights` are not of type number.

mindspore/dataset/engine/serializer_deserializer.py

@@ -27,25 +27,24 @@ def serialize(dataset, json_filepath=""):
     Serialize dataset pipeline into a JSON file.
 
     Note:
-        Currently some Python objects are not supported to be serialized.
-        Some examples of unsupported objects are callable user-defined Python functions (Python UDFs) and
-        GeneratorDataset.
-        For such unsupported objects, partially serialized JSON output is produced
-        for which later deserialization, pipeline execution of the deserialized JSON file
-        and/or re-serialization of the deserialized pipeline may result in an error.
-        For example, serialization of callable user-defined Python functions (Python UDFs) is not supported,
-        and a warning results on serialization. Any produced serialized JSON file output for this dataset pipeline
-        is not valid to be deserialized.
+        Complete serialization of Python objects is not currently supported.
+        Scenarios that are not supported include data pipelines that use `GeneratorDataset`
+        or `map` or `batch` operations that contain custom Python functions.
+        For Python objects, serialization operations do not yield the full object content,
+        which means that deserialization of the JSON file obtained by serialization may result in errors.
+        For example, when serializing the data pipeline of Python user-defined functions,
+        a related warning message is reported and the obtained JSON file cannot be deserialized
+        into a usable data pipeline.
 
     Args:
         dataset (Dataset): The starting node.
-        json_filepath (str): The filepath where a serialized JSON file will be generated (default="").
+        json_filepath (str): The filepath where a serialized JSON file will be generated. Default: ''.
 
     Returns:
-       Dict, The dictionary contains the serialized dataset graph.
+       Dict, the dictionary contains the serialized dataset graph.
 
     Raises:
-        OSError: Cannot open a file
+        OSError: Cannot open a file.
 
     Examples:
         >>> dataset = ds.MnistDataset(mnist_dataset_dir, num_samples=100)
@@ -63,9 +62,9 @@ def deserialize(input_dict=None, json_filepath=None):
     Construct dataset pipeline from a JSON file produced by dataset serialize function.
 
     Args:
-        input_dict (dict): A Python dictionary containing a serialized dataset graph (default=None).
+        input_dict (dict): A Python dictionary containing a serialized dataset graph. Default: None.
         json_filepath (str): A path to the JSON file containing dataset graph.
-            User can obtain this file by calling API `mindspore.dataset.serialize()` (default=None).
+            User can obtain this file by calling API `mindspore.dataset.serialize()` . Default: None.
 
     Returns:
         de.Dataset or None if error occurs.
@@ -110,7 +109,7 @@ def show(dataset, indentation=2):
     Args:
         dataset (Dataset): The starting node.
         indentation (int, optional): The indentation used by the JSON print.
-            Do not indent if indentation is None (default=2).
+            Do not indent if indentation is None. Default: 2.
 
     Examples:
         >>> dataset = ds.MnistDataset(mnist_dataset_dir, num_samples=100)