mindspore 2.1.0__cp38-cp38-manylinux1_x86_64.whl → 2.2.0__cp38-cp38-manylinux1_x86_64.whl

mindspore/dataset/engine/iterators.py

@@ -76,11 +76,18 @@ class Iterator:
 
         self._runtime_context = cde.PythonRuntimeContext()
         self._runtime_context.Init()
+        if dataset.get_init_step() == 0:
+            init_step = 0
+            dataset_size = -1
+        else:
+            init_step = dataset.get_init_step()
+            dataset_size = dataset.get_dataset_size()
         if get_debug_mode():
             consumer = cde.PythonPullBasedIteratorConsumer(num_epochs)
+            consumer.Init(self.ir_tree)
         else:
             consumer = cde.PythonIteratorConsumer(num_epochs)
-        consumer.Init(self.ir_tree)
+            consumer.Init(self.ir_tree, init_step, dataset_size)
         self._runtime_context.AssignConsumer(consumer)
         self._iterator = self._runtime_context.GetConsumer()
         self._output_numpy = output_numpy
@@ -173,15 +180,15 @@ class Iterator:
             self._col_names = self.__ori_dataset.get_col_names()
         return self._col_names
 
-    def _reset(self, step, epoch):
+    def _reset(self, step, dataset_size):
         """
         Reset the iterator to the given step number and epoch number.
 
         Args:
             step (int): Global step number
-            epoch (int): Global epoch number
+            dataset_size (int): The number of steps that one epoch has.
         """
-        self._iterator.Reset(step, epoch)
+        self._iterator.Reset(step, dataset_size)
 
     def __convert_python(self, obj, to_numpy):
         """

mindspore/dataset/engine/obs/obs_mindrecord_dataset.py

@@ -24,6 +24,7 @@ from multiprocessing.managers import SyncManager
 import os
 import queue
 import random
+import stat
 import sys
 import time
 
@@ -115,6 +116,9 @@ def _wait_remove_datset(num_shards, shard_id, epoch_num):
     with open(sync_file, 'w') as f:
         f.write('ok')
 
+    if os.path.exists(sync_file):
+        os.chmod(sync_file, stat.S_IRUSR | stat.S_IWUSR)
+
     while True:
         if os.path.exists(sync_dir) and not os.listdir(sync_dir):
             break

mindspore/dataset/engine/obs/util.py

@@ -19,6 +19,7 @@ This dataset module provides internal utility function for OBSMindDataset API.
 import fcntl
 import os
 import shutil
+import stat
 import sys
 import sqlite3
 import time
@@ -272,6 +273,8 @@ def exclusive_lock(func):
                     time.sleep(config.RETRY_DELTA_TIME)
                 finally:
                     fcntl.flock(fd, fcntl.LOCK_UN)
+        if os.path.exists(lock_file):
+            os.chmod(lock_file, stat.S_IRUSR | stat.S_IWUSR)
         return result
     return wrapped_func
 

mindspore/dataset/engine/samplers.py

@@ -166,7 +166,7 @@ class BuiltinSampler:
 
     def get_num_samples(self):
         """
-        Get num_samples value of the current sampler instance.
+        Get `num_samples` value of the current sampler instance.
         This parameter can be optionally passed in when defining the Sampler. Default: ``None``.
         This method will return the num_samples value.
         If the current sampler has child samplers,

mindspore/dataset/engine/validators.py

@@ -1307,8 +1307,8 @@ def check_batch(method):
         if num_parallel_workers is not None:
             check_num_parallel_workers(num_parallel_workers)
         type_check(drop_remainder, (bool,), "drop_remainder")
-        type_check(max_rowsize, (int,), "max_rowsize")
-        check_pos_int32(max_rowsize, "max_rowsize")
+
+        check_max_rowsize(max_rowsize)
 
         if (input_columns is not None) and (per_batch_map is None):
             # input_columns must be None when per_batch_map is not set
@@ -1403,6 +1403,20 @@ def get_map_kwargs_from_dict(param_dict):
     return python_multiprocessing, max_rowsize, cache, callbacks, offload
 
 
+def check_max_rowsize(max_rowsize):
+    """check the max_rowsize"""
+    type_check(max_rowsize, (int, list), "max_rowsize")
+    if isinstance(max_rowsize, int):
+        type_check(max_rowsize, (int,), "max_rowsize")
+        check_pos_int32(max_rowsize, "max_rowsize")
+    elif isinstance(max_rowsize, list) and len(max_rowsize) == 2:
+        for index, value in enumerate(max_rowsize):
+            type_check(value, (int,), "max_rowsize[{}]".format(index))
+            check_pos_int32(value, "max_rowsizei[{}]".format(index))
+    else:
+        raise TypeError("max_rowsize should be a single integer or a list[in_rowsize, out_rowsize] of length 2.")
+
+
 def check_map(method):
     """check the input arguments of map."""
 
@@ -1458,8 +1472,7 @@ def check_map(method):
             check_num_parallel_workers(num_parallel_workers)
         type_check(python_multiprocessing, (bool,), "python_multiprocessing")
         check_cache_option(cache)
-        type_check(max_rowsize, (int,), "max_rowsize")
-        check_pos_int32(max_rowsize, "max_rowsize")
+        check_max_rowsize(max_rowsize)
         if offload is not None:
             type_check(offload, (bool,), "offload")
 
@@ -1571,9 +1584,10 @@ def check_device_send(method):
 
     @wraps(method)
     def new_method(self, *args, **kwargs):
-        [send_epoch_end, create_data_info_queue], _ = parse_user_args(method, *args, **kwargs)
+        [send_epoch_end, create_data_info_queue, queue_name], _ = parse_user_args(method, *args, **kwargs)
         type_check(send_epoch_end, (bool,), "send_epoch_end")
         type_check(create_data_info_queue, (bool,), "create_data_info_queue")
+        type_check(queue_name, (str,), "queue_name")
 
         return method(self, *args, **kwargs)
 

mindspore/dataset/text/__init__.py

@@ -25,7 +25,7 @@ Common imported modules in corresponding API examples are as follows:
     import mindspore.dataset.text as text
 
 See `Text Transforms
-<https://www.mindspore.cn/tutorials/en/r2.1/beginner/transforms.html#text-transforms>`_ tutorial for more details.
+<https://www.mindspore.cn/tutorials/en/r2.2/beginner/transforms.html#text-transforms>`_ tutorial for more details.
 
 Descriptions of common data processing terms are as follows:
 
@@ -35,10 +35,10 @@ Descriptions of common data processing terms are as follows:
 The data transform operation can be executed in the data processing pipeline or in the eager mode:
 
 - Pipeline mode is generally used to process big datasets. Examples refer to
-  `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/r2.1/api_python/
+  `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/r2.2/api_python/
   mindspore.dataset.html#introduction-to-data-processing-pipeline>`_ .
 - Eager mode is more like a function call to process data. Examples refer to
-  `Lightweight Data Processing <https://www.mindspore.cn/tutorials/en/r2.1/advanced/dataset/eager.html>`_ .
+  `Lightweight Data Processing <https://www.mindspore.cn/tutorials/en/r2.2/advanced/dataset/eager.html>`_ .
 """
 import platform