mindspore 2.4.1__cp310-cp310-manylinux1_x86_64.whl → 2.5.0__cp310-cp310-manylinux1_x86_64.whl

mindspore/dataset/engine/iterators.py

@@ -15,19 +15,24 @@
 """Built-in iterators"""
 from abc import abstractmethod
 from copy import deepcopy
+import atexit
 import json
 import os
+import queue
 import signal
+import threading
 import weakref
 from functools import wraps
 import numpy as np
 
 import mindspore._c_dataengine as cde
 from mindspore.common.tensor import Tensor, np_types
+import mindspore.dataset as ds
 import mindspore.dataset.engine.offload as offload
 from mindspore.dataset.core.config import get_debug_mode
 
 from mindspore import log as logger
+from ..core.py_util_helpers import ExceptionHandler
 
 _ITERATOR_CLEANUP = False
 
@@ -87,6 +92,126 @@ def _cleanup_the_iterators_if_created(method):
     return wrapper
 
 
+def __convert_python(obj, to_numpy, _do_copy):
+    """
+    Attempts to recursively convert a python object to Numpy array(s) or tensor(s).
+
+    Args:
+        obj (any): the python object to be converted
+        to_numpy (bool): If True, convert primitive types to NumPy array. If False, convert to Tensor.
+                         (return the obj if type isn't supported)
+    """
+    if isinstance(obj, (int, float, bool, str, np.ndarray, np.str_, np.bytes_, *np_types)):
+        # error out if array is of unsupported type
+        if isinstance(obj, np.ndarray) and obj.dtype not in np_types and obj.dtype.kind not in ('U', 'S'):
+            new_line = '\n'
+            raise TypeError("A NumPy array of unsupported type detected: {}."
+                            "\nSupported types are: {}.".format(
+                                obj.dtype, new_line.join(map(str, (*np_types, np.str_, np.bytes_)))))
+        if to_numpy:
+            return np.array(obj, copy=_do_copy)
+        if _do_copy:
+            return Tensor(np.asarray(obj))
+        return Tensor.from_numpy(np.asarray(obj))
+    if isinstance(obj, dict):
+        return {key: __convert_python(val, to_numpy, _do_copy) for key, val in obj.items()}
+    if isinstance(obj, tuple):
+        return tuple([__convert_python(item, to_numpy, _do_copy) for item in obj])
+    if isinstance(obj, list):
+        return [__convert_python(item, to_numpy, _do_copy) for item in obj]
+    # if we can't convert it to Tensor, return the object as is
+    if _do_copy:
+        return deepcopy(obj)
+    return obj
+
+
+def _transform_md_to_output(t, _output_numpy, _do_copy):
+    if _output_numpy:
+        if t.type().is_python():
+            return __convert_python(t.as_python(), True, _do_copy)
+        return t.as_array()
+    return _transform_md_to_tensor(t, _do_copy)
+
+
+def _transform_md_to_tensor(t, _do_copy):
+    if t.type().is_python():
+        return __convert_python(t.as_python(), False, _do_copy)
+    array = t.as_array()
+    if _do_copy:
+        return Tensor(array)
+    return Tensor.from_numpy(array)
+
+
+def _transform_tensor_to_output(t, _output_numpy):
+    if _output_numpy:
+        return t.asnumpy()
+    return t
+
+
+def _convert_tuple_data(queue_in, queue_out, offload_model, _output_numpy, _do_copy, thread_event):
+    """
+    Convert data on tuple iterator.
+    """
+    while True:
+        try:
+            if thread_event.is_set():
+                return
+            item = queue_in.get(timeout=1)
+            if offload_model is None:
+                queue_out.put([_transform_md_to_output(t, _output_numpy, _do_copy) for t in item])
+                if not item:
+                    break
+                continue
+
+            data = [_transform_md_to_tensor(t, _do_copy) for t in item]
+            if data:
+                data = offload.apply_offload_iterators(data, offload_model)
+            queue_out.put([_transform_tensor_to_output(t, _output_numpy) for t in data])
+            if not item:
+                break
+        except queue.Empty:
+            continue
+        except Exception:  # pylint: disable=broad-except
+            result = ExceptionHandler()
+            queue_out.put(result)
+            break
+    thread_event.set()
+
+
+def _convert_dict_data(queue_in, queue_out, offload_model, _output_numpy, _do_copy, thread_event, col_names):
+    """
+    Convert data on dict iterator.
+    """
+    while True:
+        try:
+            if thread_event.is_set():
+                return
+            item = queue_in.get(timeout=1)
+            if offload_model is None:
+                queue_out.put({k: _transform_md_to_output(t, _output_numpy, _do_copy) for k, t in item})
+                if not item:
+                    break
+                continue
+            data = [_transform_md_to_tensor(t, _do_copy) for t in item]
+            if data:
+                data = offload.apply_offload_iterators(data, offload_model)
+                # Create output dictionary after offload
+                out_data = {}
+                for i, col in enumerate(col_names):
+                    out_data[col] = _transform_tensor_to_output(data[i], _output_numpy)
+                data = out_data
+            if not item:
+                break
+            queue_out.put(data)
+        except queue.Empty:
+            continue
+        except Exception:  # pylint: disable=broad-except
+            result = ExceptionHandler()
+            queue_out.put(result)
+            break
+    thread_event.set()
+
+
 class Iterator:
     """
     General Iterator over a dataset.
@@ -112,6 +237,8 @@ class Iterator:
             init_step = dataset.get_init_step()
             dataset_size = dataset.get_dataset_size()
         if get_debug_mode():
+            if dataset.get_init_step() != 0:
+                logger.warning("Dataset init step will be ignored in debug mode.")
             consumer = cde.PythonPullBasedIteratorConsumer(num_epochs)
             consumer.Init(self.ir_tree)
         else:
@@ -134,10 +261,35 @@ class Iterator:
 
         ITERATORS_LIST.append(weakref.ref(self))
         _unset_iterator_cleanup()
+        self.parallel_convert = ds.config.get_iterator_mode()["parallel_convert"]
+        if self.parallel_convert:
+            # The variable "tick" ensures that the thread is only started on the first iteration
+            self.tick = True
+            self.thread_convert = None
+            self.queue_in = queue.Queue(3)
+            self.queue_out = queue.Queue(3)
+            self.thread_event = None
+            self.enable_get_next_data = True
+            atexit.register(self.__class__.terminate, weakref.ref(self))
 
     def __iter__(self):
         return self
 
+    @staticmethod
+    def terminate(ref):
+        """
+        Interrupt the convert subthread
+        """
+        self = ref()
+        if self is None:
+            return
+        if hasattr(self, "parallel_convert"):
+            if self.parallel_convert:
+                if self.thread_event is not None and self.thread_convert is not None and \
+                    not self.thread_event.is_set():
+                    self.thread_event.set()
+                    self.thread_convert.join()
+
     def stop(self):
         """
         Manually terminate Python iterator instead of relying on out of scope destruction.
@@ -165,18 +317,26 @@ class Iterator:
         self.stop()
 
     def __del__(self):
+        if hasattr(self, "parallel_convert"):
+            if self.parallel_convert:
+                if self.thread_event is not None and self.thread_convert is not None and \
+                    not self.thread_event.is_set():
+                    self.thread_event.set()
+                    self.thread_convert.join()
         self.release()
 
     @abstractmethod
     def _get_next(self):
         raise RuntimeError("Calling base class Iterator's get_next is invalid.")
 
-    def __next__(self):
-        if not self._runtime_context:
-            logger.warning("Iterator does not have a running C++ pipeline." +
-                           "It might because Iterator stop() had been called, or C++ pipeline crashed silently.")
-            raise RuntimeError("Iterator does not have a running C++ pipeline.")
+    @abstractmethod
+    def _parallel_transformation_iteration(self):
+        raise RuntimeError("Calling base class Iterator's parallel_transformation_iteration is invalid.")
 
+    def serial_conversion_iteration(self):
+        """
+        Fetch data to serial conversion
+        """
         # Note offload is applied inside _get_next() if applicable since get_next converts to output format
         data = self._get_next()
         if not data:
@@ -189,6 +349,16 @@ class Iterator:
 
         return data
 
+    def __next__(self):
+        if not self._runtime_context:
+            logger.warning("Iterator does not have a running C++ pipeline." +
+                           "It might because Iterator stop() had been called, or C++ pipeline crashed silently.")
+            raise RuntimeError("Iterator does not have a running C++ pipeline.")
+
+        if self.parallel_convert:
+            return self._parallel_transformation_iteration()
+        return self.serial_conversion_iteration()
+
     def __deepcopy__(self, memo):
         return self
 
@@ -218,58 +388,20 @@ class Iterator:
             dataset_size (int): The number of steps that one epoch has.
         """
         self._iterator.Reset(step, dataset_size)
-
-    def __convert_python(self, obj, to_numpy):
-        """
-        Attempts to recursively convert a python object to Numpy array(s) or tensor(s).
-
-        Args:
-            obj (any): the python object to be converted
-            to_numpy (bool): If True, convert primitive types to NumPy array. If False, convert to Tensor.
-                             (return the obj if type isn't supported)
-        """
-        if isinstance(obj, (int, float, bool, str, np.ndarray, np.str_, np.bytes_, *np_types)):
-            # error out if array is of unsupported type
-            if isinstance(obj, np.ndarray) and obj.dtype not in np_types and obj.dtype.kind not in ('U', 'S'):
-                new_line = '\n'
-                raise TypeError("A NumPy array of unsupported type detected: {}."
-                                "\nSupported types are: {}.".format(
-                                    obj.dtype, new_line.join(map(str, (*np_types, np.str_, np.bytes_)))))
-            if to_numpy:
-                return np.array(obj, copy=self._do_copy)
-            if self._do_copy:
-                return Tensor(np.asarray(obj))
-            return Tensor.from_numpy(np.asarray(obj))
-        if isinstance(obj, dict):
-            return {key: self.__convert_python(val, to_numpy) for key, val in obj.items()}
-        if isinstance(obj, tuple):
-            return tuple([self.__convert_python(item, to_numpy) for item in obj])
-        if isinstance(obj, list):
-            return [self.__convert_python(item, to_numpy) for item in obj]
-        # if we can't convert it to Tensor, return the object as is
-        if self._do_copy:
-            return deepcopy(obj)
-        return obj
-
-    def _transform_md_to_output(self, t):
-        if self._output_numpy:
-            if t.type().is_python():
-                return self.__convert_python(t.as_python(), True)
-            return t.as_array()
-        return self._transform_md_to_tensor(t)
-
-    def _transform_md_to_tensor(self, t):
-        if t.type().is_python():
-            return self.__convert_python(t.as_python(), False)
-        array = t.as_array()
-        if self._do_copy:
-            return Tensor(array)
-        return Tensor.from_numpy(array)
-
-    def _transform_tensor_to_output(self, t):
-        if self._output_numpy:
-            return t.asnumpy()
-        return t
+        if self.parallel_convert:
+            while not self.queue_in.empty():
+                self.queue_in.get()
+            while not self.queue_out.empty():
+                self.queue_out.get()
+            if self.thread_event is not None:
+                if self.thread_event.is_set():
+                    self.thread_event.clear()
+                    self.tick = True
+                else:
+                    self.thread_event.set()
+                    self.thread_convert.join()
+            self.tick = True
+            self.enable_get_next_data = True
 
 
 class DictIterator(Iterator):
@@ -286,14 +418,15 @@ class DictIterator(Iterator):
         """
         try:
             if self.offload_model is None:
-                return {k: self._transform_md_to_output(t) for k, t in self._iterator.GetNextAsMap().items()}
-            data = [self._transform_md_to_tensor(t) for t in self._iterator.GetNextAsList()]
+                return {k: _transform_md_to_output(t, self._output_numpy, self._do_copy) for k, t in
+                        self._iterator.GetNextAsMap().items()}
+            data = [_transform_md_to_tensor(t, self._do_copy) for t in self._iterator.GetNextAsList()]
             if data:
                 data = offload.apply_offload_iterators(data, self.offload_model)
                 # Create output dictionary after offload
                 out_data = {}
                 for i, col in enumerate(self.get_col_names()):
-                    out_data[col] = self._transform_tensor_to_output(data[i])
+                    out_data[col] = _transform_tensor_to_output(data[i], self._output_numpy)
                 data = out_data
             return data
 
@@ -305,6 +438,53 @@ class DictIterator(Iterator):
                 os.kill(os.getpid(), signal.SIGKILL)
             raise err
 
+    def _parallel_transformation_iteration(self):
+        """
+        Launch child thread to convert tensor.
+        """
+        if self.tick:
+            self.thread_event = threading.Event()
+            self.thread_convert = threading.Thread(target=_convert_dict_data,
+                                                   name="Convert_dict_data",
+                                                   args=(self.queue_in, self.queue_out, self.offload_model,
+                                                         self._output_numpy, self._do_copy, self.thread_event,
+                                                         self.get_col_names()),
+                                                   daemon=True)
+            self.thread_convert.start()
+            self.tick = False
+        while True:
+            if self.thread_event.is_set() and self.queue_out.qsize() == 0:
+                self.tick = True
+                self.enable_get_next_data = True
+                raise StopIteration
+            try:
+                if not self.queue_in.full() and self.enable_get_next_data:
+                    if self.offload_model is None:
+                        item = self._iterator.GetNextAsMap().items()
+                        if not item:
+                            self.enable_get_next_data = False
+                        self.queue_in.put(item)
+                    else:
+                        item = self._iterator.GetNextAsList()
+                        if not item:
+                            self.enable_get_next_data = False
+                        self.queue_in.put(item)
+                data = self.queue_out.get(timeout=0.00001)
+                if not data:
+                    continue
+                if isinstance(data, ExceptionHandler):
+                    if data.except_msg.find("Out of memory") >= 0 or data.except_msg.find("MemoryError") >= 0:
+                        logger.critical("Memory error occurred, process will exit.")
+                        os.kill(os.getpid(), signal.SIGKILL)
+                    data.reraise()
+                return data
+            except queue.Empty:
+                continue
+            except Exception as err_info:
+                self.thread_event.set()
+                self.thread_convert.join()
+                raise err_info
+
 
 class TupleIterator(Iterator):
     """
@@ -327,11 +507,49 @@ class TupleIterator(Iterator):
         """
 
         if self.offload_model is None:
-            return [self._transform_md_to_output(t) for t in self._iterator.GetNextAsList()]
-        data = [self._transform_md_to_tensor(t) for t in self._iterator.GetNextAsList()]
+            return [_transform_md_to_output(t, self._output_numpy, self._do_copy) for t in
+                    self._iterator.GetNextAsList()]
+        data = [_transform_md_to_tensor(t, self._do_copy) for t in self._iterator.GetNextAsList()]
         if data:
             data = offload.apply_offload_iterators(data, self.offload_model)
-        return [self._transform_tensor_to_output(t) for t in data]
+        return [_transform_tensor_to_output(t, self._output_numpy) for t in data]
+
+    def _parallel_transformation_iteration(self):
+        """
+        Launch child thread to convert tensor.
+        """
+        if self.tick:
+            self.thread_event = threading.Event()
+            self.thread_convert = threading.Thread(target=_convert_tuple_data,
+                                                   name="Convert_tuple_data",
+                                                   args=(self.queue_in, self.queue_out, self.offload_model,
+                                                         self._output_numpy, self._do_copy, self.thread_event),
+                                                   daemon=True)
+            self.thread_convert.start()
+            self.tick = False
+        while True:
+            if self.thread_event.is_set() and self.queue_out.qsize() == 0:
+                self.tick = True
+                self.enable_get_next_data = True
+                raise StopIteration
+            try:
+                if not self.queue_in.full() and self.enable_get_next_data:
+                    item = self._iterator.GetNextAsList()
+                    if not item:
+                        self.enable_get_next_data = False
+                    self.queue_in.put(item)
+                data = self.queue_out.get(timeout=0.00001)
+                if not data:
+                    continue
+                if isinstance(data, ExceptionHandler):
+                    data.reraise()
+                return data
+            except queue.Empty:
+                continue
+            except Exception as err_info:
+                self.thread_event.set()
+                self.thread_convert.join()
+                raise err_info
 
 
 class DummyIterator:

mindspore/dataset/engine/obs/util.py

@@ -42,6 +42,14 @@ obsClient = ObsClient(
 def get_used_disk_per():
     """ Get the disk usage of working directory."""
 
+    if os.path.exists(config.WORKING_PATH) and not os.access(config.WORKING_PATH, os.R_OK | os.W_OK | os.X_OK):
+        err_msg =\
+                "Permission denied:'{}'. Suggestion: " \
+                "1) It is recommended to manually create the {} directory and add read/write permissions to it. " \
+                "2) If you can't create, we suggest you download MindRecord manually and "\
+                "read it using the MindDataset interface.".format(config.WORKING_PATH, config.WORKING_PATH)
+        raise RuntimeError(err_msg)
+
     if not os.path.exists(config.WORKING_PATH):
         try:
             os.makedirs(config.WORKING_PATH)

mindspore/dataset/engine/queue.py

@@ -24,6 +24,7 @@ import platform
 import queue
 import types
 
+import dill
 import numpy as np
 
 from mindspore import log as logger
@@ -71,6 +72,9 @@ class _SharedQueue(multiprocessing.queues.Queue):
         # reading a buffer and also have a full queue of buffers in the meta-data queue
         self.num_seg = size + 2
 
+        self.dynamic_shm = True
+        self.fd_list = []
+        self.seg_size = 0
         if platform.system().lower() != 'windows' and max_rowsize == -1:
             self.dynamic_shm = True
             self.fd_list = []
@@ -89,6 +93,42 @@ class _SharedQueue(multiprocessing.queues.Queue):
                 else:
                     self.shm_list.append(shared_array)
 
+    def __getstate__(self):
+        copy_out = dill.dumps(self.copy_out)
+        min_shared_mem = dill.dumps(self.min_shared_mem)
+        data_immediate = dill.dumps(self.data_immediate)
+        data_shared = dill.dumps(self.data_shared)
+        # cannot dill 'mmap.mmap' object : self.count
+        print_error = dill.dumps(self.print_error)
+        shm_list = dill.dumps(self.shm_list)
+        seg_pos = dill.dumps(self.seg_pos)
+        num_seg = dill.dumps(self.num_seg)
+        dynamic_shm = dill.dumps(self.dynamic_shm)
+        fd_list = dill.dumps(self.fd_list)
+        seg_size = dill.dumps(self.seg_size)
+        # multiprocessing attribute
+        _closed = dill.dumps(self._closed)
+        super_variable = super().__getstate__()
+        return (copy_out, min_shared_mem, data_immediate, data_shared, print_error, shm_list,
+                seg_pos, num_seg, dynamic_shm, fd_list, seg_size, _closed, super_variable)
+
+    def __setstate__(self, state):
+        copy_out, min_shared_mem, data_immediate, data_shared, print_error, shm_list, \
+            seg_pos, num_seg, dynamic_shm, fd_list, seg_size, _closed, super_variable = state
+        self.copy_out = dill.loads(copy_out)
+        self.min_shared_mem = dill.loads(min_shared_mem)
+        self.data_immediate = dill.loads(data_immediate)
+        self.data_shared = dill.loads(data_shared)
+        self.print_error = dill.loads(print_error)
+        self.shm_list = dill.loads(shm_list)
+        self.seg_pos = dill.loads(seg_pos)
+        self.num_seg = dill.loads(num_seg)
+        self.dynamic_shm = dill.loads(dynamic_shm)
+        self.fd_list = dill.loads(fd_list)
+        self.seg_size = dill.loads(seg_size)
+        self._closed = dill.loads(_closed)
+        super().__setstate__(super_variable)
+
     def put_until(self, data, timeout=None, exit_signal=None):
         """Put data into the queue. Block until timeout is reached or exit_signal is set."""
         while True:

mindspore/dataset/engine/samplers.py

@@ -19,7 +19,9 @@ SequentialSampler, SubsetRandomSampler, and WeightedRandomSampler.
 Users can also define a custom sampler by extending from the Sampler class.
 """
 
+import copy
 import numbers
+
 import numpy as np
 import mindspore._c_dataengine as cde
 import mindspore.dataset as ds
@@ -253,6 +255,7 @@ class Sampler(BuiltinSampler):
         self.dataset_size = 0
         self.child_sampler = None
         self.num_samples = num_samples
+        self.batch_sizes = []
 
     def __iter__(self):
         """
@@ -278,17 +281,34 @@ class Sampler(BuiltinSampler):
     def _get_indices(self):
         sampler_iter = iter(self)
         ret = []
+        batch_sizes = []
         for _ in range(self.num_samples):
             try:
                 idx = next(sampler_iter)
-                ret.append(idx)
+                # The idx can be either a number (for sampler) or a list (for batch sampler).
+                # If number, we convert it to list first. So they can be handled in the same way.
+                if isinstance(idx, numbers.Number):
+                    idx = [idx]
+                    # normal sampler does not have batch sizes
+                    batch_sizes.append(0)
+                else:
+                    # Using extend instead of append will flatten the list, so we need to save the
+                    # batch size information here.
+                    batch_sizes.append(len(idx))
+                ret.extend(idx)
             except StopIteration:
                 break
+        self.batch_sizes.append(batch_sizes)
         indices = np.array(ret)
         if indices.dtype == object:
             raise RuntimeError("Fetched indices can not be converted to a valid ndarray.")
         return indices
 
+    def _get_batch_sizes(self):
+        if not self.batch_sizes:
+            return []
+        return self.batch_sizes.pop(0)
+
     # Instance fetcher
     # Do not override this method!
     def parse(self):
@@ -818,7 +838,11 @@ class IterSampler(Sampler):
 
     def __init__(self, sampler, num_samples=None):
         if num_samples is None:
-            num_samples = len(list(sampler))
+            if hasattr(sampler, "__len__"):
+                num_samples = len(sampler)
+            else:
+                # counting on a copied sampler to prevent changing the random state of the original one
+                num_samples = len(list(copy.deepcopy(sampler)))
         super().__init__(num_samples=num_samples)
         self.sampler = sampler
 

mindspore/dataset/engine/serializer_deserializer.py

@@ -68,7 +68,7 @@ def deserialize(input_dict=None, json_filepath=None):
     Args:
         input_dict (dict): A Python dictionary containing a serialized dataset graph. Default: ``None``.
         json_filepath (str): A path to the JSON file containing dataset graph.
-            User can obtain this file by calling API `mindspore.dataset.serialize()` . Default: ``None``.
+            User can obtain this file by calling API :func:`mindspore.dataset.serialize` . Default: ``None``.
 
     Returns:
         de.Dataset or None if error occurs.