mindspore 1.10.0__cp37-none-any.whl → 2.0.0rc1__cp37-none-any.whl

mindspore/dataset/engine/datasets_user_defined.py

@@ -15,9 +15,6 @@
 """
 This file contains basic classes that help users do flexible dataset loading.
 You can define your own dataset loading class, and use GeneratorDataset to help load data.
-You can refer to the
-`tutorial <https://www.mindspore.cn/tutorials/en/r1.10/beginner/dataset.html#customizing-dataset>`_
-to help define your dataset loading.
 After declaring the dataset object, you can further apply dataset operations
 (e.g. filter, skip, concat, map, batch) on it.
 """
@@ -30,6 +27,7 @@ import multiprocessing
 from multiprocessing.util import Finalize
 import queue
 from functools import partial
+import subprocess
 import threading
 import weakref
 import platform
@@ -46,7 +44,7 @@ from . import samplers
 from .queue import _SharedQueue
 from .validators import check_generatordataset, check_numpyslicesdataset, check_paddeddataset
 from ..core.config import get_enable_shared_mem, get_prefetch_size, get_multiprocessing_timeout_interval, \
-    get_enable_watchdog
+    get_enable_watchdog, get_debug_mode
 from ..core.datatypes import mstypelist_to_detypelist
 from ..core.py_util_helpers import ExceptionHandler
 
@@ -131,11 +129,8 @@ def _fill_worker_indices(workers, indices, idx):
 
 def _convert_row(row):
     """
-    Convert Op return value to numpy
+    Convert Op return value to numpy, or keep as a dict (if already a dict)
     """
-    if isinstance(row, dict):
-        raise TypeError("Input data is expected to be " \
-                        "int, float, str, bytes, numpy.ndarray, Tensor or list/tuple of them, but got dict.")
 
     # convert single item to np.array
     prim_type = (int, float, str, bytes, np.ndarray, Tensor)
@@ -149,6 +144,9 @@ def _convert_row(row):
                                 "int or float or str, but got {}.".format(item.dtype))
         return tuple([item])
 
+    if isinstance(row, dict):
+        return tuple([row])
+
     value = []
     # convert each item to np.array
     idx = 0
@@ -157,8 +155,7 @@ def _convert_row(row):
         if isinstance(x, Tensor):      # mindspore.Tensor
             value.append(x.asnumpy())
         elif isinstance(x, dict):
-            raise TypeError("The {}th item of input data is expected to be " \
-                            "int, float, str, bytes, numpy.ndarray, Tensor, but got dict.".format(idx))
+            value.append(x)
         else:
             item = np.array(x, copy=False)
             if item.dtype == 'object':
@@ -255,6 +252,11 @@ class SamplerFn:
                 start_time = int(time.time())
                 wait_count = 1
                 while self.workers[i % self.num_worker].res_queue.empty():
+                    if self.eof.is_set():
+                        logger.warning("Generator receives a termination signal, stop waiting for data "
+                                       "from subprocess.")
+                        self._stop_subprocess()
+                        return
                     time.sleep(0.1)
                     cost_time = int(time.time()) - start_time
                     if cost_time / self.check_interval >= wait_count:
@@ -265,6 +267,14 @@ class SamplerFn:
                                        "output data is too large. You can also set the timeout interval by "
                                        "ds.config.set_multiprocessing_interval to adjust the output frequency of this "
                                        "log.")
+                        pid = self.workers[i % self.num_worker].pid
+                        logger.warning("Generator subprocess ID {} is stuck.".format(pid))
+                        install_status, _ = subprocess.getstatusoutput("py-spy --version")
+                        if install_status == 0:
+                            stack = subprocess.getoutput("py-spy dump -p {} -l -s".format(pid))
+                            logger.warning("Generator subprocess stack:\n{}".format(stack))
+                        else:
+                            logger.warning("Please `pip install py-spy` to get the stacks of the stuck process.")
 
                 result = self.workers[i % self.num_worker].get()
                 if isinstance(result, ExceptionHandler):
@@ -509,28 +519,30 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
             iter(source).next().
             Random accessible source is required to return a tuple of NumPy arrays as a row of the dataset on
             source[idx].
-        column_names (Union[str, list[str]], optional): List of column names of the dataset (default=None). Users are
+        column_names (Union[str, list[str]], optional): List of column names of the dataset. Default: None. Users are
             required to provide either column_names or schema.
-        column_types (list[mindspore.dtype], optional): List of column data types of the dataset (default=None).
+        column_types (list[mindspore.dtype], optional): List of column data types of the dataset. Default: None.
             If provided, sanity check will be performed on generator output.
-        schema (Union[Schema, str], optional): Path to the JSON schema file or schema object (default=None). Users are
-            required to provide either column_names or schema. If both are provided, schema will be used.
-        num_samples (int, optional): The number of samples to be included in the dataset
-            (default=None, all images).
-        num_parallel_workers (int, optional): Number of subprocesses used to fetch the dataset in parallel (default=1).
+        schema (Union[str, Schema], optional): Data format policy, which specifies the data types and shapes of the data
+            column to be read. Both JSON file path and objects constructed by mindspore.dataset.Schema are acceptable.
+            Default: None.
+        num_samples (int, optional): The number of samples to be included in the dataset.
+            Default: None, all images.
+        num_parallel_workers (int, optional): Number of worker threads/subprocesses used to
+            fetch the dataset in parallel. Default: 1.
         shuffle (bool, optional): Whether or not to perform shuffle on the dataset. Random accessible input is required.
-            (default=None, expected order behavior shown in the table).
+            Default: None, expected order behavior shown in the table below.
         sampler (Union[Sampler, Iterable], optional): Object used to choose samples from the dataset. Random accessible
-            input is required (default=None, expected order behavior shown in the table).
-        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).
+            input is required. Default: None, expected order behavior shown in the table below.
+        num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
             Random accessible input is required. When this argument is specified, `num_samples` reflects the maximum
             sample number of per shard.
-        shard_id (int, optional): The shard ID within `num_shards` (default=None). This argument must be specified only
-            when num_shards is also specified. Random accessible input is required.
+        shard_id (int, optional): The shard ID within `num_shards` . Default: None. This argument must be specified only
+            when `num_shards` is also specified. Random accessible input is required.
         python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker process. This
-            option could be beneficial if the Python operation is computational heavy (default=True).
+            option could be beneficial if the Python operation is computational heavy. Default: True.
         max_rowsize(int, optional): Maximum size of row in MB that is used for shared memory allocation to copy
-            data between processes.  This is only used if python_multiprocessing is set to True (default 6 MB).
+            data between processes.  This is only used if python_multiprocessing is set to True. Default: 6 MB.
 
     Raises:
         RuntimeError: If source raises an exception during execution.
@@ -538,14 +550,25 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
         ValueError: If `num_parallel_workers` exceeds the max thread numbers.
         ValueError: If sampler and shuffle are specified at the same time.
         ValueError: If sampler and sharding are specified at the same time.
-        ValueError: If num_shards is specified but shard_id is None.
+        ValueError: If `num_shards` is specified but shard_id is None.
         ValueError: If shard_id is specified but `num_shards` is None.
-        ValueError: If `shard_id` is invalid (< 0 or >= `num_shards`).
+        ValueError: If `shard_id` is not in range of [0, `num_shards` ).
 
     Note:
+        - If you configure `python_multiprocessing=True (default: True)` and `num_parallel_workers>1 (default: 1)`
+          indicates that the multi-process mode is started for data load acceleration. At this time, as the dataset
+          iterates, the memory consumption of the subprocess will gradually increase, mainly because the subprocess
+          of the user-defined dataset obtains the member variables from the main process in the Copy On Write way.
+          Example: If you define a dataset with `__ init__` function which contains a large number of member variable
+          data (for example, a very large file name list is loaded during the dataset construction) and uses the
+          multi-process mode, which may cause the problem of OOM (the estimated total memory usage is:
+          `(num_parallel_workers+1) * size of the parent process` ). The simplest solution is to replace Python objects
+          (such as list/dict/int/float/string) with non referenced data types
+          (such as Pandas, Numpy or PyArrow objects) for member variables, or load less meta data in member variables,
+          or configure `python_multiprocessing=False` to use multi-threading mode.
         - Input `source` accepts user-defined Python functions (PyFuncs), Do not add network computing operators from
-          mindspore.nn and mindspore.ops or others into this `source`.
-        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.
+          mindspore.nn and mindspore.ops or others into this `source` .
+        - This dataset can take in a `sampler` . `sampler` and `shuffle` are mutually exclusive.
           The table below shows what input arguments are allowed and their expected behavior.
 
     .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`
@@ -640,7 +663,7 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
         super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples,
                          shuffle=shuffle, num_shards=num_shards, shard_id=shard_id)
         if isinstance(source, builtins.zip):
-            # Although zip is iteratable, it does not have the feature of repeated iteration, so pass it to the array.
+            # Although zip is iterable, it does not have the feature of repeated iteration, so pass it to the array.
             self.source = [item for item in source]
         else:
             self.source = source
@@ -656,6 +679,10 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
         if platform.system().lower() == 'windows' and num_parallel_workers > 1 and python_multiprocessing:
             logger.warning("Python multiprocessing is not supported on Windows platform.")
         self.python_multiprocessing = python_multiprocessing if platform.system().lower() != 'windows' else False
+        if self.python_multiprocessing and get_debug_mode():
+            logger.warning("Python multiprocessing is not supported in debug mode."
+                           " Ignoring Python multiprocessing for GeneratorDataset.")
+            self.python_multiprocessing = False
 
         self.column_names = to_list(column_names)
 
@@ -675,6 +702,10 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
         if hasattr(self.source, "__len__"):
             self.source_len = len(self.source)
 
+            # if user defined sampler, update the self.source_len
+            if isinstance(self.sampler, samplers.Sampler) or hasattr(self.sampler, "__iter__"):
+                self.source_len = len(list(sampler))
+
         self.max_rowsize = max_rowsize
         self.sample_fn = None
 
@@ -728,6 +759,13 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
             return self.sampler.is_sharded()
         return False
 
+    def split(self, sizes, randomize=True):
+        if hasattr(self.source, "__getitem__"):
+            # If the source has __getitem__ attribute, call the split method of MappableDataset.
+            # Otherwise, call the split method of Dataset.
+            return super().split(sizes, randomize)
+        return super(MappableDataset, self).split(sizes, randomize)
+
     def parse(self, children=None):
         if self.schema is None:
             return cde.GeneratorNode(self.prepared_source, self.column_names, self.column_types, self.source_len,
@@ -779,12 +817,9 @@ class _NumpySlicesDataset:
             data = self.process_dict(data)
 
         if isinstance(data, tuple):
-            self.data = ()
-            data_len = len(data)
-            for i in range(data_len):
-                self.data = self.data + (np.array(data[i]),)
+            self.data = data
         else:
-            self.data = (np.array(data),)
+            self.data = (data,)
 
         # check whether the data length in each column is equal
         data_len = [len(data_item) for data_item in self.data]
@@ -801,7 +836,7 @@ class _NumpySlicesDataset:
                 self.column_list.append("column_" + str(i))
 
     def __getitem__(self, index):
-        data_row = [d[index, ...] for d in self.data]
+        data_row = [d[index] for d in self.data]
         data_res = tuple(data_row)
         return data_res
 
@@ -844,23 +879,23 @@ class NumpySlicesDataset(GeneratorDataset):
             NumPy formats. Input data will be sliced along the first dimension and generate additional rows, if input is
             list, there will be one column in each row, otherwise there tends to be multi columns. Large data is not
             recommended to be loaded in this way as data is loading into memory.
-        column_names (list[str], optional): List of column names of the dataset (default=None). If column_names is not
+        column_names (list[str], optional): List of column names of the dataset. Default: None. If column_names is not
             provided, the output column names will be named as the keys of dict when the input data is a dict,
             otherwise they will be named like column_0, column_1 ...
-        num_samples (int, optional): The number of samples to be included in the dataset (default=None, all samples).
-        num_parallel_workers (int, optional): Number of subprocesses used to fetch the dataset in parallel (default=1).
-        shuffle (bool, optional): Whether or not to perform shuffle on the dataset. Random accessible input is required.
-            (default=None, expected order behavior shown in the table).
-        sampler (Union[Sampler, Iterable], optional): Object used to choose samples from the dataset. Random accessible
-            input is required (default=None, expected order behavior shown in the table).
-        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).
-            Random accessible input is required. When this argument is specified, `num_samples` reflects the max
-            sample number of per shard.
-        shard_id (int, optional): The shard ID within `num_shards` (default=None). This argument must be specified only
-            when num_shards is also specified. Random accessible input is required.
+        num_samples (int, optional): The number of samples to be included in the dataset. Default: None, all samples.
+        num_parallel_workers (int, optional): Number of worker subprocesses used to
+            fetch the dataset in parallel. Default: 1.
+        shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
+            Default: None, expected order behavior shown in the table below.
+        sampler (Union[Sampler, Iterable], optional): Object used to choose samples from the dataset.
+            Default: None, expected order behavior shown in the table below.
+        num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
+            When this argument is specified, `num_samples` reflects the max sample number of per shard.
+        shard_id (int, optional): The shard ID within `num_shards` . Default: None. This argument must be specified only
+            when `num_shards` is also specified.
 
     Note:
-        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.
+        - This dataset can take in a `sampler` . `sampler` and `shuffle` are mutually exclusive.
           The table below shows what input arguments are allowed and their expected behavior.
 
     .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`
@@ -894,9 +929,9 @@ class NumpySlicesDataset(GeneratorDataset):
         ValueError: If `num_parallel_workers` exceeds the max thread numbers.
         ValueError: If sampler and shuffle are specified at the same time.
         ValueError: If sampler and sharding are specified at the same time.
-        ValueError: If num_shards is specified but shard_id is None.
+        ValueError: If `num_shards` is specified but shard_id is None.
         ValueError: If shard_id is specified but `num_shards` is None.
-        ValueError: If `shard_id` is invalid (< 0 or >= `num_shards`).
+        ValueError: If `shard_id` is not in range of [0, `num_shards` ).
 
     Examples:
         >>> # 1) Input data can be a list