mindspore 2.0.0a0__cp37-none-any.whl → 2.0.0rc1__cp37-none-any.whl

mindspore/dataset/engine/datasets_user_defined.py

@@ -27,6 +27,7 @@ import multiprocessing
 from multiprocessing.util import Finalize
 import queue
 from functools import partial
+import subprocess
 import threading
 import weakref
 import platform
@@ -43,7 +44,7 @@ from . import samplers
 from .queue import _SharedQueue
 from .validators import check_generatordataset, check_numpyslicesdataset, check_paddeddataset
 from ..core.config import get_enable_shared_mem, get_prefetch_size, get_multiprocessing_timeout_interval, \
-    get_enable_watchdog
+    get_enable_watchdog, get_debug_mode
 from ..core.datatypes import mstypelist_to_detypelist
 from ..core.py_util_helpers import ExceptionHandler
 
@@ -128,11 +129,8 @@ def _fill_worker_indices(workers, indices, idx):
 
 def _convert_row(row):
     """
-    Convert Op return value to numpy
+    Convert Op return value to numpy, or keep as a dict (if already a dict)
     """
-    if isinstance(row, dict):
-        raise TypeError("Input data is expected to be " \
-                        "int, float, str, bytes, numpy.ndarray, Tensor or list/tuple of them, but got dict.")
 
     # convert single item to np.array
     prim_type = (int, float, str, bytes, np.ndarray, Tensor)
@@ -146,6 +144,9 @@ def _convert_row(row):
                                 "int or float or str, but got {}.".format(item.dtype))
         return tuple([item])
 
+    if isinstance(row, dict):
+        return tuple([row])
+
     value = []
     # convert each item to np.array
     idx = 0
@@ -154,8 +155,7 @@ def _convert_row(row):
         if isinstance(x, Tensor):      # mindspore.Tensor
             value.append(x.asnumpy())
         elif isinstance(x, dict):
-            raise TypeError("The {}th item of input data is expected to be " \
-                            "int, float, str, bytes, numpy.ndarray, Tensor, but got dict.".format(idx))
+            value.append(x)
         else:
             item = np.array(x, copy=False)
             if item.dtype == 'object':
@@ -267,6 +267,14 @@ class SamplerFn:
                                        "output data is too large. You can also set the timeout interval by "
                                        "ds.config.set_multiprocessing_interval to adjust the output frequency of this "
                                        "log.")
+                        pid = self.workers[i % self.num_worker].pid
+                        logger.warning("Generator subprocess ID {} is stuck.".format(pid))
+                        install_status, _ = subprocess.getstatusoutput("py-spy --version")
+                        if install_status == 0:
+                            stack = subprocess.getoutput("py-spy dump -p {} -l -s".format(pid))
+                            logger.warning("Generator subprocess stack:\n{}".format(stack))
+                        else:
+                            logger.warning("Please `pip install py-spy` to get the stacks of the stuck process.")
 
                 result = self.workers[i % self.num_worker].get()
                 if isinstance(result, ExceptionHandler):
@@ -520,7 +528,8 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
             Default: None.
         num_samples (int, optional): The number of samples to be included in the dataset.
             Default: None, all images.
-        num_parallel_workers (int, optional): Number of subprocesses used to fetch the dataset in parallel. Default: 1.
+        num_parallel_workers (int, optional): Number of worker threads/subprocesses used to
+            fetch the dataset in parallel. Default: 1.
         shuffle (bool, optional): Whether or not to perform shuffle on the dataset. Random accessible input is required.
             Default: None, expected order behavior shown in the table below.
         sampler (Union[Sampler, Iterable], optional): Object used to choose samples from the dataset. Random accessible
@@ -529,7 +538,7 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
             Random accessible input is required. When this argument is specified, `num_samples` reflects the maximum
             sample number of per shard.
         shard_id (int, optional): The shard ID within `num_shards` . Default: None. This argument must be specified only
-            when num_shards is also specified. Random accessible input is required.
+            when `num_shards` is also specified. Random accessible input is required.
         python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker process. This
             option could be beneficial if the Python operation is computational heavy. Default: True.
         max_rowsize(int, optional): Maximum size of row in MB that is used for shared memory allocation to copy
@@ -541,11 +550,22 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
         ValueError: If `num_parallel_workers` exceeds the max thread numbers.
         ValueError: If sampler and shuffle are specified at the same time.
         ValueError: If sampler and sharding are specified at the same time.
-        ValueError: If num_shards is specified but shard_id is None.
+        ValueError: If `num_shards` is specified but shard_id is None.
         ValueError: If shard_id is specified but `num_shards` is None.
-        ValueError: If `shard_id` is invalid (< 0 or >= `num_shards`).
+        ValueError: If `shard_id` is not in range of [0, `num_shards` ).
 
     Note:
+        - If you configure `python_multiprocessing=True (default: True)` and `num_parallel_workers>1 (default: 1)`
+          indicates that the multi-process mode is started for data load acceleration. At this time, as the dataset
+          iterates, the memory consumption of the subprocess will gradually increase, mainly because the subprocess
+          of the user-defined dataset obtains the member variables from the main process in the Copy On Write way.
+          Example: If you define a dataset with `__ init__` function which contains a large number of member variable
+          data (for example, a very large file name list is loaded during the dataset construction) and uses the
+          multi-process mode, which may cause the problem of OOM (the estimated total memory usage is:
+          `(num_parallel_workers+1) * size of the parent process` ). The simplest solution is to replace Python objects
+          (such as list/dict/int/float/string) with non referenced data types
+          (such as Pandas, Numpy or PyArrow objects) for member variables, or load less meta data in member variables,
+          or configure `python_multiprocessing=False` to use multi-threading mode.
         - Input `source` accepts user-defined Python functions (PyFuncs), Do not add network computing operators from
           mindspore.nn and mindspore.ops or others into this `source` .
         - This dataset can take in a `sampler` . `sampler` and `shuffle` are mutually exclusive.
@@ -659,6 +679,10 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
         if platform.system().lower() == 'windows' and num_parallel_workers > 1 and python_multiprocessing:
             logger.warning("Python multiprocessing is not supported on Windows platform.")
         self.python_multiprocessing = python_multiprocessing if platform.system().lower() != 'windows' else False
+        if self.python_multiprocessing and get_debug_mode():
+            logger.warning("Python multiprocessing is not supported in debug mode."
+                           " Ignoring Python multiprocessing for GeneratorDataset.")
+            self.python_multiprocessing = False
 
         self.column_names = to_list(column_names)
 
@@ -794,9 +818,8 @@ class _NumpySlicesDataset:
 
         if isinstance(data, tuple):
             self.data = data
-            data_len = len(data)
         else:
-            self.data = (np.array(data),)
+            self.data = (data,)
 
         # check whether the data length in each column is equal
         data_len = [len(data_item) for data_item in self.data]
@@ -860,7 +883,8 @@ class NumpySlicesDataset(GeneratorDataset):
             provided, the output column names will be named as the keys of dict when the input data is a dict,
             otherwise they will be named like column_0, column_1 ...
         num_samples (int, optional): The number of samples to be included in the dataset. Default: None, all samples.
-        num_parallel_workers (int, optional): Number of subprocesses used to fetch the dataset in parallel. Default: 1.
+        num_parallel_workers (int, optional): Number of worker subprocesses used to
+            fetch the dataset in parallel. Default: 1.
         shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
             Default: None, expected order behavior shown in the table below.
         sampler (Union[Sampler, Iterable], optional): Object used to choose samples from the dataset.
@@ -868,7 +892,7 @@ class NumpySlicesDataset(GeneratorDataset):
         num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
             When this argument is specified, `num_samples` reflects the max sample number of per shard.
         shard_id (int, optional): The shard ID within `num_shards` . Default: None. This argument must be specified only
-            when num_shards is also specified.
+            when `num_shards` is also specified.
 
     Note:
         - This dataset can take in a `sampler` . `sampler` and `shuffle` are mutually exclusive.
@@ -905,9 +929,9 @@ class NumpySlicesDataset(GeneratorDataset):
         ValueError: If `num_parallel_workers` exceeds the max thread numbers.
         ValueError: If sampler and shuffle are specified at the same time.
         ValueError: If sampler and sharding are specified at the same time.
-        ValueError: If num_shards is specified but shard_id is None.
+        ValueError: If `num_shards` is specified but shard_id is None.
         ValueError: If shard_id is specified but `num_shards` is None.
-        ValueError: If `shard_id` is invalid (< 0 or >= `num_shards`).
+        ValueError: If `shard_id` is not in range of [0, `num_shards` ).
 
     Examples:
         >>> # 1) Input data can be a list