mindspore 2.6.0rc1__cp311-cp311-win_amd64.whl → 2.7.0__cp311-cp311-win_amd64.whl

mindspore/dataset/engine/datasets_user_defined.py

@@ -48,7 +48,8 @@ from .datasets import UnionBaseDataset, MappableDataset, Schema, to_list, _Pytho
 from .queue import _SharedQueue
 from .validators import check_generator_dataset, check_numpy_slices_dataset, check_padded_dataset
 from ..core.config import get_enable_shared_mem, get_prefetch_size, get_multiprocessing_timeout_interval, \
-    get_enable_watchdog, get_debug_mode, get_seed, set_seed, get_multiprocessing_start_method
+    get_enable_watchdog, get_debug_mode, get_seed, set_seed, get_multiprocessing_start_method, get_video_backend, \
+    set_video_backend
 from ..core.datatypes import mstypelist_to_detypelist
 from ..core.py_util_helpers import ExceptionHandler
 from ..core.validator_helpers import type_check
@@ -220,6 +221,7 @@ class SamplerFn(cde.PythonMultiprocessingRuntime):
 
         self.ppid = os.getpid()
         self.pids = []
+        self.thread_ids = []
         self.check_interval = get_multiprocessing_timeout_interval()  # the interval of check queue's size
 
         if self.multi_process is True:
@@ -277,12 +279,32 @@ class SamplerFn(cde.PythonMultiprocessingRuntime):
                 worker = _GeneratorWorkerMt(self.dataset, self.eof, worker_id)
                 worker.daemon = True
                 self.need_join = True
+                worker.start()
+                self.thread_ids.append(worker.ident)
                 self.workers.append(worker)
 
         # Register a termination function using weakref to avoid the object from unable to properly destruct.
         atexit.register(lambda cleanup: cleanup()() if cleanup() is not None else None,
                         weakref.WeakMethod(self.terminate))
 
+    def get_worker_ids(self):
+        """
+        Get dict of worker's ids
+
+        Returns:
+            dict of strings
+        """
+        if not self.is_mp_enabled():
+            return {}
+        worker_ids = {}
+        if self.multi_process is True:
+            worker_ids["is_thread"] = False
+            worker_ids["worker_id"] = self.pids
+        else:
+            worker_ids["is_thread"] = True
+            worker_ids["worker_id"] = self.thread_ids
+        return worker_ids
+
     def terminate(self):
         self._stop_subprocess()
 
@@ -345,12 +367,12 @@ class SamplerFn(cde.PythonMultiprocessingRuntime):
                 start_time = int(time.time())
                 wait_count = 1
                 while self.workers[i % self.num_worker].res_queue.empty():
+                    time.sleep(0.1)
                     if self.eof.is_set():
                         logger.warning("Generator receives a termination signal, stop waiting for data "
                                        "from subprocess.")
                         self._stop_subprocess()
                         return
-                    time.sleep(0.1)
                     wait_count = self._interval_log(i, start_time, wait_count)
                 result = self.workers[i % self.num_worker].get()
                 if isinstance(result, ExceptionHandler):
@@ -421,9 +443,10 @@ class SamplerFn(cde.PythonMultiprocessingRuntime):
                     "process(es): {}".format(self.cleaning_process.pid, [worker.pid for worker in self.workers]))
 
         if get_enable_watchdog():
-            worker_ids = [worker.pid for worker in self.workers]
+            worker_ids = [os.getpid()]
+            worker_ids.extend([worker.pid for worker in self.workers])
             worker_ids.append(self.cleaning_process.pid)
-            cde.register_worker_pids(id(self), set(worker_ids))
+            cde.register_worker_pids(id(self), worker_ids)
 
     def _release_fd(self):
         """Release the file descriptor by subprocess"""
@@ -480,6 +503,8 @@ class SamplerFn(cde.PythonMultiprocessingRuntime):
                     except Exception:  # pylint: disable=W0703
                         # Block all errors when join
                         continue
+                elif not self.multi_process:
+                    w.join(timeout=5)
 
             if self.multi_process is True:
                 self._release_fd()
@@ -497,6 +522,10 @@ class SamplerFn(cde.PythonMultiprocessingRuntime):
             cde.deregister_worker_pids(id(self))
         if hasattr(self, 'eof') and self.eof is not None:
             self.eof.set()
+            # send QUIT flag to workers, and the worker's while loop could check the eof flag
+            for worker in self.workers:
+                if not worker.queue_full():
+                    worker.put("QUIT")
         if hasattr(self, 'cleaning_process') and self.cleaning_process is not None:
             # let the quit event notify the cleaning process to exit
             self.cleaning_process.join(timeout=5)
@@ -538,7 +567,8 @@ def _main_process_already_exit(eof, is_multiprocessing, idx_queue, result_queue,
     return False
 
 
-def _generator_worker_loop(dataset, idx_queue, result_queue, eof, is_multiprocessing, worker_id, ppid=-1):
+def _generator_worker_loop(dataset, idx_queue, result_queue, eof, is_multiprocessing, worker_id, ppid=-1,
+                           video_backend=None):
     """
     Multithread or multiprocess generator worker process loop.
     """
@@ -546,6 +576,9 @@ def _generator_worker_loop(dataset, idx_queue, result_queue, eof, is_multiproces
     cde.register_worker_handlers()
 
     if is_multiprocessing:
+        if video_backend is not None:
+            set_video_backend(video_backend)
+
         result_queue.cancel_join_thread()  # Ensure that the process does not hang when exiting
 
         # init the random seed and np.random seed for the subprocess
@@ -634,6 +667,9 @@ class _GeneratorWorkerMt(threading.Thread):
             return False
         return True
 
+    def queue_full(self):
+        return self.idx_queue.full()
+
 
 class _GeneratorWorkerMp(multiprocessing.Process):
     """
@@ -647,8 +683,9 @@ class _GeneratorWorkerMp(multiprocessing.Process):
         else:
             self.res_queue = multiprocessing.Queue(queue_size)
         self.idx_queue.cancel_join_thread()  # Ensure that the process does not hang when exiting
+        video_backend = get_video_backend() if multiprocessing.get_start_method() == "spawn" else None
         super().__init__(target=_generator_worker_loop,
-                         args=(dataset, self.idx_queue, self.res_queue, eof, True, worker_id, ppid),
+                         args=(dataset, self.idx_queue, self.res_queue, eof, True, worker_id, ppid, video_backend),
                          name="GeneratorWorkerProcess" + str(worker_id))
 
     def put(self, item):
@@ -674,6 +711,9 @@ class _GeneratorWorkerMp(multiprocessing.Process):
             return False
         return True
 
+    def queue_full(self):
+        return self.idx_queue.full()
+
     def __del__(self):
         # del all the Queue & SharedQueue when the iter had been deleted from ITERATORS_LIST
         if hasattr(self, 'idx_queue'):
@@ -776,24 +816,11 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
         collate_fn (Callable[List[numpy.ndarray]], optional): Define how to merge a list of data into a batch.
             Only valid if `batch_sampler` is used. Default: ``None`` , do not use collation function.
 
-    Raises:
-        RuntimeError: If source raises an exception during execution.
-        RuntimeError: If len of column_names does not match output len of source.
-        ValueError: If `num_parallel_workers` exceeds the max thread numbers.
-        ValueError: If sampler and shuffle are specified at the same time.
-        ValueError: If sampler and sharding are specified at the same time.
-        ValueError: If `num_shards` is specified but shard_id is None.
-        ValueError: If shard_id is specified but `num_shards` is None.
-        ValueError: If `shard_id` is not in range of [0, `num_shards` ).
-        TypeError: If `batch_sampler` is not iterable.
-        ValueError: If `batch_sampler` is specified together with `num_samples` ,
-            `shuffle` , `num_shards` , `shard_id` and `sampler`.
-        TypeError: If `collate_fn` is not callable.
-        ValueError: If `collate_fn` is specified while `batch_sampler` is None.
-
-    Tutorial Examples:
-        - `Load & Process Data With Dataset Pipeline
-          <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_
+    .. warning::
+        `GeneratorDataset` uses `dill` module implicitly in multiprocessing `spawn` mode to serialize/deserialize
+        `source`, which is known to be insecure. It is possible to construct malicious pickle data which will
+        execute arbitrary code during unpickling. Never load data that could have come from untrusted sources,
+        or has been tampered with.
 
     Note:
         - If you configure `python_multiprocessing=True` (Default: ``True`` ) and `num_parallel_workers>1`
@@ -843,6 +870,21 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
 
     .. include:: mindspore.dataset.sampler.txt
 
+    Raises:
+        RuntimeError: If source raises an exception during execution.
+        RuntimeError: If len of column_names does not match output len of source.
+        ValueError: If `num_parallel_workers` exceeds the max thread numbers.
+        ValueError: If sampler and shuffle are specified at the same time.
+        ValueError: If sampler and sharding are specified at the same time.
+        ValueError: If `num_shards` is specified but shard_id is None.
+        ValueError: If shard_id is specified but `num_shards` is None.
+        ValueError: If `shard_id` is not in range of [0, `num_shards` ).
+        ValueError: If `batch_sampler` is specified together with `num_samples` ,
+            `shuffle` , `num_shards` , `shard_id` and `sampler`.
+        ValueError: If `collate_fn` is specified while `batch_sampler` is None.
+        TypeError: If `batch_sampler` is not iterable.
+        TypeError: If `collate_fn` is not callable.
+
     Examples:
         >>> import mindspore.dataset as ds
         >>> import numpy as np
@@ -901,6 +943,10 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
         >>>
         >>> # list, dict, tuple of Python is also random accessible
         >>> dataset = ds.GeneratorDataset(source=[(np.array(0),), (np.array(1),), (np.array(2),)], column_names=["col"])
+
+    Tutorial Examples:
+        - `Load & Process Data With Dataset Pipeline
+          <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_
     """
 
     @check_generator_dataset

mindspore/dataset/engine/iterators.py

@@ -220,7 +220,7 @@ class Iterator:
         dataset: Dataset to be iterated over
     """
 
-    def __init__(self, dataset, num_epochs=-1, output_numpy=False, do_copy=True):
+    def __init__(self, dataset, num_epochs=-1, output_numpy=False, do_copy=False):
         self._col_names = None
 
         # create a copy of tree and work on it.
@@ -493,7 +493,7 @@ class TupleIterator(Iterator):
     The derived class of Iterator with list type.
     """
 
-    def __init__(self, dataset, columns=None, num_epochs=-1, output_numpy=False, do_copy=True):
+    def __init__(self, dataset, columns=None, num_epochs=-1, output_numpy=False, do_copy=False):
         if columns is not None:
             if not isinstance(columns, list):
                 columns = [columns]

mindspore/dataset/engine/obs/config_loader.py

@@ -57,12 +57,12 @@ class _Config:
 
     def _convert_type(self, key):
         if key not in self.config:
-            return  os.environ[key]
+            return os.environ[key]
         if isinstance(self.config[key], int):
             return int(os.environ[key])
         if isinstance(self.config[key], float):
             return float(os.environ[key])
-        return  os.environ[key]
+        return os.environ[key]
 
 
 config = _Config()

mindspore/dataset/engine/obs/obs_mindrecord_dataset.py

@@ -506,3 +506,11 @@ class MindRecordFromOBS:
         path = os.path.join(self._local_path, target_dataset)
         _iteration = MindDataset(dataset_files=[path], shuffle=False)
         return _iteration.get_col_names()
+
+    def close(self):
+        if self._pool:
+            self._pool.terminate()
+            self._pool = None
+
+    def __del__(self):
+        self.close()

mindspore/dataset/transforms/c_transforms.py

@@ -286,7 +286,7 @@ class Mask(TensorOperation):
         operator (Relational): relational operators, it can be any of [Relational.EQ, Relational.NE, Relational.LT,
             Relational.GT, Relational.LE, Relational.GE], take Relational.EQ as example, EQ refers to equal.
         constant (Union[str, int, float, bool]): Constant to be compared to.
-        dtype (mindspore.dtype, optional): Type of the generated mask. Default: mstype.bool_.
+        dtype (mindspore.dtype, optional): Type of the generated mask. Default: mstype.bool.
 
     Raises:
         TypeError: `operator` is not of type Relational.
@@ -315,7 +315,7 @@ class Mask(TensorOperation):
 
     @deprecated_c_transforms()
     @check_mask_op
-    def __init__(self, operator, constant, dtype=mstype.bool_):
+    def __init__(self, operator, constant, dtype=mstype.bool):
         self.operator = operator
         self.dtype = mstype_to_detype(dtype)
         self.constant = cde.Tensor(np.array(constant))

mindspore/dataset/transforms/py_transforms.py

@@ -51,10 +51,14 @@ class PyTensorOperation:
         if "transforms" in json_obj.keys():
             # operations which have transforms as input, need to call _from_json() for each transform to deseriallize
             transforms = []
+            valid_module = ['mindspore.dataset.vision', 'mindspore.dataset.text',
+                            'mindspore.dataset.audio', 'mindspore.dataset.transforms']
             for json_op in json_obj["transforms"]:
-                transforms.append(getattr(
-                    sys.modules.get(json_op.get("python_module")),
-                    json_op.get("tensor_op_name")).from_json(json.dumps(json_op.get("tensor_op_params"))))
+                py_module = sys.modules.get(json_op.get("python_module"))
+                if py_module.__package__ not in valid_module:
+                    raise RuntimeError('Invalid json content, try to serialzie dataset again.')
+                transforms.append(getattr(py_module, json_op.get("tensor_op_name")).from_json(
+                    json.dumps(json_op.get("tensor_op_params"))))
             new_op.transforms = transforms
         if "output_type" in json_obj.keys():
             output_type = np.dtype(json_obj["output_type"])

mindspore/dataset/transforms/transforms.py

@@ -165,10 +165,14 @@ class PyTensorOperation:
         if "transforms" in json_obj.keys():
             # operations which have transforms as input, need to call _from_json() for each transform to deseriallize
             transforms = []
+            valid_module = ['mindspore.dataset.vision', 'mindspore.dataset.text',
+                            'mindspore.dataset.audio', 'mindspore.dataset.transforms']
             for json_op in json_obj["transforms"]:
-                transforms.append(getattr(
-                    sys.modules.get(json_op.get("python_module")),
-                    json_op["tensor_op_name"]).from_json(json.dumps(json_op["tensor_op_params"])))
+                py_module = sys.modules.get(json_op.get("python_module"))
+                if py_module.__package__ not in valid_module:
+                    raise RuntimeError('Invalid json content, try to serialzie dataset again.')
+                transforms.append(getattr(py_module, json_op["tensor_op_name"]).from_json(
+                    json.dumps(json_op["tensor_op_params"])))
             new_op.transforms = transforms
         if "output_type" in json_obj.keys():
             output_type = np.dtype(json_obj["output_type"])
@@ -609,7 +613,7 @@ class Mask(TensorOperation):
             ``Relational.GT``, ``Relational.LE``, ``Relational.GE``, take ``Relational.EQ`` as example,
             EQ refers to equal.
         constant (Union[str, int, float, bool]): Constant to be compared to.
-        dtype (mindspore.dtype, optional): Type of the generated mask. Default: ``mstype.bool_``.
+        dtype (mindspore.dtype, optional): Type of the generated mask. Default: ``mstype.bool``.
 
     Raises:
         TypeError: `operator` is not of type Relational.
@@ -650,7 +654,7 @@ class Mask(TensorOperation):
     """
 
     @check_mask_op_new
-    def __init__(self, operator, constant, dtype=mstype.bool_):
+    def __init__(self, operator, constant, dtype=mstype.bool):
         super().__init__()
         self.operator = operator
         self.dtype = mstype_to_detype(dtype)
@@ -1167,7 +1171,7 @@ class TypeCast(TensorOperation):
             to be cast to.
 
     Raises:
-        TypeError: If `data_type` is not of MindSpore data type bool, int, float, string or type :class:`numpy.dtype` .
+        TypeError: If `data_type` is not of MindSpore data type bool, int, float, string or type `numpy.dtype` .
 
     Supported Platforms:
         ``CPU`` ``GPU`` ``Ascend``

mindspore/dataset/vision/__init__.py

@@ -54,5 +54,5 @@ from .transforms import AdjustBrightness, AdjustContrast, AdjustGamma, AdjustHue
     ResizeWithBBox, RgbToHsv, Rotate, SlicePatches, Solarize, TenCrop, ToNumpy, ToPIL, ToTensor, ToType, \
     TrivialAugmentWide, UniformAugment, VerticalFlip, not_random
 from .utils import AutoAugmentPolicy, Border, ConvertMode, ImageBatchFormat, ImageReadMode, Inter, SliceMode, \
-    encode_jpeg, encode_png, get_image_num_channels, get_image_size, read_file, read_image, read_video, \
+    VideoDecoder, encode_jpeg, encode_png, get_image_num_channels, get_image_size, read_file, read_image, read_video, \
     read_video_timestamps, write_file, write_jpeg, write_png

mindspore/dataset/vision/py_transforms.py

@@ -493,7 +493,7 @@ class HWC2CHW(py_transforms.PyTensorOperation):
     If the input image is of shape <H, W>, it will remain unchanged.
 
     Raises:
-        TypeError: If the input image is not of type :class:`numpy.ndarray` .
+        TypeError: If the input image is not of type `numpy.ndarray` .
         TypeError: If shape of the input image is not <H, W> or <H, W, C>.
 
     Supported Platforms:
@@ -575,8 +575,8 @@ class LinearTransformation(py_transforms.PyTensorOperation):
         mean_vector (numpy.ndarray): A mean vector in shape of (D,), where :math:`D = C \times H \times W` .
 
     Raises:
-        TypeError: If `transformation_matrix` is not of type :class:`numpy.ndarray` .
-        TypeError: If `mean_vector` is not of type :class:`numpy.ndarray` .
+        TypeError: If `transformation_matrix` is not of type `numpy.ndarray` .
+        TypeError: If `mean_vector` is not of type `numpy.ndarray` .
 
     Supported Platforms:
         ``CPU``
@@ -711,7 +711,7 @@ class Normalize(py_transforms.PyTensorOperation):
             and be arranged in channel order.
 
     Raises:
-        TypeError: If the input image is not of type :class:`numpy.ndarray` .
+        TypeError: If the input image is not of type `numpy.ndarray` .
         TypeError: If dimension of the input image is not 3.
         NotImplementedError: If dtype of the input image is int.
         ValueError: If lengths of `mean` and `std` are not equal.
@@ -780,7 +780,7 @@ class NormalizePad(py_transforms.PyTensorOperation):
             Default: ``"float32"``.
 
     Raises:
-        TypeError: If the input image is not of type :class:`numpy.ndarray` .
+        TypeError: If the input image is not of type `numpy.ndarray` .
         TypeError: If dimension of the input image is not 3.
         NotImplementedError: If dtype of the input image is int.
         ValueError: If lengths of `mean` and `std` are not equal.
@@ -1930,7 +1930,7 @@ class ToPIL(py_transforms.PyTensorOperation):
         The conversion mode will be determined by the data type using `PIL.Image.fromarray` .
 
     Raises:
-        TypeError: If the input image is not of type :class:`numpy.ndarray` or `PIL.Image.Image` .
+        TypeError: If the input image is not of type `numpy.ndarray` or `PIL.Image.Image` .
 
     Supported Platforms:
         ``CPU``
@@ -1974,7 +1974,7 @@ class ToTensor(py_transforms.PyTensorOperation):
         output_type (numpy.dtype, optional): The desired dtype of the output image. Default: ``numpy.float32`` .
 
     Raises:
-        TypeError: If the input image is not of type `PIL.Image.Image` or :class:`numpy.ndarray` .
+        TypeError: If the input image is not of type `PIL.Image.Image` or `numpy.ndarray` .
         TypeError: If dimension of the input image is not 2 or 3.
 
     Supported Platforms:
@@ -2018,7 +2018,7 @@ class ToType(py_transforms.PyTensorOperation):
         output_type (numpy.dtype): The desired dtype of the output image, e.g. ``numpy.float32`` .
 
     Raises:
-        TypeError: If the input image is not of type :class:`numpy.ndarray` .
+        TypeError: If the input image is not of type `numpy.ndarray` .
 
     Supported Platforms:
         ``CPU``

mindspore/dataset/vision/transforms.py

@@ -2759,8 +2759,8 @@ class LinearTransformation(PyTensorOperation):
         mean_vector (numpy.ndarray): A mean vector in shape of (D,), where :math:`D = C \times H \times W` .
 
     Raises:
-        TypeError: If `transformation_matrix` is not of type :class:`numpy.ndarray` .
-        TypeError: If `mean_vector` is not of type :class:`numpy.ndarray` .
+        TypeError: If `transformation_matrix` is not of type `numpy.ndarray` .
+        TypeError: If `mean_vector` is not of type `numpy.ndarray` .
 
     Supported Platforms:
         ``CPU``
@@ -6911,7 +6911,7 @@ class ToPIL(PyTensorOperation):
     Convert the input decoded numpy.ndarray image to PIL Image.
 
     Raises:
-        TypeError: If the input image is not of type :class:`numpy.ndarray` or `PIL.Image.Image` .
+        TypeError: If the input image is not of type `numpy.ndarray` or `PIL.Image.Image` .
 
     Supported Platforms:
         ``CPU``
@@ -6975,7 +6975,7 @@ class ToTensor(ImageTensorOperation):
             Default: ``np.float32`` .
 
     Raises:
-        TypeError: If the input image is not of type `PIL.Image.Image` or :class:`numpy.ndarray` .
+        TypeError: If the input image is not of type `PIL.Image.Image` or `numpy.ndarray` .
         TypeError: If dimension of the input image is not 2 or 3.
 
     Supported Platforms:
@@ -7025,6 +7025,18 @@ class ToTensor(ImageTensorOperation):
     def parse(self):
         return cde.ToTensorOperation(self.output_type)
 
+    def _execute_py(self, img):
+        """
+        Execute method.
+
+        Args:
+            img (PIL Image): Convert PIL Image to np.ndarray by method HWC to CHW and divide 255.
+
+        Returns:
+            Image converted to numpy.ndarray.
+        """
+        return util.to_tensor(img, self.output_type)
+
 
 class ToType(TypeCast):
     """
@@ -7041,7 +7053,7 @@ class ToType(TypeCast):
             such as ``numpy.float32`` .
 
     Raises:
-        TypeError: If `data_type` is not of type :class:`mindspore.dtype` or :class:`numpy.dtype` .
+        TypeError: If `data_type` is not of type :class:`mindspore.dtype` or `numpy.dtype` .
 
     Supported Platforms:
         ``CPU`` ``GPU`` ``Ascend``