mindspore 2.7.0rc1__cp310-cp310-win_amd64.whl → 2.7.1__cp310-cp310-win_amd64.whl

mindspore/dataset/engine/datasets_user_defined.py

@@ -48,7 +48,8 @@ from .datasets import UnionBaseDataset, MappableDataset, Schema, to_list, _Pytho
 from .queue import _SharedQueue
 from .validators import check_generator_dataset, check_numpy_slices_dataset, check_padded_dataset
 from ..core.config import get_enable_shared_mem, get_prefetch_size, get_multiprocessing_timeout_interval, \
-    get_enable_watchdog, get_debug_mode, get_seed, set_seed, get_multiprocessing_start_method
+    get_enable_watchdog, get_debug_mode, get_seed, set_seed, get_multiprocessing_start_method, get_video_backend, \
+    set_video_backend
 from ..core.datatypes import mstypelist_to_detypelist
 from ..core.py_util_helpers import ExceptionHandler
 from ..core.validator_helpers import type_check
@@ -521,6 +522,10 @@ class SamplerFn(cde.PythonMultiprocessingRuntime):
             cde.deregister_worker_pids(id(self))
         if hasattr(self, 'eof') and self.eof is not None:
             self.eof.set()
+            # send QUIT flag to workers, and the worker's while loop could check the eof flag
+            for worker in self.workers:
+                if not worker.queue_full():
+                    worker.put("QUIT")
         if hasattr(self, 'cleaning_process') and self.cleaning_process is not None:
             # let the quit event notify the cleaning process to exit
             self.cleaning_process.join(timeout=5)
@@ -562,7 +567,8 @@ def _main_process_already_exit(eof, is_multiprocessing, idx_queue, result_queue,
     return False
 
 
-def _generator_worker_loop(dataset, idx_queue, result_queue, eof, is_multiprocessing, worker_id, ppid=-1):
+def _generator_worker_loop(dataset, idx_queue, result_queue, eof, is_multiprocessing, worker_id, ppid=-1,
+                           video_backend=None):
     """
     Multithread or multiprocess generator worker process loop.
     """
@@ -570,6 +576,9 @@ def _generator_worker_loop(dataset, idx_queue, result_queue, eof, is_multiproces
     cde.register_worker_handlers()
 
     if is_multiprocessing:
+        if video_backend is not None:
+            set_video_backend(video_backend)
+
         result_queue.cancel_join_thread()  # Ensure that the process does not hang when exiting
 
         # init the random seed and np.random seed for the subprocess
@@ -658,6 +667,9 @@ class _GeneratorWorkerMt(threading.Thread):
             return False
         return True
 
+    def queue_full(self):
+        return self.idx_queue.full()
+
 
 class _GeneratorWorkerMp(multiprocessing.Process):
     """
@@ -671,8 +683,9 @@ class _GeneratorWorkerMp(multiprocessing.Process):
         else:
             self.res_queue = multiprocessing.Queue(queue_size)
         self.idx_queue.cancel_join_thread()  # Ensure that the process does not hang when exiting
+        video_backend = get_video_backend() if multiprocessing.get_start_method() == "spawn" else None
         super().__init__(target=_generator_worker_loop,
-                         args=(dataset, self.idx_queue, self.res_queue, eof, True, worker_id, ppid),
+                         args=(dataset, self.idx_queue, self.res_queue, eof, True, worker_id, ppid, video_backend),
                          name="GeneratorWorkerProcess" + str(worker_id))
 
     def put(self, item):
@@ -698,6 +711,9 @@ class _GeneratorWorkerMp(multiprocessing.Process):
             return False
         return True
 
+    def queue_full(self):
+        return self.idx_queue.full()
+
     def __del__(self):
         # del all the Queue & SharedQueue when the iter had been deleted from ITERATORS_LIST
         if hasattr(self, 'idx_queue'):
@@ -800,25 +816,6 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
         collate_fn (Callable[List[numpy.ndarray]], optional): Define how to merge a list of data into a batch.
             Only valid if `batch_sampler` is used. Default: ``None`` , do not use collation function.
 
-    Raises:
-        RuntimeError: If source raises an exception during execution.
-        RuntimeError: If len of column_names does not match output len of source.
-        ValueError: If `num_parallel_workers` exceeds the max thread numbers.
-        ValueError: If sampler and shuffle are specified at the same time.
-        ValueError: If sampler and sharding are specified at the same time.
-        ValueError: If `num_shards` is specified but shard_id is None.
-        ValueError: If shard_id is specified but `num_shards` is None.
-        ValueError: If `shard_id` is not in range of [0, `num_shards` ).
-        TypeError: If `batch_sampler` is not iterable.
-        ValueError: If `batch_sampler` is specified together with `num_samples` ,
-            `shuffle` , `num_shards` , `shard_id` and `sampler`.
-        TypeError: If `collate_fn` is not callable.
-        ValueError: If `collate_fn` is specified while `batch_sampler` is None.
-
-    Tutorial Examples:
-        - `Load & Process Data With Dataset Pipeline
-          <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_
-
     .. warning::
         `GeneratorDataset` uses `dill` module implicitly in multiprocessing `spawn` mode to serialize/deserialize
         `source`, which is known to be insecure. It is possible to construct malicious pickle data which will
@@ -873,6 +870,21 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
 
     .. include:: mindspore.dataset.sampler.txt
 
+    Raises:
+        RuntimeError: If source raises an exception during execution.
+        RuntimeError: If len of column_names does not match output len of source.
+        ValueError: If `num_parallel_workers` exceeds the max thread numbers.
+        ValueError: If sampler and shuffle are specified at the same time.
+        ValueError: If sampler and sharding are specified at the same time.
+        ValueError: If `num_shards` is specified but shard_id is None.
+        ValueError: If shard_id is specified but `num_shards` is None.
+        ValueError: If `shard_id` is not in range of [0, `num_shards` ).
+        ValueError: If `batch_sampler` is specified together with `num_samples` ,
+            `shuffle` , `num_shards` , `shard_id` and `sampler`.
+        ValueError: If `collate_fn` is specified while `batch_sampler` is None.
+        TypeError: If `batch_sampler` is not iterable.
+        TypeError: If `collate_fn` is not callable.
+
     Examples:
         >>> import mindspore.dataset as ds
         >>> import numpy as np
@@ -931,6 +943,10 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
         >>>
         >>> # list, dict, tuple of Python is also random accessible
         >>> dataset = ds.GeneratorDataset(source=[(np.array(0),), (np.array(1),), (np.array(2),)], column_names=["col"])
+
+    Tutorial Examples:
+        - `Load & Process Data With Dataset Pipeline
+          <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_
     """
 
     @check_generator_dataset

mindspore/dataset/engine/datasets_vision.py

@@ -2179,7 +2179,7 @@ class ImageFolderDataset(MappableDataset, VisionBaseDataset):
           <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_
 
     Note:
-        - The shape of the image column is [image_size] if `decode` flag is ``False``, or [H,W,C] otherwise.
+        - The shape of the image column is [undecoded_image_size] if `decode` flag is ``False``, or [H,W,C] otherwise.
         - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
           used in the dataset, and their effects when combined with parameter `sampler` are as follows.
 

mindspore/dataset/engine/validators.py

@@ -1481,23 +1481,9 @@ def check_map(method):
     @wraps(method)
     def new_method(self, *args, **kwargs):
         from mindspore.dataset.callback import DSCallback
-        [operations, input_columns, output_columns, column_order, num_parallel_workers, param_dict], _ = \
+        [operations, input_columns, output_columns, num_parallel_workers, param_dict], _ = \
             parse_user_args(method, *args, **kwargs)
 
-        if column_order is not None:
-            raise ValueError("The parameter 'column_order' had been deleted in map operation. "
-                             "Please use '.project' operation instead.\n"
-                             ">> # Usage of old api:\n"
-                             ">> dataset = dataset.map(operations=PyFunc,\n"
-                             ">>                       input_columns=[\"column_a\"],\n"
-                             ">>                       output_columns=[\"column_b\", \"column_c\"],\n"
-                             ">>                       column_order=[\"column_b\", \"column_c\"])\n"
-                             ">> # Usage of new api:\n"
-                             ">> dataset = dataset.map(operations=PyFunc,\n"
-                             ">>                       input_columns=[\"column_a\"],\n"
-                             ">>                       output_columns=[\"column_b\", \"column_c\"])\n"
-                             ">> dataset = dataset.project([\"column_b\", \"column_c\"])")
-
         (python_multiprocessing, max_rowsize, cache, callbacks, offload) = get_map_kwargs_from_dict(param_dict)
 
         # check whether network computing operator exist in input operations(python function)

mindspore/dataset/transforms/c_transforms.py

@@ -286,7 +286,7 @@ class Mask(TensorOperation):
         operator (Relational): relational operators, it can be any of [Relational.EQ, Relational.NE, Relational.LT,
             Relational.GT, Relational.LE, Relational.GE], take Relational.EQ as example, EQ refers to equal.
         constant (Union[str, int, float, bool]): Constant to be compared to.
-        dtype (mindspore.dtype, optional): Type of the generated mask. Default: mstype.bool_.
+        dtype (mindspore.dtype, optional): Type of the generated mask. Default: mstype.bool.
 
     Raises:
         TypeError: `operator` is not of type Relational.
@@ -315,7 +315,7 @@ class Mask(TensorOperation):
 
     @deprecated_c_transforms()
     @check_mask_op
-    def __init__(self, operator, constant, dtype=mstype.bool_):
+    def __init__(self, operator, constant, dtype=mstype.bool):
         self.operator = operator
         self.dtype = mstype_to_detype(dtype)
         self.constant = cde.Tensor(np.array(constant))

mindspore/dataset/transforms/transforms.py

@@ -613,7 +613,7 @@ class Mask(TensorOperation):
             ``Relational.GT``, ``Relational.LE``, ``Relational.GE``, take ``Relational.EQ`` as example,
             EQ refers to equal.
         constant (Union[str, int, float, bool]): Constant to be compared to.
-        dtype (mindspore.dtype, optional): Type of the generated mask. Default: ``mstype.bool_``.
+        dtype (mindspore.dtype, optional): Type of the generated mask. Default: ``mstype.bool``.
 
     Raises:
         TypeError: `operator` is not of type Relational.
@@ -654,7 +654,7 @@ class Mask(TensorOperation):
     """
 
     @check_mask_op_new
-    def __init__(self, operator, constant, dtype=mstype.bool_):
+    def __init__(self, operator, constant, dtype=mstype.bool):
         super().__init__()
         self.operator = operator
         self.dtype = mstype_to_detype(dtype)
@@ -1171,7 +1171,7 @@ class TypeCast(TensorOperation):
             to be cast to.
 
     Raises:
-        TypeError: If `data_type` is not of MindSpore data type bool, int, float, string or type :class:`numpy.dtype` .
+        TypeError: If `data_type` is not of MindSpore data type bool, int, float, string or type `numpy.dtype` .
 
     Supported Platforms:
         ``CPU`` ``GPU`` ``Ascend``

mindspore/dataset/vision/__init__.py

@@ -54,5 +54,5 @@ from .transforms import AdjustBrightness, AdjustContrast, AdjustGamma, AdjustHue
     ResizeWithBBox, RgbToHsv, Rotate, SlicePatches, Solarize, TenCrop, ToNumpy, ToPIL, ToTensor, ToType, \
     TrivialAugmentWide, UniformAugment, VerticalFlip, not_random
 from .utils import AutoAugmentPolicy, Border, ConvertMode, ImageBatchFormat, ImageReadMode, Inter, SliceMode, \
-    encode_jpeg, encode_png, get_image_num_channels, get_image_size, read_file, read_image, read_video, \
+    VideoDecoder, encode_jpeg, encode_png, get_image_num_channels, get_image_size, read_file, read_image, read_video, \
     read_video_timestamps, write_file, write_jpeg, write_png

mindspore/dataset/vision/py_transforms.py

@@ -493,7 +493,7 @@ class HWC2CHW(py_transforms.PyTensorOperation):
     If the input image is of shape <H, W>, it will remain unchanged.
 
     Raises:
-        TypeError: If the input image is not of type :class:`numpy.ndarray` .
+        TypeError: If the input image is not of type `numpy.ndarray` .
         TypeError: If shape of the input image is not <H, W> or <H, W, C>.
 
     Supported Platforms:
@@ -575,8 +575,8 @@ class LinearTransformation(py_transforms.PyTensorOperation):
         mean_vector (numpy.ndarray): A mean vector in shape of (D,), where :math:`D = C \times H \times W` .
 
     Raises:
-        TypeError: If `transformation_matrix` is not of type :class:`numpy.ndarray` .
-        TypeError: If `mean_vector` is not of type :class:`numpy.ndarray` .
+        TypeError: If `transformation_matrix` is not of type `numpy.ndarray` .
+        TypeError: If `mean_vector` is not of type `numpy.ndarray` .
 
     Supported Platforms:
         ``CPU``
@@ -711,7 +711,7 @@ class Normalize(py_transforms.PyTensorOperation):
             and be arranged in channel order.
 
     Raises:
-        TypeError: If the input image is not of type :class:`numpy.ndarray` .
+        TypeError: If the input image is not of type `numpy.ndarray` .
         TypeError: If dimension of the input image is not 3.
         NotImplementedError: If dtype of the input image is int.
         ValueError: If lengths of `mean` and `std` are not equal.
@@ -780,7 +780,7 @@ class NormalizePad(py_transforms.PyTensorOperation):
             Default: ``"float32"``.
 
     Raises:
-        TypeError: If the input image is not of type :class:`numpy.ndarray` .
+        TypeError: If the input image is not of type `numpy.ndarray` .
         TypeError: If dimension of the input image is not 3.
         NotImplementedError: If dtype of the input image is int.
         ValueError: If lengths of `mean` and `std` are not equal.
@@ -1930,7 +1930,7 @@ class ToPIL(py_transforms.PyTensorOperation):
         The conversion mode will be determined by the data type using `PIL.Image.fromarray` .
 
     Raises:
-        TypeError: If the input image is not of type :class:`numpy.ndarray` or `PIL.Image.Image` .
+        TypeError: If the input image is not of type `numpy.ndarray` or `PIL.Image.Image` .
 
     Supported Platforms:
         ``CPU``
@@ -1974,7 +1974,7 @@ class ToTensor(py_transforms.PyTensorOperation):
         output_type (numpy.dtype, optional): The desired dtype of the output image. Default: ``numpy.float32`` .
 
     Raises:
-        TypeError: If the input image is not of type `PIL.Image.Image` or :class:`numpy.ndarray` .
+        TypeError: If the input image is not of type `PIL.Image.Image` or `numpy.ndarray` .
         TypeError: If dimension of the input image is not 2 or 3.
 
     Supported Platforms:
@@ -2018,7 +2018,7 @@ class ToType(py_transforms.PyTensorOperation):
         output_type (numpy.dtype): The desired dtype of the output image, e.g. ``numpy.float32`` .
 
     Raises:
-        TypeError: If the input image is not of type :class:`numpy.ndarray` .
+        TypeError: If the input image is not of type `numpy.ndarray` .
 
     Supported Platforms:
         ``CPU``

mindspore/dataset/vision/transforms.py

@@ -2759,8 +2759,8 @@ class LinearTransformation(PyTensorOperation):
         mean_vector (numpy.ndarray): A mean vector in shape of (D,), where :math:`D = C \times H \times W` .
 
     Raises:
-        TypeError: If `transformation_matrix` is not of type :class:`numpy.ndarray` .
-        TypeError: If `mean_vector` is not of type :class:`numpy.ndarray` .
+        TypeError: If `transformation_matrix` is not of type `numpy.ndarray` .
+        TypeError: If `mean_vector` is not of type `numpy.ndarray` .
 
     Supported Platforms:
         ``CPU``
@@ -6911,7 +6911,7 @@ class ToPIL(PyTensorOperation):
     Convert the input decoded numpy.ndarray image to PIL Image.
 
     Raises:
-        TypeError: If the input image is not of type :class:`numpy.ndarray` or `PIL.Image.Image` .
+        TypeError: If the input image is not of type `numpy.ndarray` or `PIL.Image.Image` .
 
     Supported Platforms:
         ``CPU``
@@ -6975,7 +6975,7 @@ class ToTensor(ImageTensorOperation):
             Default: ``np.float32`` .
 
     Raises:
-        TypeError: If the input image is not of type `PIL.Image.Image` or :class:`numpy.ndarray` .
+        TypeError: If the input image is not of type `PIL.Image.Image` or `numpy.ndarray` .
         TypeError: If dimension of the input image is not 2 or 3.
 
     Supported Platforms:
@@ -7025,6 +7025,18 @@ class ToTensor(ImageTensorOperation):
     def parse(self):
         return cde.ToTensorOperation(self.output_type)
 
+    def _execute_py(self, img):
+        """
+        Execute method.
+
+        Args:
+            img (PIL Image): Convert PIL Image to np.ndarray by method HWC to CHW and divide 255.
+
+        Returns:
+            Image converted to numpy.ndarray.
+        """
+        return util.to_tensor(img, self.output_type)
+
 
 class ToType(TypeCast):
     """
@@ -7041,7 +7053,7 @@ class ToType(TypeCast):
             such as ``numpy.float32`` .
 
     Raises:
-        TypeError: If `data_type` is not of type :class:`mindspore.dtype` or :class:`numpy.dtype` .
+        TypeError: If `data_type` is not of type :class:`mindspore.dtype` or `numpy.dtype` .
 
     Supported Platforms:
         ``CPU`` ``GPU`` ``Ascend``