mindspore 2.7.0__cp310-cp310-win_amd64.whl → 2.7.0rc1__cp310-cp310-win_amd64.whl

mindspore/dataset/engine/datasets_user_defined.py

@@ -48,8 +48,7 @@ from .datasets import UnionBaseDataset, MappableDataset, Schema, to_list, _Pytho
 from .queue import _SharedQueue
 from .validators import check_generator_dataset, check_numpy_slices_dataset, check_padded_dataset
 from ..core.config import get_enable_shared_mem, get_prefetch_size, get_multiprocessing_timeout_interval, \
-    get_enable_watchdog, get_debug_mode, get_seed, set_seed, get_multiprocessing_start_method, get_video_backend, \
-    set_video_backend
+    get_enable_watchdog, get_debug_mode, get_seed, set_seed, get_multiprocessing_start_method
 from ..core.datatypes import mstypelist_to_detypelist
 from ..core.py_util_helpers import ExceptionHandler
 from ..core.validator_helpers import type_check
@@ -522,10 +521,6 @@ class SamplerFn(cde.PythonMultiprocessingRuntime):
             cde.deregister_worker_pids(id(self))
         if hasattr(self, 'eof') and self.eof is not None:
             self.eof.set()
-            # send QUIT flag to workers, and the worker's while loop could check the eof flag
-            for worker in self.workers:
-                if not worker.queue_full():
-                    worker.put("QUIT")
         if hasattr(self, 'cleaning_process') and self.cleaning_process is not None:
             # let the quit event notify the cleaning process to exit
             self.cleaning_process.join(timeout=5)
@@ -567,8 +562,7 @@ def _main_process_already_exit(eof, is_multiprocessing, idx_queue, result_queue,
     return False
 
 
-def _generator_worker_loop(dataset, idx_queue, result_queue, eof, is_multiprocessing, worker_id, ppid=-1,
-                           video_backend=None):
+def _generator_worker_loop(dataset, idx_queue, result_queue, eof, is_multiprocessing, worker_id, ppid=-1):
     """
     Multithread or multiprocess generator worker process loop.
     """
@@ -576,9 +570,6 @@ def _generator_worker_loop(dataset, idx_queue, result_queue, eof, is_multiproces
     cde.register_worker_handlers()
 
     if is_multiprocessing:
-        if video_backend is not None:
-            set_video_backend(video_backend)
-
         result_queue.cancel_join_thread()  # Ensure that the process does not hang when exiting
 
         # init the random seed and np.random seed for the subprocess
@@ -667,9 +658,6 @@ class _GeneratorWorkerMt(threading.Thread):
             return False
         return True
 
-    def queue_full(self):
-        return self.idx_queue.full()
-
 
 class _GeneratorWorkerMp(multiprocessing.Process):
     """
@@ -683,9 +671,8 @@ class _GeneratorWorkerMp(multiprocessing.Process):
         else:
             self.res_queue = multiprocessing.Queue(queue_size)
         self.idx_queue.cancel_join_thread()  # Ensure that the process does not hang when exiting
-        video_backend = get_video_backend() if multiprocessing.get_start_method() == "spawn" else None
         super().__init__(target=_generator_worker_loop,
-                         args=(dataset, self.idx_queue, self.res_queue, eof, True, worker_id, ppid, video_backend),
+                         args=(dataset, self.idx_queue, self.res_queue, eof, True, worker_id, ppid),
                          name="GeneratorWorkerProcess" + str(worker_id))
 
     def put(self, item):
@@ -711,9 +698,6 @@ class _GeneratorWorkerMp(multiprocessing.Process):
             return False
         return True
 
-    def queue_full(self):
-        return self.idx_queue.full()
-
     def __del__(self):
         # del all the Queue & SharedQueue when the iter had been deleted from ITERATORS_LIST
         if hasattr(self, 'idx_queue'):
@@ -816,6 +800,25 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
         collate_fn (Callable[List[numpy.ndarray]], optional): Define how to merge a list of data into a batch.
             Only valid if `batch_sampler` is used. Default: ``None`` , do not use collation function.
 
+    Raises:
+        RuntimeError: If source raises an exception during execution.
+        RuntimeError: If len of column_names does not match output len of source.
+        ValueError: If `num_parallel_workers` exceeds the max thread numbers.
+        ValueError: If sampler and shuffle are specified at the same time.
+        ValueError: If sampler and sharding are specified at the same time.
+        ValueError: If `num_shards` is specified but shard_id is None.
+        ValueError: If shard_id is specified but `num_shards` is None.
+        ValueError: If `shard_id` is not in range of [0, `num_shards` ).
+        TypeError: If `batch_sampler` is not iterable.
+        ValueError: If `batch_sampler` is specified together with `num_samples` ,
+            `shuffle` , `num_shards` , `shard_id` and `sampler`.
+        TypeError: If `collate_fn` is not callable.
+        ValueError: If `collate_fn` is specified while `batch_sampler` is None.
+
+    Tutorial Examples:
+        - `Load & Process Data With Dataset Pipeline
+          <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_
+
     .. warning::
         `GeneratorDataset` uses `dill` module implicitly in multiprocessing `spawn` mode to serialize/deserialize
         `source`, which is known to be insecure. It is possible to construct malicious pickle data which will
@@ -870,21 +873,6 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
 
     .. include:: mindspore.dataset.sampler.txt
 
-    Raises:
-        RuntimeError: If source raises an exception during execution.
-        RuntimeError: If len of column_names does not match output len of source.
-        ValueError: If `num_parallel_workers` exceeds the max thread numbers.
-        ValueError: If sampler and shuffle are specified at the same time.
-        ValueError: If sampler and sharding are specified at the same time.
-        ValueError: If `num_shards` is specified but shard_id is None.
-        ValueError: If shard_id is specified but `num_shards` is None.
-        ValueError: If `shard_id` is not in range of [0, `num_shards` ).
-        ValueError: If `batch_sampler` is specified together with `num_samples` ,
-            `shuffle` , `num_shards` , `shard_id` and `sampler`.
-        ValueError: If `collate_fn` is specified while `batch_sampler` is None.
-        TypeError: If `batch_sampler` is not iterable.
-        TypeError: If `collate_fn` is not callable.
-
     Examples:
         >>> import mindspore.dataset as ds
         >>> import numpy as np
@@ -943,10 +931,6 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset):
         >>>
         >>> # list, dict, tuple of Python is also random accessible
         >>> dataset = ds.GeneratorDataset(source=[(np.array(0),), (np.array(1),), (np.array(2),)], column_names=["col"])
-
-    Tutorial Examples:
-        - `Load & Process Data With Dataset Pipeline
-          <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_
     """
 
     @check_generator_dataset

mindspore/dataset/transforms/c_transforms.py

@@ -286,7 +286,7 @@ class Mask(TensorOperation):
         operator (Relational): relational operators, it can be any of [Relational.EQ, Relational.NE, Relational.LT,
             Relational.GT, Relational.LE, Relational.GE], take Relational.EQ as example, EQ refers to equal.
         constant (Union[str, int, float, bool]): Constant to be compared to.
-        dtype (mindspore.dtype, optional): Type of the generated mask. Default: mstype.bool.
+        dtype (mindspore.dtype, optional): Type of the generated mask. Default: mstype.bool_.
 
     Raises:
         TypeError: `operator` is not of type Relational.
@@ -315,7 +315,7 @@ class Mask(TensorOperation):
 
     @deprecated_c_transforms()
     @check_mask_op
-    def __init__(self, operator, constant, dtype=mstype.bool):
+    def __init__(self, operator, constant, dtype=mstype.bool_):
         self.operator = operator
         self.dtype = mstype_to_detype(dtype)
         self.constant = cde.Tensor(np.array(constant))

mindspore/dataset/transforms/transforms.py

@@ -613,7 +613,7 @@ class Mask(TensorOperation):
             ``Relational.GT``, ``Relational.LE``, ``Relational.GE``, take ``Relational.EQ`` as example,
             EQ refers to equal.
         constant (Union[str, int, float, bool]): Constant to be compared to.
-        dtype (mindspore.dtype, optional): Type of the generated mask. Default: ``mstype.bool``.
+        dtype (mindspore.dtype, optional): Type of the generated mask. Default: ``mstype.bool_``.
 
     Raises:
         TypeError: `operator` is not of type Relational.
@@ -654,7 +654,7 @@ class Mask(TensorOperation):
     """
 
     @check_mask_op_new
-    def __init__(self, operator, constant, dtype=mstype.bool):
+    def __init__(self, operator, constant, dtype=mstype.bool_):
         super().__init__()
         self.operator = operator
         self.dtype = mstype_to_detype(dtype)
@@ -1171,7 +1171,7 @@ class TypeCast(TensorOperation):
             to be cast to.
 
     Raises:
-        TypeError: If `data_type` is not of MindSpore data type bool, int, float, string or type `numpy.dtype` .
+        TypeError: If `data_type` is not of MindSpore data type bool, int, float, string or type :class:`numpy.dtype` .
 
     Supported Platforms:
         ``CPU`` ``GPU`` ``Ascend``

mindspore/dataset/vision/__init__.py

@@ -54,5 +54,5 @@ from .transforms import AdjustBrightness, AdjustContrast, AdjustGamma, AdjustHue
     ResizeWithBBox, RgbToHsv, Rotate, SlicePatches, Solarize, TenCrop, ToNumpy, ToPIL, ToTensor, ToType, \
     TrivialAugmentWide, UniformAugment, VerticalFlip, not_random
 from .utils import AutoAugmentPolicy, Border, ConvertMode, ImageBatchFormat, ImageReadMode, Inter, SliceMode, \
-    VideoDecoder, encode_jpeg, encode_png, get_image_num_channels, get_image_size, read_file, read_image, read_video, \
+    encode_jpeg, encode_png, get_image_num_channels, get_image_size, read_file, read_image, read_video, \
     read_video_timestamps, write_file, write_jpeg, write_png

mindspore/dataset/vision/py_transforms.py

@@ -493,7 +493,7 @@ class HWC2CHW(py_transforms.PyTensorOperation):
     If the input image is of shape <H, W>, it will remain unchanged.
 
     Raises:
-        TypeError: If the input image is not of type `numpy.ndarray` .
+        TypeError: If the input image is not of type :class:`numpy.ndarray` .
         TypeError: If shape of the input image is not <H, W> or <H, W, C>.
 
     Supported Platforms:
@@ -575,8 +575,8 @@ class LinearTransformation(py_transforms.PyTensorOperation):
         mean_vector (numpy.ndarray): A mean vector in shape of (D,), where :math:`D = C \times H \times W` .
 
     Raises:
-        TypeError: If `transformation_matrix` is not of type `numpy.ndarray` .
-        TypeError: If `mean_vector` is not of type `numpy.ndarray` .
+        TypeError: If `transformation_matrix` is not of type :class:`numpy.ndarray` .
+        TypeError: If `mean_vector` is not of type :class:`numpy.ndarray` .
 
     Supported Platforms:
         ``CPU``
@@ -711,7 +711,7 @@ class Normalize(py_transforms.PyTensorOperation):
             and be arranged in channel order.
 
     Raises:
-        TypeError: If the input image is not of type `numpy.ndarray` .
+        TypeError: If the input image is not of type :class:`numpy.ndarray` .
         TypeError: If dimension of the input image is not 3.
         NotImplementedError: If dtype of the input image is int.
         ValueError: If lengths of `mean` and `std` are not equal.
@@ -780,7 +780,7 @@ class NormalizePad(py_transforms.PyTensorOperation):
             Default: ``"float32"``.
 
     Raises:
-        TypeError: If the input image is not of type `numpy.ndarray` .
+        TypeError: If the input image is not of type :class:`numpy.ndarray` .
         TypeError: If dimension of the input image is not 3.
         NotImplementedError: If dtype of the input image is int.
         ValueError: If lengths of `mean` and `std` are not equal.
@@ -1930,7 +1930,7 @@ class ToPIL(py_transforms.PyTensorOperation):
         The conversion mode will be determined by the data type using `PIL.Image.fromarray` .
 
     Raises:
-        TypeError: If the input image is not of type `numpy.ndarray` or `PIL.Image.Image` .
+        TypeError: If the input image is not of type :class:`numpy.ndarray` or `PIL.Image.Image` .
 
     Supported Platforms:
         ``CPU``
@@ -1974,7 +1974,7 @@ class ToTensor(py_transforms.PyTensorOperation):
         output_type (numpy.dtype, optional): The desired dtype of the output image. Default: ``numpy.float32`` .
 
     Raises:
-        TypeError: If the input image is not of type `PIL.Image.Image` or `numpy.ndarray` .
+        TypeError: If the input image is not of type `PIL.Image.Image` or :class:`numpy.ndarray` .
         TypeError: If dimension of the input image is not 2 or 3.
 
     Supported Platforms:
@@ -2018,7 +2018,7 @@ class ToType(py_transforms.PyTensorOperation):
         output_type (numpy.dtype): The desired dtype of the output image, e.g. ``numpy.float32`` .
 
     Raises:
-        TypeError: If the input image is not of type `numpy.ndarray` .
+        TypeError: If the input image is not of type :class:`numpy.ndarray` .
 
     Supported Platforms:
         ``CPU``

mindspore/dataset/vision/transforms.py

@@ -2759,8 +2759,8 @@ class LinearTransformation(PyTensorOperation):
         mean_vector (numpy.ndarray): A mean vector in shape of (D,), where :math:`D = C \times H \times W` .
 
     Raises:
-        TypeError: If `transformation_matrix` is not of type `numpy.ndarray` .
-        TypeError: If `mean_vector` is not of type `numpy.ndarray` .
+        TypeError: If `transformation_matrix` is not of type :class:`numpy.ndarray` .
+        TypeError: If `mean_vector` is not of type :class:`numpy.ndarray` .
 
     Supported Platforms:
         ``CPU``
@@ -6911,7 +6911,7 @@ class ToPIL(PyTensorOperation):
     Convert the input decoded numpy.ndarray image to PIL Image.
 
     Raises:
-        TypeError: If the input image is not of type `numpy.ndarray` or `PIL.Image.Image` .
+        TypeError: If the input image is not of type :class:`numpy.ndarray` or `PIL.Image.Image` .
 
     Supported Platforms:
         ``CPU``
@@ -6975,7 +6975,7 @@ class ToTensor(ImageTensorOperation):
             Default: ``np.float32`` .
 
     Raises:
-        TypeError: If the input image is not of type `PIL.Image.Image` or `numpy.ndarray` .
+        TypeError: If the input image is not of type `PIL.Image.Image` or :class:`numpy.ndarray` .
         TypeError: If dimension of the input image is not 2 or 3.
 
     Supported Platforms:
@@ -7025,18 +7025,6 @@ class ToTensor(ImageTensorOperation):
     def parse(self):
         return cde.ToTensorOperation(self.output_type)
 
-    def _execute_py(self, img):
-        """
-        Execute method.
-
-        Args:
-            img (PIL Image): Convert PIL Image to np.ndarray by method HWC to CHW and divide 255.
-
-        Returns:
-            Image converted to numpy.ndarray.
-        """
-        return util.to_tensor(img, self.output_type)
-
 
 class ToType(TypeCast):
     """
@@ -7053,7 +7041,7 @@ class ToType(TypeCast):
             such as ``numpy.float32`` .
 
     Raises:
-        TypeError: If `data_type` is not of type :class:`mindspore.dtype` or `numpy.dtype` .
+        TypeError: If `data_type` is not of type :class:`mindspore.dtype` or :class:`numpy.dtype` .
 
     Supported Platforms:
         ``CPU`` ``GPU`` ``Ascend``