mindspore 2.3.0__cp39-none-any.whl → 2.3.0rc2__cp39-none-any.whl

mindspore/dataset/engine/queue.py

@@ -20,14 +20,12 @@ but it will pass large data through shared memory.
 
 import errno
 import multiprocessing
-import platform
 import queue
 import types
 
 import numpy as np
 
 from mindspore import log as logger
-import mindspore._c_dataengine as cde
 from ..transforms.py_transforms_util import ExceptionHandler
 
 
@@ -47,36 +45,32 @@ class _SharedQueue(multiprocessing.queues.Queue):
 
         self.copy_out = copy_out
 
+        # change max_rowsize in MB into bytes
+        self.seg_size = max_rowsize * 1024 * 1024
         # pipe can hold up to 65,636 bytes at a time
         # there is less benefit for small data. To small data it can be slower as we need to pass 100 bytes of metadata
         # and then access the shared memory.
         self.min_shared_mem = 10000
+        self.shm_list = []
+        self.seg_pos = 0
+        # num_seg has to be 2 more than the queue size.  We can have remote worker filling a buffer, main process
+        # reading a buffer and also have a full queue of buffers in the meta-data queue
+        self.num_seg = size + 2
         self.data_immediate = 0
         self.data_shared = 1
         self.count = count
         self.print_error = True
 
-        if platform.system().lower() != 'windows' and max_rowsize == -1:
-            self.dynamic_shm = True
-        else:
-            self.dynamic_shm = False
-            # change max_rowsize in MB into bytes
-            self.seg_size = max_rowsize * 1024 * 1024
-            self.shm_list = []
-            self.seg_pos = 0
-            # num_seg has to be 2 more than the queue size.  We can have remote worker filling a buffer, main process
-            # reading a buffer and also have a full queue of buffers in the meta-data queue
-            self.num_seg = size + 2
-            for _ in range(self.num_seg):
-                try:
-                    a = multiprocessing.Array("b", self.seg_size)
-                except OSError as e:
-                    if e.errno == errno.ENOMEM:
-                        raise RuntimeError("Failed to allocate shared memory for {0} elements of {1}MB: {2}"
-                                           .format(self.num_seg, self.seg_size / 1024 / 1024, e))
-                    raise
-                else:
-                    self.shm_list.append(a)
+        for _ in range(self.num_seg):
+            try:
+                a = multiprocessing.Array("b", self.seg_size)
+            except OSError as e:
+                if e.errno == errno.ENOMEM:
+                    raise RuntimeError("Failed to allocate shared memory for {0} elements of {1}MB: {2}"
+                                       .format(self.num_seg, self.seg_size / 1024 / 1024, e))
+                raise
+            else:
+                self.shm_list.append(a)
 
     def put_until(self, data, timeout=None, exit_signal=None):
         """Put data into the queue. Block until timeout is reached or exit_signal is set."""
@@ -96,6 +90,7 @@ class _SharedQueue(multiprocessing.queues.Queue):
             super().put(data, timeout=timeout)
         else:
             name_list = []
+            count = 0
             start_bytes = 0
             if not isinstance(data, tuple):
                 data = (data,)
@@ -107,16 +102,8 @@ class _SharedQueue(multiprocessing.queues.Queue):
                     if isinstance(r, types.GeneratorType):
                         raise TypeError("Cannot pickle {} object, please verify pyfunc return with numpy array"
                                         .format(type(r)))
-                    if isinstance(r, np.ndarray) and self.dynamic_shm:
-                        byte = r.nbytes
-                        shm = cde.SharedMemory(None, True, -1, byte)
-                        dest = np.ndarray(r.shape, r.dtype, buffer=shm.buf())
-                        np.copyto(dest, r)
-                        fd = shm.fd()
-                        df = multiprocessing.reduction.DupFd(fd)
-                        name_list.append((self.data_shared, r.dtype, r.shape, shm.name(), df, shm.size()))
-                    elif (isinstance(r, np.ndarray) and r.size > self.min_shared_mem
-                          and start_bytes + r.nbytes < self.seg_size):
+                    if (isinstance(r, np.ndarray) and r.size > self.min_shared_mem
+                            and start_bytes + r.nbytes < self.seg_size):
                         # need to convert start_bytes to offset in array
                         start_offset = start_bytes
                         dest = np.ndarray(r.shape, r.dtype, buffer=self.shm_list[self.seg_pos].get_obj(),
@@ -126,6 +113,7 @@ class _SharedQueue(multiprocessing.queues.Queue):
                         byte = 8 * ((byte + 7) // 8)
                         start_bytes += byte
                         name_list.append((self.data_shared, self.seg_pos, byte, r.dtype, r.shape))
+                        count += 1
                     else:
                         if isinstance(r, np.ndarray) and r.size > self.min_shared_mem:
                             # Only print out error the first time it happens
@@ -175,24 +163,19 @@ class _SharedQueue(multiprocessing.queues.Queue):
         start_bytes = 0
         for x in result:
             if x[0] == self.data_shared:
-                if self.dynamic_shm:
-                    dtype, shape, shm_name, df, buf_size = x[1:]
-                    fd = df.detach()
-                    shm = cde.SharedMemory(shm_name, False, fd, buf_size)
-                    data = np.ndarray(shape, dtype, buffer=shm.buf())
-                    dest = np.copy(data)
-                    r.append(dest)
+                seg_pos = x[1]
+                byte = x[2]
+                dtype = x[3]
+                shape = x[4]
+                start_offset = start_bytes
+                b = self.shm_list[seg_pos]
+                data = np.ndarray(shape, dtype, buffer=b.get_obj(), offset=start_offset)
+                start_bytes += byte
+                if self.copy_out:
+                    data2 = np.copy(data)
+                    r.append(data2)
                 else:
-                    seg_pos, byte, dtype, shape = x[1:]
-                    start_offset = start_bytes
-                    b = self.shm_list[seg_pos]
-                    data = np.ndarray(shape, dtype, buffer=b.get_obj(), offset=start_offset)
-                    start_bytes += byte
-                    if self.copy_out:
-                        dest = np.copy(data)
-                        r.append(dest)
-                    else:
-                        r.append(data)
+                    r.append(data)
             elif x[0] == self.data_immediate:
                 r.append(x[1])
             else:
@@ -200,12 +183,11 @@ class _SharedQueue(multiprocessing.queues.Queue):
         return tuple(r)
 
     def __del__(self):
-        if not self.dynamic_shm:
-            shm_list_len = len(self.shm_list)
-            for idx in range(shm_list_len):
-                del self.shm_list[shm_list_len - idx - 1]
-            self.shm_list.clear()
-            del self.shm_list
+        shm_list_len = len(self.shm_list)
+        for idx in range(shm_list_len):
+            del self.shm_list[shm_list_len - idx - 1]
+        self.shm_list.clear()
+        del self.shm_list
 
         self.close()
         self.join_thread()

mindspore/dataset/engine/validators.py

@@ -27,7 +27,7 @@ from ..core.validator_helpers import parse_user_args, type_check, type_check_lis
     INT32_MAX, check_valid_detype, check_dir, check_file, check_sampler_shuffle_shard_options, \
     validate_dataset_param_value, check_padding_options, \
     check_num_parallel_workers, check_columns, check_pos_int32, check_valid_str, check_dataset_num_shards_shard_id, \
-    check_valid_list_tuple, check_int32
+    check_valid_list_tuple
 
 from . import datasets
 from . import samplers
@@ -1021,8 +1021,7 @@ def check_minddataset(method):
         dataset_file = param_dict.get('dataset_files')
         if isinstance(dataset_file, list):
             if len(dataset_file) > 4096:
-                logger.warning("The number of MindRecord files greater than 4096"
-                               "may cause slow dataset initialization.")
+                log.warning("The number of MindRecord files greater than 4096 may cause slow dataset initialization.")
             for f in dataset_file:
                 check_file(f)
         else:
@@ -1114,8 +1113,7 @@ def check_generatordataset(method):
         nreq_param_bool = ["shuffle", "python_multiprocessing"]
         validate_dataset_param_value(nreq_param_bool, param_dict, bool)
 
-        check_value(param_dict.get("max_rowsize"), [-1, INT32_MAX], "max_rowsize")
-
+        check_pos_int32(param_dict.get("max_rowsize"), "max_rowsize")
         num_shards = param_dict.get("num_shards")
         shard_id = param_dict.get("shard_id")
         check_dataset_num_shards_shard_id(num_shards, shard_id)
@@ -1410,11 +1408,11 @@ def check_max_rowsize(max_rowsize):
     type_check(max_rowsize, (int, list), "max_rowsize")
     if isinstance(max_rowsize, int):
         type_check(max_rowsize, (int,), "max_rowsize")
-        check_value(max_rowsize, [-1, INT32_MAX], "max_rowsize")
+        check_pos_int32(max_rowsize, "max_rowsize")
     elif isinstance(max_rowsize, list) and len(max_rowsize) == 2:
         for index, value in enumerate(max_rowsize):
             type_check(value, (int,), "max_rowsize[{}]".format(index))
-            check_value(value, [-1, INT32_MAX], "max_rowsizei[{}]".format(index))
+            check_pos_int32(value, "max_rowsizei[{}]".format(index))
     else:
         raise TypeError("max_rowsize should be a single integer or a list[in_rowsize, out_rowsize] of length 2.")
 
@@ -1596,10 +1594,6 @@ def check_device_send(method):
     return new_method
 
 
-def check_total_batch(total_batch):
-    check_int32(total_batch, "total_batch")
-
-
 def check_zip(method):
     """check the input arguments of zip."""
 

mindspore/dataset/vision/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2020-2024 Huawei Technologies Co., Ltd
+# Copyright 2020-2022 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -54,8 +54,8 @@ from . import transforms
 from . import utils
 from .transforms import AdjustBrightness, AdjustContrast, AdjustGamma, AdjustHue, AdjustSaturation, AdjustSharpness, \
     Affine, AutoAugment, AutoContrast, BoundingBoxAugment, CenterCrop, ConvertColor, Crop, CutMixBatch, CutOut, \
-    Decode, DecodeVideo, Equalize, Erase, FiveCrop, GaussianBlur, Grayscale, HorizontalFlip, HsvToRgb, HWC2CHW, \
-    Invert, LinearTransformation, MixUp, MixUpBatch, Normalize, NormalizePad, Pad, PadToSize, Perspective, Posterize, \
+    Decode, Equalize, Erase, FiveCrop, GaussianBlur, Grayscale, HorizontalFlip, HsvToRgb, HWC2CHW, Invert, \
+    LinearTransformation, MixUp, MixUpBatch, Normalize, NormalizePad, Pad, PadToSize, Perspective, Posterize, \
     RandAugment, RandomAdjustSharpness, RandomAffine, RandomAutoContrast, RandomColor, RandomColorAdjust, RandomCrop, \
     RandomCropDecodeResize, RandomCropWithBBox, RandomEqualize, RandomErasing, RandomGrayscale, RandomHorizontalFlip, \
     RandomHorizontalFlipWithBBox, RandomInvert, RandomLighting, RandomPerspective, RandomPosterize, RandomResizedCrop, \
@@ -64,5 +64,5 @@ from .transforms import AdjustBrightness, AdjustContrast, AdjustGamma, AdjustHue
     ResizeWithBBox, RgbToHsv, Rotate, SlicePatches, Solarize, TenCrop, ToNumpy, ToPIL, ToTensor, ToType, \
     TrivialAugmentWide, UniformAugment, VerticalFlip, not_random
 from .utils import AutoAugmentPolicy, Border, ConvertMode, ImageBatchFormat, ImageReadMode, Inter, SliceMode, \
-    encode_jpeg, encode_png, get_image_num_channels, get_image_size, read_file, read_image, read_video, \
-    read_video_timestamps, write_file, write_jpeg, write_png
+    encode_jpeg, encode_png, get_image_num_channels, get_image_size, read_file, read_image, write_file, write_jpeg, \
+    write_png

mindspore/dataset/vision/c_transforms.py

@@ -283,7 +283,7 @@ class AutoContrast(ImageTensorOperation):
         self.ignore = ignore
 
     def parse(self):
-        return cde.AutoContrastOperation(self.cutoff, self.ignore, "CPU")
+        return cde.AutoContrastOperation(self.cutoff, self.ignore)
 
 
 class BoundingBoxAugment(ImageTensorOperation):
@@ -442,7 +442,7 @@ class ConvertColor(ImageTensorOperation):
         self.convert_mode = convert_mode
 
     def parse(self):
-        return cde.ConvertColorOperation(DE_C_CONVERT_COLOR_MODE.get(self.convert_mode), "CPU")
+        return cde.ConvertColorOperation(DE_C_CONVERT_COLOR_MODE.get(self.convert_mode))
 
 
 class Crop(ImageTensorOperation):
@@ -635,7 +635,7 @@ class Equalize(ImageTensorOperation):
         super().__init__()
 
     def parse(self):
-        return cde.EqualizeOperation("CPU")
+        return cde.EqualizeOperation()
 
 
 class GaussianBlur(ImageTensorOperation):
@@ -760,7 +760,7 @@ class Invert(ImageTensorOperation):
         super().__init__()
 
     def parse(self):
-        return cde.InvertOperation("CPU")
+        return cde.InvertOperation()
 
 
 class MixUpBatch(ImageTensorOperation):
@@ -2422,7 +2422,7 @@ class Rotate(ImageTensorOperation):
 
     def parse(self):
         return cde.RotateOperation(self.degrees, DE_C_INTER_MODE.get(self.resample), self.expand, self.center,
-                                   self.fill_value, "CPU")
+                                   self.fill_value)
 
 
 class SlicePatches(ImageTensorOperation):

mindspore/dataset/vision/py_transforms_util.py

@@ -1569,7 +1569,7 @@ def adjust_gamma(img, gamma, gain):
     if not is_pil(img):
         raise TypeError("img should be PIL image. Got {}.".format(type(img)))
 
-    gamma_table = [int((255 + 1 - 1e-3) * gain * pow(x / 255., gamma)) for x in range(256)]
+    gamma_table = [(255 + 1 - 1e-3) * gain * pow(x / 255., gamma) for x in range(256)]
     if len(img.split()) == 3:
         gamma_table = gamma_table * 3
         img = img.point(gamma_table)