mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/dataset/engine/iterators.py

@@ -19,6 +19,7 @@ import json
 import os
 import signal
 import weakref
+from functools import wraps
 import numpy as np
 
 import mindspore._c_dataengine as cde
@@ -58,6 +59,34 @@ def _cleanup():
             itr.release()
 
 
+def _cleanup_the_iterators_if_created(method):
+    """Release the iterators which is new created by the method"""
+
+    @wraps(method)
+    def wrapper(self, *args, **kwargs):
+        original_iterators = deepcopy(ITERATORS_LIST)
+
+        result = method(self, *args, **kwargs)
+
+        # it is used to attribute function like: dataset_size / output_shapes / output_types and
+        # it is a GeneratorDataset with two stage pipeline. The first pipeline will create a new iterator
+        # which need to be released after dataset_size / output_shapes / output_types end.
+        # 1. find the iterators which are started by dataset_size / output_shapes / output_types with two stage pipeline
+        iterators_to_be_released = []
+        for index, item in enumerate(ITERATORS_LIST):
+            if item not in original_iterators:
+                iterators_to_be_released.append(index)
+
+        # 2. release the iterators
+        for index in reversed(iterators_to_be_released):
+            itr = ITERATORS_LIST[index]()
+            if itr is not None:
+                itr.release()
+
+        return result
+    return wrapper
+
+
 class Iterator:
     """
     General Iterator over a dataset.

mindspore/dataset/engine/obs/util.py

@@ -47,6 +47,13 @@ def get_used_disk_per():
             os.makedirs(config.WORKING_PATH)
         except FileExistsError:
             pass
+        except PermissionError as e:
+            err_msg =\
+                str(e) + ". Suggestion: " \
+                "1) It is recommended to manually create the {} directory and add read/write permissions to it. " \
+                "2) If you can't create, we suggest you download MindRecord manually and "\
+                "read it using the MindDataset interface.".format(config.WORKING_PATH)
+            raise RuntimeError(err_msg)
 
     total, used, _ = shutil.disk_usage(config.WORKING_PATH)
     return used / total

mindspore/dataset/engine/offload.py

@@ -46,13 +46,11 @@ def check_concat_zip_dataset(dataset):
     """
     Check if dataset is concatenated or zipped.
     """
-    while dataset:
-        if len(dataset.children) > 1:
-            raise RuntimeError("Offload module currently does not support concatenated or zipped datasets.")
-        if dataset.children:
-            dataset = dataset.children[0]
-            continue
-        dataset = dataset.children
+    child_len = len(dataset.children)
+    if child_len == 1:
+        check_concat_zip_dataset(dataset.children[0])
+    elif child_len > 1:
+        raise RuntimeError("Offload module currently does not support concatenated or zipped datasets.")
 
 
 def get_col_idxs(node_cols, ds_cols):

mindspore/dataset/engine/queue.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
+# Copyright 2021-2024 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,57 +20,74 @@ but it will pass large data through shared memory.
 
 import errno
 import multiprocessing
+import platform
 import queue
 import types
 
 import numpy as np
 
 from mindspore import log as logger
+import mindspore._c_dataengine as cde
 from ..transforms.py_transforms_util import ExceptionHandler
 
 
+def get_total_size(data):
+    """Calculate the total size of numpy arrays."""
+    total_size = 0
+    for column in data:
+        if isinstance(column, np.ndarray):
+            total_size += column.nbytes
+    return total_size
+
+
 class _SharedQueue(multiprocessing.queues.Queue):
     """
     Class to implement a queue using shared memory for better performance.
     Args:
         size: Number of elements in the queue.
         count: Shared variable to suppress log printing.
-        copy_out: Flag to indidcate whether an extra copy should be done before returning.  If data will immediately be
-                  copied before returning, then this can be set to False.
-        max_rowsize: Maximum size of any element in the Queue in MB.
+        copy_out: Whether to copy the data from shared memory to process virtual memory. Default: ``True``.
+        max_rowsize: Maximum size of row in MB that is used for shared memory allocation to copy
+            data between processes. If set to -1, shared memory will be dynamically allocated with
+            the actual size of data. Default: -1.
     """
 
-    def __init__(self, size, count, copy_out=False, max_rowsize=6):
+    def __init__(self, size, count, copy_out=True, max_rowsize=-1):
         super().__init__(size, ctx=multiprocessing.get_context())
 
         self.copy_out = copy_out
 
-        # change max_rowsize in MB into bytes
-        self.seg_size = max_rowsize * 1024 * 1024
         # pipe can hold up to 65,636 bytes at a time
         # there is less benefit for small data. To small data it can be slower as we need to pass 100 bytes of metadata
         # and then access the shared memory.
         self.min_shared_mem = 10000
-        self.shm_list = []
-        self.seg_pos = 0
-        # num_seg has to be 2 more than the queue size.  We can have remote worker filling a buffer, main process
-        # reading a buffer and also have a full queue of buffers in the meta-data queue
-        self.num_seg = size + 2
         self.data_immediate = 0
         self.data_shared = 1
         self.count = count
         self.print_error = True
+        self.shm_list = []
+        self.seg_pos = 0
+        # num_seg has to be 2 more than the queue size. We can have remote worker filling a buffer, main process
+        # reading a buffer and also have a full queue of buffers in the meta-data queue
+        self.num_seg = size + 2
 
-        for _ in range(self.num_seg):
-            try:
-                a = multiprocessing.Array("b", self.seg_size)
-            except OSError as e:
-                if e.errno == errno.ENOMEM:
-                    raise RuntimeError("Failed to allocate shared memory for {0} elements of {1}MB: {2}"
-                                       .format(self.num_seg, self.seg_size / 1024 / 1024, e))
-                raise
-            else:
-                self.shm_list.append(a)
+        if platform.system().lower() != 'windows' and max_rowsize == -1:
+            self.dynamic_shm = True
+            self.fd_list = []
+        else:
+            self.dynamic_shm = False
+            # change max_rowsize in MB into bytes
+            self.seg_size = max_rowsize * 1024 * 1024
+            for _ in range(self.num_seg):
+                try:
+                    shared_array = multiprocessing.Array("b", self.seg_size)
+                except OSError as e:
+                    if e.errno == errno.ENOMEM:
+                        raise RuntimeError("Failed to allocate shared memory for {0} elements of {1}MB: {2}"
+                                           .format(self.num_seg, self.seg_size / 1024 / 1024, e))
+                    raise
+                else:
+                    self.shm_list.append(shared_array)
 
     def put_until(self, data, timeout=None, exit_signal=None):
         """Put data into the queue. Block until timeout is reached or exit_signal is set."""
@@ -90,32 +107,42 @@ class _SharedQueue(multiprocessing.queues.Queue):
             super().put(data, timeout=timeout)
         else:
             name_list = []
-            count = 0
             start_bytes = 0
             if not isinstance(data, tuple):
                 data = (data,)
             if isinstance(data, np.ndarray):
                 name_list.append((self.data_immediate, np.array(data)))
             else:
-                for r in data:
-                    # the map:pyfunc is a yield generator which can't be serialize
-                    if isinstance(r, types.GeneratorType):
+                if self.dynamic_shm:
+                    total_size = get_total_size(data)
+                    if total_size > 0:
+                        self.check_and_create_shm(total_size)
+                for column in data:
+                    # the map:pyfunc is a yield generator which can't be serialized
+                    if isinstance(column, types.GeneratorType):
                         raise TypeError("Cannot pickle {} object, please verify pyfunc return with numpy array"
-                                        .format(type(r)))
-                    if (isinstance(r, np.ndarray) and r.size > self.min_shared_mem
-                            and start_bytes + r.nbytes < self.seg_size):
+                                        .format(type(column)))
+                    if self.dynamic_shm and isinstance(column, np.ndarray) and column.nbytes > 0:
+                        shm = self.shm_list[self.seg_pos]
+                        fd = self.fd_list[self.seg_pos]
+                        dest = np.ndarray(column.shape, column.dtype, buffer=shm.buf(), offset=start_bytes)
+                        np.copyto(dest, column)
+                        start_bytes += column.nbytes
+                        shm_metadata = (shm.name(), fd, total_size)
+                        name_list.append((self.data_shared, self.seg_pos, column.dtype, column.shape, shm_metadata))
+                    elif (isinstance(column, np.ndarray) and column.size > self.min_shared_mem
+                          and start_bytes + column.nbytes < self.seg_size):
                         # need to convert start_bytes to offset in array
                         start_offset = start_bytes
-                        dest = np.ndarray(r.shape, r.dtype, buffer=self.shm_list[self.seg_pos].get_obj(),
-                                          offset=start_offset)
-                        np.copyto(dest, r)
-                        byte = r.nbytes
+                        shm = self.shm_list[self.seg_pos]
+                        dest = np.ndarray(column.shape, column.dtype, buffer=shm.get_obj(), offset=start_offset)
+                        np.copyto(dest, column)
+                        byte = column.nbytes
                         byte = 8 * ((byte + 7) // 8)
                         start_bytes += byte
-                        name_list.append((self.data_shared, self.seg_pos, byte, r.dtype, r.shape))
-                        count += 1
+                        name_list.append((self.data_shared, self.seg_pos, byte, column.dtype, column.shape))
                     else:
-                        if isinstance(r, np.ndarray) and r.size > self.min_shared_mem:
+                        if isinstance(column, np.ndarray) and column.size > self.min_shared_mem:
                             # Only print out error the first time it happens
                             if self.count.value == 0 and self.print_error:
                                 logger.warning(
@@ -123,12 +150,12 @@ class _SharedQueue(multiprocessing.queues.Queue):
                                     + "max_rowsize: "
                                     + str(self.seg_size / 1024 / 1024)
                                     + "MB, current rowsize: "
-                                    + str((start_bytes + r.nbytes) / 1024 / 1024)
+                                    + str((start_bytes + column.nbytes) / 1024 / 1024)
                                     + "MB."
                                 )
                                 self.print_error = False
                                 self.count.value += 1
-                        name_list.append((self.data_immediate, r))
+                        name_list.append((self.data_immediate, column))
             super().put(name_list, timeout=timeout)
             # note above could generate a queue full exception.  It will be handled by teh caller
             # only increment seg_pos after successfully adding to metadata queue
@@ -140,54 +167,99 @@ class _SharedQueue(multiprocessing.queues.Queue):
         """Get data from the queue. Block until timeout is reached or exit_signal is set."""
         while True:
             try:
-                r = self.get(timeout=timeout)
+                result = self.get(timeout=timeout)
             except queue.Empty as e:
                 if exit_signal is None:
                     raise e
                 if exit_signal.is_set():
                     return None
                 continue
-            if r is None:
+            if result is None:
                 # receive finish signal
                 return None
             if exit_signal.is_set():
                 # loop until the queue becomes empty
                 continue
-            return r
+            return result
 
     def get(self, timeout=None):
-        result = super().get(timeout=timeout)
-        if isinstance(result, ExceptionHandler):
-            return result
-        r = []
+        raw_data = super().get(timeout=timeout)
+        if isinstance(raw_data, ExceptionHandler):
+            return raw_data
+        result = []
         start_bytes = 0
-        for x in result:
-            if x[0] == self.data_shared:
-                seg_pos = x[1]
-                byte = x[2]
-                dtype = x[3]
-                shape = x[4]
-                start_offset = start_bytes
-                b = self.shm_list[seg_pos]
-                data = np.ndarray(shape, dtype, buffer=b.get_obj(), offset=start_offset)
-                start_bytes += byte
+        for column in raw_data:
+            if column[0] == self.data_shared:
+                if self.dynamic_shm:
+                    seg_pos, dtype, shape, shm_metadata = column[1:]
+                    if start_bytes == 0:
+                        # only need to check once since all the columns are stored in the same shared memory
+                        self.check_and_attach_shm(seg_pos, shm_metadata)
+                    shm = self.shm_list[seg_pos]
+                    array = np.ndarray(shape, dtype, buffer=shm.buf(), offset=start_bytes)
+                    start_bytes += array.nbytes
+                else:
+                    seg_pos, byte, dtype, shape = column[1:]
+                    start_offset = start_bytes
+                    shm = self.shm_list[seg_pos]
+                    array = np.ndarray(shape, dtype, buffer=shm.get_obj(), offset=start_offset)
+                    start_bytes += byte
                 if self.copy_out:
-                    data2 = np.copy(data)
-                    r.append(data2)
+                    result.append(np.copy(array))
                 else:
-                    r.append(data)
-            elif x[0] == self.data_immediate:
-                r.append(x[1])
+                    result.append(array)
+            elif column[0] == self.data_immediate:
+                result.append(column[1])
             else:
                 raise RuntimeError("SharedQueue, invalid entry in metadata.")
-        return tuple(r)
+        return tuple(result)
+
+    def check_and_create_shm(self, size):
+        """Check if the shared memory is initialized and of sufficient size."""
+        if len(self.shm_list) == self.seg_pos:
+            # shared memory has not been created and appended to the cache list
+            shm = cde.SharedMemory(None, True, -1, size)
+            shared_fd = multiprocessing.reduction.DupFd(shm.fd())
+            self.shm_list.append(shm)
+            self.fd_list.append(shared_fd)
+        elif len(self.shm_list) > self.seg_pos:
+            if self.shm_list[self.seg_pos].size() < size:
+                # shared memory is not big enough to hold the data
+                shm = cde.SharedMemory(None, True, -1, size)
+                shared_fd = multiprocessing.reduction.DupFd(shm.fd())
+                self.shm_list[self.seg_pos] = shm
+                self.fd_list[self.seg_pos] = shared_fd
+        else:
+            raise RuntimeError("The shared memory index is larger than the length of shared memory list. "
+                               "Uninitialized shared memory may exist.")
+
+    def check_and_attach_shm(self, shm_index, shm_metadata):
+        """Check if the shared memory is initialized and is the same as the current one."""
+        shm_name, fd, size = shm_metadata
+        if len(self.shm_list) == shm_index:
+            # shared memory has not been created and appended to the cache list
+            fd = fd.detach()
+            shm = cde.SharedMemory(shm_name, False, fd, size)
+            self.shm_list.append(shm)
+            self.fd_list.append(fd)
+        elif len(self.shm_list) > shm_index:
+            if self.shm_list[shm_index].name() != shm_name:
+                # shared memory has changed
+                fd = fd.detach()
+                shm = cde.SharedMemory(shm_name, False, fd, size)
+                self.shm_list[shm_index] = shm
+                self.fd_list[shm_index] = fd
+        else:
+            raise RuntimeError("The shared memory index is larger than the length of shared memory list. "
+                               "Uninitialized shared memory may exist.")
 
     def __del__(self):
-        shm_list_len = len(self.shm_list)
-        for idx in range(shm_list_len):
-            del self.shm_list[shm_list_len - idx - 1]
-        self.shm_list.clear()
-        del self.shm_list
+        if not self.dynamic_shm:
+            shm_list_len = len(self.shm_list)
+            for idx in range(shm_list_len):
+                del self.shm_list[shm_list_len - idx - 1]
+            self.shm_list.clear()
+            del self.shm_list
 
         self.close()
         self.join_thread()

mindspore/dataset/engine/serializer_deserializer.py

@@ -107,9 +107,9 @@ def deserialize(input_dict=None, json_filepath=None):
 def expand_path(node_repr, key, val):
     """Convert relative to absolute path."""
     if isinstance(val, list):
-        node_repr[key] = [os.path.abspath(file) for file in val]
+        node_repr[key] = [os.path.realpath(file) for file in val]
     else:
-        node_repr[key] = os.path.abspath(val)
+        node_repr[key] = os.path.realpath(val)
 
 
 def show(dataset, indentation=2):

mindspore/dataset/engine/validators.py

@@ -27,7 +27,7 @@ from ..core.validator_helpers import parse_user_args, type_check, type_check_lis
     INT32_MAX, check_valid_detype, check_dir, check_file, check_sampler_shuffle_shard_options, \
     validate_dataset_param_value, check_padding_options, \
     check_num_parallel_workers, check_columns, check_pos_int32, check_valid_str, check_dataset_num_shards_shard_id, \
-    check_valid_list_tuple
+    check_valid_list_tuple, check_int32, check_independent_mode
 
 from . import datasets
 from . import samplers
@@ -1021,7 +1021,8 @@ def check_minddataset(method):
         dataset_file = param_dict.get('dataset_files')
         if isinstance(dataset_file, list):
             if len(dataset_file) > 4096:
-                log.warning("The number of MindRecord files greater than 4096 may cause slow dataset initialization.")
+                logger.warning("The number of MindRecord files greater than 4096"
+                               "may cause slow dataset initialization.")
             for f in dataset_file:
                 check_file(f)
         else:
@@ -1034,6 +1035,7 @@ def check_minddataset(method):
         check_sampler_shuffle_shard_options(param_dict)
 
         check_padding_options(param_dict)
+        check_cache_option(param_dict.get('cache'))
         return method(self, *args, **kwargs)
 
     return new_method
@@ -1113,7 +1115,10 @@ def check_generatordataset(method):
         nreq_param_bool = ["shuffle", "python_multiprocessing"]
         validate_dataset_param_value(nreq_param_bool, param_dict, bool)
 
-        check_pos_int32(param_dict.get("max_rowsize"), "max_rowsize")
+        max_rowsize = param_dict.get("max_rowsize")
+        if max_rowsize is not None:
+            check_value(max_rowsize, [-1, INT32_MAX], "max_rowsize")
+
         num_shards = param_dict.get("num_shards")
         shard_id = param_dict.get("shard_id")
         check_dataset_num_shards_shard_id(num_shards, shard_id)
@@ -1280,7 +1285,7 @@ def get_batch_kwargs_from_dict(param_dict):
         input_columns = param_dict.get("input_columns", None)
         output_columns = param_dict.get("output_columns", None)
         python_multiprocessing = param_dict.get("python_multiprocessing", False)
-        max_rowsize = param_dict.get("max_rowsize", 16)
+        max_rowsize = param_dict.get("max_rowsize", None)
     return per_batch_map, input_columns, output_columns, python_multiprocessing, max_rowsize
 
 
@@ -1371,6 +1376,19 @@ def check_sync_wait(method):
         type_check(condition_name, (str,), "condition_name")
         type_check(num_batch, (int,), "num_batch")
 
+        check_independent_mode("Dataset sync wait")
+
+        return method(self, *args, **kwargs)
+
+    return new_method
+
+
+def check_sync_update(method):
+    """check the input arguments of sync_update."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        check_independent_mode("Dataset sync update")
         return method(self, *args, **kwargs)
 
     return new_method
@@ -1396,7 +1414,7 @@ def get_map_kwargs_from_dict(param_dict):
     """get map operation kwargs parameters."""
     if param_dict is not None:
         python_multiprocessing = param_dict.get("python_multiprocessing", False)
-        max_rowsize = param_dict.get("max_rowsize", 16)
+        max_rowsize = param_dict.get("max_rowsize", None)
         cache = param_dict.get("cache", None)
         callbacks = param_dict.get("callbacks", None)
         offload = param_dict.get("offload", None)
@@ -1405,16 +1423,17 @@ def get_map_kwargs_from_dict(param_dict):
 
 def check_max_rowsize(max_rowsize):
     """check the max_rowsize"""
-    type_check(max_rowsize, (int, list), "max_rowsize")
-    if isinstance(max_rowsize, int):
-        type_check(max_rowsize, (int,), "max_rowsize")
-        check_pos_int32(max_rowsize, "max_rowsize")
-    elif isinstance(max_rowsize, list) and len(max_rowsize) == 2:
-        for index, value in enumerate(max_rowsize):
-            type_check(value, (int,), "max_rowsize[{}]".format(index))
-            check_pos_int32(value, "max_rowsizei[{}]".format(index))
-    else:
-        raise TypeError("max_rowsize should be a single integer or a list[in_rowsize, out_rowsize] of length 2.")
+    if max_rowsize is not None:
+        type_check(max_rowsize, (int, list), "max_rowsize")
+        if isinstance(max_rowsize, int):
+            type_check(max_rowsize, (int,), "max_rowsize")
+            check_value(max_rowsize, [-1, INT32_MAX], "max_rowsize")
+        elif isinstance(max_rowsize, list) and len(max_rowsize) == 2:
+            for index, value in enumerate(max_rowsize):
+                type_check(value, (int,), "max_rowsize[{}]".format(index))
+                check_value(value, [-1, INT32_MAX], "max_rowsizei[{}]".format(index))
+        else:
+            raise TypeError("max_rowsize should be a single integer or a list[in_rowsize, out_rowsize] of length 2.")
 
 
 def check_map(method):
@@ -1475,12 +1494,14 @@ def check_map(method):
         check_max_rowsize(max_rowsize)
         if offload is not None:
             type_check(offload, (bool,), "offload")
+            check_independent_mode("Dataset Offload", offload)
 
         if callbacks is not None:
             if isinstance(callbacks, (list, tuple)):
                 type_check_list(callbacks, (DSCallback,), "callbacks")
             else:
                 type_check(callbacks, (DSCallback,), "callbacks")
+            check_independent_mode("Dataset Callbacks")
 
         for param_name, param in zip(nreq_param_columns, [input_columns, output_columns]):
             if param is not None:
@@ -1594,6 +1615,10 @@ def check_device_send(method):
     return new_method
 
 
+def check_total_batch(total_batch):
+    check_int32(total_batch, "total_batch")
+
+
 def check_zip(method):
     """check the input arguments of zip."""
 
@@ -2014,6 +2039,7 @@ def check_cache_option(cache):
     """Sanity check for cache parameter"""
     if cache is not None:
         type_check(cache, (cache_client.DatasetCache,), "cache")
+        check_independent_mode("Dataset Cache")
 
 
 def check_to_device_send(method):

mindspore/dataset/text/__init__.py

@@ -24,9 +24,6 @@ Common imported modules in corresponding API examples are as follows:
     import mindspore.dataset as ds
     import mindspore.dataset.text as text
 
-See `Text Transforms
-<https://www.mindspore.cn/tutorials/en/r2.2/beginner/transforms.html#text-transforms>`_ tutorial for more details.
-
 Descriptions of common data processing terms are as follows:
 
 - TensorOperation, the base class of all data processing operations implemented in C++.
@@ -35,10 +32,10 @@ Descriptions of common data processing terms are as follows:
 The data transform operation can be executed in the data processing pipeline or in the eager mode:
 
 - Pipeline mode is generally used to process big datasets. Examples refer to
-  `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/r2.2/api_python/
+  `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/master/api_python/
   mindspore.dataset.html#introduction-to-data-processing-pipeline>`_ .
 - Eager mode is more like a function call to process data. Examples refer to
-  `Lightweight Data Processing <https://www.mindspore.cn/tutorials/en/r2.2/advanced/dataset/eager.html>`_ .
+  `Lightweight Data Processing <https://www.mindspore.cn/docs/en/master/model_train/dataset/eager.html>`_ .
 """
 import platform