mindspore 2.4.10__cp310-cp310-win_amd64.whl → 2.6.0rc1__cp310-cp310-win_amd64.whl

mindspore/dataset/engine/validators.py

@@ -16,9 +16,12 @@
 """
 Built-in validators.
 """
+
+import collections
 import inspect as ins
 import os
 from functools import wraps
+
 import numpy as np
 
 from mindspore._c_expression import typing
@@ -28,7 +31,6 @@ from ..core.validator_helpers import parse_user_args, type_check, type_check_lis
     validate_dataset_param_value, check_padding_options, \
     check_num_parallel_workers, check_columns, check_pos_int32, check_valid_str, check_dataset_num_shards_shard_id, \
     check_valid_list_tuple, check_int32, check_independent_mode
-
 from . import datasets
 from . import samplers
 from . import cache_client
@@ -976,7 +978,7 @@ def check_tuple_iterator(method):
     @wraps(method)
     def new_method(self, *args, **kwargs):
         [columns, num_epochs, _, _], param_dict = parse_user_args(method, *args, **kwargs)
-        nreq_param_bool = ['output_numpy']
+        nreq_param_bool = ['output_numpy', 'do_copy']
         validate_dataset_param_value(nreq_param_bool, param_dict, bool)
         if num_epochs is not None:
             type_check(num_epochs, (int,), "num_epochs")
@@ -996,7 +998,7 @@ def check_dict_iterator(method):
     @wraps(method)
     def new_method(self, *args, **kwargs):
         [num_epochs, _, _], param_dict = parse_user_args(method, *args, **kwargs)
-        nreq_param_bool = ['output_numpy']
+        nreq_param_bool = ['output_numpy', 'do_copy']
         validate_dataset_param_value(nreq_param_bool, param_dict, bool)
         if num_epochs is not None:
             type_check(num_epochs, (int,), "num_epochs")
@@ -1071,7 +1073,34 @@ def check_source_function(source):
     return str(var) + source_doc
 
 
-def check_generatordataset(method):
+def check_batch_sampler(batch_sampler, num_samples, shuffle, num_shards, shard_id, sampler, collate_fn):
+    """Check whether the params are valid with batch_sampler."""
+    if batch_sampler is not None:
+        if not all(param is None for param in [num_samples, shuffle, num_shards, shard_id, sampler]):
+            raise ValueError("batch_sampler is mutually exclusive with num_samples, shuffle, num_shards, "
+                             "shard_id and sampler.")
+        if not isinstance(batch_sampler, (samplers.BuiltinSampler, collections.abc.Iterable)):
+            raise TypeError("batch_sampler should have __getitem__ or __iter__ method.")
+
+    if collate_fn is not None:
+        if batch_sampler is None:
+            raise ValueError("collate_fn can be specified only when batch_sampler is set.")
+        if not isinstance(collate_fn, collections.abc.Callable):
+            raise TypeError("collate_fn should be callable.")
+
+
+def check_valid_map_style_dataset(source, sampler, batch_sampler, num_shards):
+    """Check whether the params are valid for map style dataset."""
+    if not hasattr(source, "__getitem__"):
+        if sampler is not None:
+            raise ValueError("sampler is not supported if source does not have attribute '__getitem__'.")
+        if batch_sampler is not None:
+            raise ValueError("batch_sampler is not supported if source does not have attribute '__getitem__'.")
+        if num_shards is not None:
+            raise ValueError("num_shards is not supported if source does not have attribute '__getitem__'.")
+
+
+def check_generator_dataset(method):
     """A wrapper that wraps a parameter checker around the original Dataset(GeneratorDataset)."""
 
     @wraps(method)
@@ -1104,9 +1133,13 @@ def check_generatordataset(method):
             raise ValueError("Neither columns_names nor schema are provided.")
 
         if schema is not None:
+            logger.warning("'schema' is deprecated from version 2.6 and will be removed in a future version.")
             if not isinstance(schema, (datasets.Schema, str)):
                 raise ValueError("schema should be a path to schema file or a schema object.")
 
+        if param_dict.get('column_types') is not None:
+            logger.warning("'column_types' is deprecated from version 2.6 and will be removed in a future version.")
+
         # check optional argument
         nreq_param_int = ["max_rowsize", "num_samples", "num_parallel_workers", "num_shards", "shard_id"]
         validate_dataset_param_value(nreq_param_int, param_dict, int)
@@ -1117,6 +1150,7 @@ def check_generatordataset(method):
 
         max_rowsize = param_dict.get("max_rowsize")
         if max_rowsize is not None:
+            logger.warning("'max_rowsize' is deprecated from version 2.6 and will be removed in a future version.")
             check_value(max_rowsize, [-1, INT32_MAX], "max_rowsize")
 
         num_shards = param_dict.get("num_shards")
@@ -1133,10 +1167,13 @@ def check_generatordataset(method):
                 except TypeError:
                     raise TypeError("sampler should be either iterable or from mindspore.dataset.samplers.")
 
-        if sampler is not None and not hasattr(source, "__getitem__"):
-            raise ValueError("sampler is not supported if source does not have attribute '__getitem__'.")
-        if num_shards is not None and not hasattr(source, "__getitem__"):
-            raise ValueError("num_shards is not supported if source does not have attribute '__getitem__'.")
+        batch_sampler = param_dict.get("batch_sampler")
+        num_samples = param_dict.get("num_samples")
+        shuffle = param_dict.get("shuffle")
+        collate_fn = param_dict.get("collate_fn")
+        check_batch_sampler(batch_sampler, num_samples, shuffle, num_shards, shard_id, sampler, collate_fn)
+
+        check_valid_map_style_dataset(source, sampler, batch_sampler, num_shards)
 
         return method(self, *args, **kwargs)
 
@@ -1306,13 +1343,15 @@ def check_batch(method):
             sig = ins.signature(batch_size)
             if len(sig.parameters) != 1:
                 raise ValueError("callable batch_size should take one parameter (BatchInfo).")
-        else:
+        elif batch_size != -1:
             check_pos_int32(int(batch_size), "batch_size")
 
         if num_parallel_workers is not None:
             check_num_parallel_workers(num_parallel_workers)
         type_check(drop_remainder, (bool,), "drop_remainder")
 
+        if max_rowsize is not None:
+            logger.warning("'max_rowsize' is deprecated from version 2.6 and will be removed in a future version.")
         check_max_rowsize(max_rowsize)
 
         if (input_columns is not None) and (per_batch_map is None):
@@ -1491,6 +1530,9 @@ def check_map(method):
             check_num_parallel_workers(num_parallel_workers)
         type_check(python_multiprocessing, (bool,), "python_multiprocessing")
         check_cache_option(cache)
+
+        if max_rowsize is not None:
+            logger.warning("'max_rowsize' is deprecated from version 2.6 and will be removed in a future version.")
         check_max_rowsize(max_rowsize)
         if offload is not None:
             type_check(offload, (bool,), "offload")
@@ -1977,7 +2019,7 @@ def check_hostname(hostname):
     return all(allowed.match(x) for x in hostname.split("."))
 
 
-def check_numpyslicesdataset(method):
+def check_numpy_slices_dataset(method):
     """A wrapper that wraps a parameter checker around the original Dataset(NumpySlicesDataset)."""
 
     @wraps(method)
@@ -2018,7 +2060,7 @@ def check_numpyslicesdataset(method):
     return new_method
 
 
-def check_paddeddataset(method):
+def check_padded_dataset(method):
     """A wrapper that wraps a parameter checker around the original Dataset(PaddedDataset)."""
 
     @wraps(method)

mindspore/dataset/text/__init__.py

@@ -12,10 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-This module is to support text processing for NLP. It includes two parts:
-text transforms and utils. text transforms is a high performance
-NLP text processing module which is developed with ICU4C and cppjieba.
-utils provides some general methods for NLP text processing.
+This module is designed for text data augmentation and comprises two submodules: `transforms` and `utils`.
+
+`transforms` is a high-performance text data augmentation lib that supports common text data augmentation operations.
+
+`utils` provides a collection of utility methods for text processing.
 
 Common imported modules in corresponding API examples are as follows:
 
@@ -33,9 +34,9 @@ The data transform operation can be executed in the data processing pipeline or
 
 - Pipeline mode is generally used to process big datasets. Examples refer to
   `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/master/api_python/
-  mindspore.dataset.html#introduction-to-data-processing-pipeline>`_ .
+  mindspore.dataset.loading.html#introduction-to-data-processing-pipeline>`_ .
 - Eager mode is more like a function call to process data. Examples refer to
-  `Lightweight Data Processing <https://www.mindspore.cn/docs/en/master/model_train/dataset/eager.html>`_ .
+  `Lightweight Data Processing <https://www.mindspore.cn/tutorials/en/master/dataset/eager.html>`_ .
 """
 import platform
 

mindspore/dataset/text/transforms.py

@@ -595,7 +595,7 @@ class SentencePieceTokenizer(TextTensorOperation):
         >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=tokenizer)
         >>> for item in numpy_slices_dataset.create_dict_iterator(num_epochs=1, output_numpy=True):
         ...     print(item["text"])
-        ['▁H' 'e' 'l' 'lo' '▁w' 'o' 'r' 'l' 'd']
+        ['▁H' 'el' 'lo' '▁' 'w' 'or' 'l' 'd']
         >>>
         >>> # Use the transform in eager mode
         >>> data = "Hello world"
@@ -603,7 +603,7 @@ class SentencePieceTokenizer(TextTensorOperation):
         ...                                           SentencePieceModel.UNIGRAM, {})
         >>> output = text.SentencePieceTokenizer(vocab, out_type=SPieceTokenizerOutType.STRING)(data)
         >>> print(output)
-        ['▁' 'H' 'e' 'l' 'l' 'o' '▁' 'w' 'o' 'r' 'l' 'd']
+        ['▁' 'H' 'e' 'l' 'l' 'o' '▁' 'w' 'or' 'l' 'd']
 
     Tutorial Examples:
         - `Illustration of text transforms
@@ -623,11 +623,11 @@ class SentencePieceTokenizer(TextTensorOperation):
 
 class SlidingWindow(TextTensorOperation):
     """
-    Construct a tensor from given data (only support 1-D for now), where each element in the dimension axis
+    Construct a tensor from given data (only support 1-D for now), where each element in the dimension `axis`
     is a slice of data starting at the corresponding position, with a specified width.
 
     Args:
-        width (int): The width of the window. It must be an integer and greater than zero.
+        width (int): The width of the window. Must be greater than zero.
         axis (int, optional): The axis along which the sliding window is computed. Default: ``0``.
 
     Raises:
@@ -1332,7 +1332,8 @@ if platform.system().lower() != 'windows':
 
     class FilterWikipediaXML(TextTensorOperation):
         """
-        Filter Wikipedia XML dumps to "clean" text consisting only of lowercase letters (a-z, converted from A-Z),
+        Filter the text in the format of Wikipedia XML dumps to "clean" text
+        consisting only of lowercase letters (a-z, converted from A-Z),
         and spaces (never consecutive).
 
         Note:

mindspore/dataset/text/utils.py

@@ -49,7 +49,7 @@ class CharNGram(cde.CharNGram):
                 this value can limit the size of the loaded set. Default: ``None``, no upper limit.
 
         Returns:
-            CharNGram, CharNGram pre-training vectors.
+            CharNGram, pre-training vectors.
 
         Raises:
             TypeError: If `file_path` is not of type str.
@@ -92,7 +92,7 @@ class FastText(cde.FastText):
                 this value can limit the size of the loaded set. Default: ``None``, no upper limit.
 
         Returns:
-            FastText, FastText pre-training vectors.
+            FastText, pre-training vectors.
 
         Raises:
             TypeError: If `file_path` is not of type str.
@@ -133,7 +133,7 @@ class GloVe(cde.GloVe):
                 this value can limit the size of the loaded set. Default: ``None``, no upper limit.
 
         Returns:
-            GloVe, GloVe pre-training vectors.
+            GloVe, pre-training vectors.
 
         Raises:
             TypeError: If `file_path` is not of type str.

mindspore/dataset/transforms/__init__.py

@@ -23,19 +23,10 @@ Common imported modules in corresponding API examples are as follows:
     import mindspore.dataset as ds
     import mindspore.dataset.transforms as transforms
 
-Note: Legacy c_transforms and py_transforms are deprecated but can still be imported as follows:
-
-.. code-block::
-
-    from mindspore.dataset.transforms import c_transforms
-    from mindspore.dataset.transforms import py_transforms
-
 Descriptions of common data processing terms are as follows:
 
 - TensorOperation, the base class of all data processing operations implemented in C++.
 - PyTensorOperation, the base class of all data processing operations implemented in Python.
-
-Note: In eager mode, non-NumPy input is implicitly converted to NumPy format and sent to MindSpore.
 """
 from .. import vision
 from . import c_transforms

mindspore/dataset/transforms/py_transforms_util.py

@@ -19,6 +19,8 @@ import json
 import random
 from enum import IntEnum
 from types import FunctionType, MethodType
+
+import dill
 import numpy as np
 
 from mindspore import log as logger
@@ -187,6 +189,7 @@ class FuncWrapper:
             raise ValueError("Input operations should be callable Python function, but got: " + str(transform))
         self.transform = transform
         self.implementation = Implementation.C
+        self.random = False
         try:
             if hasattr(self.transform, "random") and not self.transform.random:
                 self.random = False
@@ -211,6 +214,20 @@ class FuncWrapper:
                 logger.warning(warn_msg)
         return result
 
+    def __getstate__(self):
+        transform = dill.dumps(self.transform)
+        implementation = dill.dumps(self.implementation)
+        local_random = dill.dumps(self.random)
+        logged_list_mixed_type_warning = dill.dumps(self.logged_list_mixed_type_warning)
+        return (transform, implementation, local_random, logged_list_mixed_type_warning)
+
+    def __setstate__(self, state):
+        transform, implementation, local_random, logged_list_mixed_type_warning = state
+        self.transform = dill.loads(transform)
+        self.implementation = dill.loads(implementation)
+        self.random = dill.loads(local_random)
+        self.logged_list_mixed_type_warning = dill.loads(logged_list_mixed_type_warning)
+
     def to_json(self):
         """ Serialize to JSON format """
         # User-defined Python functions cannot be fully nor correctly serialized.

mindspore/dataset/transforms/transforms.py

@@ -55,7 +55,7 @@ def clean_unused_executors():
     """
     clean the unused executor object in UDF or map with PyFunc process / thread mode
     """
-    key = str(os.getpid()) + "_" + str(threading.currentThread().ident)
+    key = str(os.getpid()) + "_" + str(threading.current_thread().ident)
     if key in EXECUTORS_LIST:
         EXECUTORS_LIST.pop(key)
 
@@ -94,19 +94,36 @@ class TensorOperation:
 
         # get or create the executor from EXECUTORS_LIST
         executor = None
-        key = str(os.getpid()) + "_" + str(threading.currentThread().ident)
-        if key in EXECUTORS_LIST:
-            # get the executor by process id and thread id
-            executor = EXECUTORS_LIST[key]
-            # remove the old transform which in executor and update the new transform
-            executor.UpdateOperation(self.parse())
-        else:
-            # create a new executor by process id and thread_id
-            executor = cde.Execute(self.parse())
-            # add the executor the global EXECUTORS_LIST
-            EXECUTORS_LIST[key] = executor
-
-        output_tensor_list = executor(tensor_row)
+        key = str(os.getpid()) + "_" + str(threading.current_thread().ident)
+        try:
+            if key in EXECUTORS_LIST:
+                # get the executor by process id and thread id
+                executor = EXECUTORS_LIST[key]
+                # remove the old transform which in executor and update the new transform
+                executor.UpdateOperation(self.parse())
+            else:
+                # create a new executor by process id and thread_id
+                executor = cde.Execute(self.parse())
+                # add the executor the global EXECUTORS_LIST
+                EXECUTORS_LIST[key] = executor
+
+            output_tensor_list = executor(tensor_row)
+        except RuntimeError as e:
+            if "Create stream failed" in str(e):
+                raise RuntimeError("Cannot reset NPU device in forked subprocess.\n    "
+                                   "Note: the following several scenarios are not supported yet.\n"
+                                   "    1. GeneratorDataset with num_parallel_workers>1 and "
+                                   "python_multiprocessing=True.\n    2. Independent dataset mode (export "
+                                   "MS_INDEPENDENT_DATASET=True):\n        1) Use the eager mode of dvpp "
+                                   "in the main process, and then start the dataset independent process. "
+                                   "GeneratorDataset / map / batch performs dvpp operations in thread mode.\n"
+                                   "        2) Initialize the device in the main process, and then start the "
+                                   "dataset independent process. GeneratorDataset / map / batch executes the "
+                                   "dvpp operation in thread mode.\n    "
+                                   "Suggestion: except for the scenes above to use NPU with multiprocessing, "
+                                   "you can set ds.config.set_multiprocessing_start_method('spawn') in your "
+                                   "script and rerun.")
+            raise e
         output_numpy_list = [x.as_array() for x in output_tensor_list]
         return output_numpy_list[0] if len(output_numpy_list) == 1 else tuple(output_numpy_list)
 

mindspore/dataset/utils/browse_dataset.py

@@ -31,7 +31,7 @@ def imshow_det_bbox(image, bboxes, labels, segm=None, class_names=None, score_th
         image (numpy.ndarray): The image to be displayed, shaped :math:`<C, H, W>` or :math:`<H, W, C>`,
             formatted RGB.
         bboxes (numpy.ndarray): Bounding boxes (with scores), shaped :math:`<N, 4>` or :math:`<N, 5>`,
-            data should be ordered with <N, x, y, w, h>.
+            data should be ordered with :math:`<N,X,Y,W,H>`.
         labels (numpy.ndarray): Labels of bboxes, shaped :math:`<N, 1>`.
         segm (numpy.ndarray): The segmentation masks of image in M classes, shaped :math:`<M, H, W>`.
             Default: ``None``.

mindspore/dataset/vision/__init__.py

@@ -24,13 +24,6 @@ Common imported modules in corresponding API examples are as follows:
     import mindspore.dataset.vision as vision
     import mindspore.dataset.vision.utils as utils
 
-Note: Legacy c_transforms and py_transforms are deprecated but can still be imported as follows:
-
-.. code-block::
-
-    import mindspore.dataset.vision.c_transforms as c_vision
-    import mindspore.dataset.vision.py_transforms as py_vision
-
 Descriptions of common data processing terms are as follows:
 
 - TensorOperation, the base class of all data processing operations implemented in C++.
@@ -41,9 +34,9 @@ The data transform operation can be executed in the data processing pipeline or
 
 - Pipeline mode is generally used to process big datasets. Examples refer to
   `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/master/api_python/
-  mindspore.dataset.html#introduction-to-data-processing-pipeline>`_ .
+  mindspore.dataset.loading.html#introduction-to-data-processing-pipeline>`_ .
 - Eager mode is more like a function call to process data. Examples refer to
-  `Lightweight Data Processing <https://www.mindspore.cn/docs/en/master/model_train/dataset/eager.html>`_ .
+  `Lightweight Data Processing <https://www.mindspore.cn/tutorials/en/master/dataset/eager.html>`_ .
 """
 from . import c_transforms
 from . import py_transforms