mindspore 1.10.0__cp37-none-any.whl → 2.0.0rc1__cp37-none-any.whl

mindspore/dataset/engine/validators.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2022 Huawei Technologies Co., Ltd
+# Copyright 2019-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,10 +20,10 @@ import inspect as ins
 import os
 import re
 from functools import wraps
-
 import numpy as np
-from mindspore import log
+
 from mindspore._c_expression import typing
+from mindspore import log as logger
 from ..core.validator_helpers import parse_user_args, type_check, type_check_list, check_value, \
     INT32_MAX, check_valid_detype, check_dir, check_file, check_sampler_shuffle_shard_options, \
     validate_dataset_param_value, check_padding_options, check_gnn_list_or_ndarray, check_gnn_list_of_pair_or_ndarray, \
@@ -585,6 +585,23 @@ def check_tfrecorddataset(method):
         validate_dataset_param_value(nreq_param_list, param_dict, list)
         validate_dataset_param_value(nreq_param_bool, param_dict, bool)
 
+        compression_type = param_dict.get('compression_type')
+        if compression_type is not None and compression_type not in ['', 'ZLIB', 'GZIP']:
+            raise ValueError("Input compression_type can only be either '' (no compression), 'ZLIB', or 'GZIP', " +
+                             "but got '" + str(compression_type) + "'.")
+        if compression_type is not None and compression_type in ['ZLIB', 'GZIP'] and \
+            param_dict.get('num_samples') is not None:
+            if param_dict.get('num_shards') is not None and ((isinstance(dataset_files, str) and \
+                param_dict.get('num_shards') > 1) or (isinstance(dataset_files, list) and \
+                len(dataset_files) < param_dict.get('num_shards'))):
+                num_files = len(dataset_files) if isinstance(dataset_files, list) else 1
+                act_num_shard = param_dict.get('num_shards') if param_dict.get('num_shards') is not None else 1
+                raise ValueError("When compression_type is provided, the number of dataset files cannot be less " +
+                                 "than num_shards, but the actual number of files is " + str(num_files) +
+                                 " and actual num_shards is " + str(act_num_shard) + ".")
+            if param_dict.get('shard_equal_rows') is None or not param_dict.get('shard_equal_rows'):
+                logger.warning("If compression_type is set, shard_equal_rows will be ignored.")
+
         check_sampler_shuffle_shard_options(param_dict)
 
         cache = param_dict.get('cache')
@@ -935,7 +952,7 @@ def check_lfw_dataset(method):
 
 
 def check_save(method):
-    """A wrapper that wraps a parameter checker around the saved operator."""
+    """A wrapper that wraps a parameter checker around the saved operation."""
 
     @wraps(method)
     def new_method(self, *args, **kwargs):
@@ -979,7 +996,7 @@ def check_dict_iterator(method):
 
     @wraps(method)
     def new_method(self, *args, **kwargs):
-        [num_epochs, _], param_dict = parse_user_args(method, *args, **kwargs)
+        [num_epochs, _, _], param_dict = parse_user_args(method, *args, **kwargs)
         nreq_param_bool = ['output_numpy']
         validate_dataset_param_value(nreq_param_bool, param_dict, bool)
         if num_epochs is not None:
@@ -1047,7 +1064,7 @@ def check_source_function(source):
             elif '__next__' in source_attr:
                 var = var + ins.getclosurevars(source.__class__.__next__)
                 source_doc = source_doc + ins.getsource(source.__class__.__next__)
-        except TypeError:
+        except (TypeError, OSError):
             # case: like input is LambdaType or GeneratorType, it will go to else branch, and unable to run normally
             pass
     return str(var) + source_doc
@@ -1147,6 +1164,34 @@ def check_random_dataset(method):
     return new_method
 
 
+def check_rendered_sst2_dataset(method):
+    """A wrapper that wraps a parameter checker around the original Dataset(RenderedSST2Dataset)."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        _, param_dict = parse_user_args(method, *args, **kwargs)
+
+        nreq_param_int = ['num_samples', 'num_parallel_workers', 'num_shards', 'shard_id']
+        nreq_param_bool = ['shuffle', 'decode']
+
+        dataset_dir = param_dict.get('dataset_dir')
+        usage = param_dict.get('usage')
+        check_dir(dataset_dir)
+        if usage is not None:
+            check_valid_str(usage, ['val', 'all', 'train', 'test'])
+
+        validate_dataset_param_value(nreq_param_int, param_dict, int)
+        validate_dataset_param_value(nreq_param_bool, param_dict, bool)
+        check_sampler_shuffle_shard_options(param_dict)
+
+        cache = param_dict.get('cache')
+        check_cache_option(cache)
+
+        return method(self, *args, **kwargs)
+
+    return new_method
+
+
 def check_pad_info(key, val):
     """check the key and value pair of pad_info in batch"""
     type_check(key, (str,), "key in pad_info")
@@ -1229,14 +1274,26 @@ def check_bucket_batch_by_length(method):
     return new_method
 
 
+def get_batch_kwargs_from_dict(param_dict):
+    """get batch operation kwargs parameters."""
+    if param_dict is not None:
+        per_batch_map = param_dict.get("per_batch_map", None)
+        input_columns = param_dict.get("input_columns", None)
+        output_columns = param_dict.get("output_columns", None)
+        python_multiprocessing = param_dict.get("python_multiprocessing", False)
+        max_rowsize = param_dict.get("max_rowsize", 16)
+    return per_batch_map, input_columns, output_columns, python_multiprocessing, max_rowsize
+
+
 def check_batch(method):
     """check the input arguments of batch."""
 
     @wraps(method)
     def new_method(self, *args, **kwargs):
-        [batch_size, drop_remainder, num_parallel_workers, per_batch_map,
-         input_columns, output_columns, column_order, pad_info,
-         python_multiprocessing, max_rowsize], param_dict = parse_user_args(method, *args, **kwargs)
+        [batch_size, drop_remainder, num_parallel_workers, param_dict], _ = parse_user_args(method, *args, **kwargs)
+
+        (per_batch_map, input_columns, output_columns, python_multiprocessing, max_rowsize) = \
+            get_batch_kwargs_from_dict(param_dict)
 
         if not (isinstance(batch_size, int) or (callable(batch_size))):
             raise TypeError("batch_size should either be an int or a callable.")
@@ -1254,14 +1311,6 @@ def check_batch(method):
         type_check(max_rowsize, (int,), "max_rowsize")
         check_pos_int32(max_rowsize, "max_rowsize")
 
-        if (pad_info is not None) and (per_batch_map is not None):
-            raise ValueError("pad_info and per_batch_map can't both be set.")
-
-        if pad_info is not None:
-            type_check(param_dict["pad_info"], (dict,), "pad_info")
-            for k, v in param_dict.get('pad_info').items():
-                check_pad_info(k, v)
-
         if (input_columns is not None) and (per_batch_map is None):
             # input_columns must be None when per_batch_map is not set
             raise ValueError("input_columns can be specified only when per_batch_map is set.")
@@ -1274,9 +1323,6 @@ def check_batch(method):
         if output_columns is not None:
             check_columns(output_columns, "output_columns")
 
-        if column_order is not None:
-            check_columns(column_order, "column_order")
-
         if python_multiprocessing is not None:
             type_check(python_multiprocessing, (bool,), "python_multiprocessing")
 
@@ -1285,6 +1331,37 @@ def check_batch(method):
     return new_method
 
 
+def check_padded_batch(method):
+    """check the input arguments of padded_batch."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        [batch_size, drop_remainder, num_parallel_workers, pad_info], _ = parse_user_args(method, *args, **kwargs)
+
+        if not (isinstance(batch_size, int) or (callable(batch_size))):
+            raise TypeError("batch_size should either be an int or a callable.")
+
+        if callable(batch_size):
+            sig = ins.signature(batch_size)
+            if len(sig.parameters) != 1:
+                raise ValueError("callable batch_size should take one parameter (BatchInfo).")
+        else:
+            check_pos_int32(int(batch_size), "batch_size")
+
+        if num_parallel_workers is not None:
+            check_num_parallel_workers(num_parallel_workers)
+        type_check(drop_remainder, (bool,), "drop_remainder")
+
+        if pad_info is not None:
+            type_check(pad_info, (dict,), "pad_info")
+            for k, v in pad_info.items():
+                check_pad_info(k, v)
+
+        return method(self, *args, **kwargs)
+
+    return new_method
+
+
 def check_sync_wait(method):
     """check the input arguments of sync_wait."""
 
@@ -1316,16 +1393,42 @@ def check_shuffle(method):
     return new_method
 
 
+def get_map_kwargs_from_dict(param_dict):
+    """get map operation kwargs parameters."""
+    if param_dict is not None:
+        python_multiprocessing = param_dict.get("python_multiprocessing", False)
+        max_rowsize = param_dict.get("max_rowsize", 16)
+        cache = param_dict.get("cache", None)
+        callbacks = param_dict.get("callbacks", None)
+        offload = param_dict.get("offload", None)
+    return python_multiprocessing, max_rowsize, cache, callbacks, offload
+
+
 def check_map(method):
     """check the input arguments of map."""
 
     @wraps(method)
     def new_method(self, *args, **kwargs):
         from mindspore.dataset.callback import DSCallback
-        [operations, input_columns, output_columns, column_order, num_parallel_workers, python_multiprocessing, cache,
-         callbacks, max_rowsize, offload], _ = \
+        [operations, input_columns, output_columns, column_order, num_parallel_workers, param_dict], _ = \
             parse_user_args(method, *args, **kwargs)
 
+        if column_order is not None:
+            raise ValueError("The parameter 'column_order' had been deleted in map operation. "
+                             "Please use '.project' operation instead.\n"
+                             ">> # Usage of old api:\n"
+                             ">> dataset = dataset.map(operations=PyFunc,\n"
+                             ">>                       input_columns=[\"column_a\"],\n"
+                             ">>                       output_columns=[\"column_b\", \"column_c\"],\n"
+                             ">>                       column_order=[\"column_b\", \"column_c\"])\n"
+                             ">> # Usage of new api:\n"
+                             ">> dataset = dataset.map(operations=PyFunc,\n"
+                             ">>                       input_columns=[\"column_a\"],\n"
+                             ">>                       output_columns=[\"column_b\", \"column_c\"])\n"
+                             ">> dataset = dataset.project([\"column_b\", \"column_c\"])")
+
+        (python_multiprocessing, max_rowsize, cache, callbacks, offload) = get_map_kwargs_from_dict(param_dict)
+
         # check whether network computing operator exist in input operations(python function)
         # check used variable and function document whether contain computing operator
         from types import FunctionType
@@ -1350,10 +1453,8 @@ def check_map(method):
                 raise ValueError("Input operations should not contain network computing operator like in "
                                  "mindspore.nn or mindspore.ops, got operation: ", str(item))
 
-        nreq_param_columns = ['input_columns', 'output_columns', 'column_order']
+        nreq_param_columns = ['input_columns', 'output_columns']
 
-        if column_order is not None:
-            type_check(column_order, (list,), "column_order")
         if num_parallel_workers is not None:
             check_num_parallel_workers(num_parallel_workers)
         type_check(python_multiprocessing, (bool,), "python_multiprocessing")
@@ -1369,7 +1470,7 @@ def check_map(method):
             else:
                 type_check(callbacks, (DSCallback,), "callbacks")
 
-        for param_name, param in zip(nreq_param_columns, [input_columns, output_columns, column_order]):
+        for param_name, param in zip(nreq_param_columns, [input_columns, output_columns]):
             if param is not None:
                 check_columns(param, param_name)
         if callbacks is not None:
@@ -1467,7 +1568,7 @@ def check_positive_int32(method):
 
 
 def check_device_send(method):
-    """check the input argument for to_device and device_que."""
+    """check the input argument of device_que."""
 
     @wraps(method)
     def new_method(self, *args, **kwargs):
@@ -1494,7 +1595,7 @@ def check_zip(method):
 
 
 def check_zip_dataset(method):
-    """check the input arguments of zip method in `Dataset`."""
+    """check the input arguments of zip method in `Dataset` ."""
 
     @wraps(method)
     def new_method(self, *args, **kwargs):
@@ -1507,7 +1608,7 @@ def check_zip_dataset(method):
 
 
 def check_concat(method):
-    """check the input arguments of concat method in `Dataset`."""
+    """check the input arguments of concat method in `Dataset` ."""
 
     @wraps(method)
     def new_method(self, *args, **kwargs):
@@ -2302,6 +2403,36 @@ def check_flickr_dataset(method):
     return new_method
 
 
+def check_food101_dataset(method):
+    """A wrapper that wraps a parameter checker around the Food101Dataset."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        _, param_dict = parse_user_args(method, *args, **kwargs)
+
+        nreq_param_int = ['num_samples', 'num_parallel_workers', 'num_shards', 'shard_id']
+        nreq_param_bool = ['decode', 'shuffle']
+
+        dataset_dir = param_dict.get('dataset_dir')
+        check_dir(dataset_dir)
+
+        usage = param_dict.get('usage')
+        if usage is not None:
+            check_valid_str(usage, ["train", "test", "all"], "usage")
+
+        validate_dataset_param_value(nreq_param_int, param_dict, int)
+        validate_dataset_param_value(nreq_param_bool, param_dict, bool)
+
+        check_sampler_shuffle_shard_options(param_dict)
+
+        cache = param_dict.get('cache')
+        check_cache_option(cache)
+
+        return method(self, *args, **kwargs)
+
+    return new_method
+
+
 def check_sb_dataset(method):
     """A wrapper that wraps a parameter checker around the original Semantic Boundaries Dataset."""
 
@@ -2714,6 +2845,34 @@ def check_svhn_dataset(method):
     return new_method
 
 
+def check_sst2_dataset(method):
+    """A wrapper that wraps a parameter checker around the original SST2 Dataset."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        _, param_dict = parse_user_args(method, *args, **kwargs)
+
+        nreq_param_int = ['num_samples', 'num_parallel_workers', 'num_shards', 'shard_id']
+
+        dataset_dir = param_dict.get('dataset_dir')
+        check_dir(dataset_dir)
+
+        usage = param_dict.get('usage')
+        if usage is not None:
+            check_valid_str(usage, ["train", "test", "dev"], "usage")
+
+        validate_dataset_param_value(nreq_param_int, param_dict, int)
+
+        check_sampler_shuffle_shard_options(param_dict)
+
+        cache = param_dict.get('cache')
+        check_cache_option(cache)
+
+        return method(self, *args, **kwargs)
+
+    return new_method
+
+
 def check_stl10_dataset(method):
     """A wrapper that wraps a parameter checker around the original Dataset(STL10Dataset)."""
 
@@ -2760,6 +2919,31 @@ def check_stl10_dataset(method):
     return new_method
 
 
+def check_sun397_dataset(method):
+    """A wrapper that wraps a parameter checker around the original Dataset(SUN397Dataset)."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        _, param_dict = parse_user_args(method, *args, **kwargs)
+
+        nreq_param_int = ['num_samples', 'num_parallel_workers', 'num_shards', 'shard_id']
+        nreq_param_bool = ['shuffle', 'decode']
+
+        dataset_dir = param_dict.get('dataset_dir')
+        check_dir(dataset_dir)
+
+        validate_dataset_param_value(nreq_param_int, param_dict, int)
+        validate_dataset_param_value(nreq_param_bool, param_dict, bool)
+        check_sampler_shuffle_shard_options(param_dict)
+
+        cache = param_dict.get('cache')
+        check_cache_option(cache)
+
+        return method(self, *args, **kwargs)
+
+    return new_method
+
+
 def check_yahoo_answers_dataset(method):
     """A wrapper that wraps a parameter checker around the original YahooAnswers Dataset."""
 
@@ -2961,29 +3145,6 @@ def check_multi30k_dataset(method):
     return new_method
 
 
-def deprecated(version, substitute=None):
-    """deprecated warning
-
-    Args:
-        version (str): version that the operator or function is deprecated.
-        substitute (str): the substitute name for deprecated operator or function.
-    """
-
-    def decorate(func):
-        def wrapper(*args, **kwargs):
-            name = func.__name__
-            message = f"'{name}' is deprecated from version {version} and will be removed in a future version. "
-            if substitute:
-                message += f"Use '{substitute}' instead."
-            log.warning(message)
-            ret = func(*args, **kwargs)
-            return ret
-
-        return wrapper
-
-    return decorate
-
-
 def check_obsminddataset(method):
     """A wrapper that wraps a parameter checker around the original Dataset(OBSMindDataset)."""
 

mindspore/dataset/text/__init__.py

@@ -24,24 +24,24 @@ Common imported modules in corresponding API examples are as follows:
     import mindspore.dataset as ds
     import mindspore.dataset.text as text
 
-See `Text Data Processing and Augmentation
-<https://www.mindspore.cn/tutorials/en/r1.10/beginner/transforms.html#text-transforms>`_ tutorial for more details.
+See `Text Transforms
+<https://www.mindspore.cn/tutorials/en/r2.0/beginner/transforms.html#text-transforms>`_ tutorial for more details.
 
 Descriptions of common data processing terms are as follows:
 
 - TensorOperation, the base class of all data processing operations implemented in C++.
 - TextTensorOperation, the base class of all text processing operations. It is a derived class of TensorOperation.
 
-The data transform operator can be executed in the data processing pipeline or in the eager mode:
+The data transform operation can be executed in the data processing pipeline or in the eager mode:
 
 - Pipeline mode is generally used to process datasets. For examples, please refer to
-  `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/r1.10/api_python/
-  mindspore.dataset.html#introduction-to-data-processing-pipeline>`_.
+  `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/r2.0/api_python/
+  mindspore.dataset.html#introduction-to-data-processing-pipeline>`_ .
 - Eager mode is generally used for scattered samples. Examples of text preprocessing are as follows:
 
   .. code-block::
 
-      from mindspore.dataset import text
+      import mindspore.dataset.text as text
       from mindspore.dataset.text import NormalizeForm
 
       # construct vocab
@@ -65,13 +65,15 @@ The data transform operator can be executed in the data processing pipeline or i
       # ids to token
       tokens_from_ids = vocab.ids_to_tokens([15, 3, 7])
       print("token to id: {}".format(tokens_from_ids), flush=True)
+
+Note: In eager mode, non-NumPy input is implicitly converted to NumPy format and sent to MindSpore.
 """
 import platform
 
 from . import transforms
 from . import utils
-from .transforms import JiebaTokenizer, Lookup, Ngram, PythonTokenizer, SentencePieceTokenizer, SlidingWindow, \
-    ToNumber, ToVectors, TruncateSequencePair, UnicodeCharTokenizer, WordpieceTokenizer
+from .transforms import AddToken, JiebaTokenizer, Lookup, Ngram, PythonTokenizer, SentencePieceTokenizer, \
+    SlidingWindow, ToNumber, ToVectors, Truncate, TruncateSequencePair, UnicodeCharTokenizer, WordpieceTokenizer
 from .utils import CharNGram, FastText, GloVe, JiebaMode, NormalizeForm, SentencePieceModel, SentencePieceVocab, \
     SPieceTokenizerLoadType, SPieceTokenizerOutType, Vectors, Vocab, to_bytes, to_str