mindspore 2.0.0rc1__cp38-cp38-manylinux1_x86_64.whl → 2.2.0__cp38-cp38-manylinux1_x86_64.whl

mindspore/dataset/engine/validators.py

@@ -18,7 +18,6 @@ Built-in validators.
 """
 import inspect as ins
 import os
-import re
 from functools import wraps
 import numpy as np
 
@@ -26,9 +25,9 @@ from mindspore._c_expression import typing
 from mindspore import log as logger
 from ..core.validator_helpers import parse_user_args, type_check, type_check_list, check_value, \
     INT32_MAX, check_valid_detype, check_dir, check_file, check_sampler_shuffle_shard_options, \
-    validate_dataset_param_value, check_padding_options, check_gnn_list_or_ndarray, check_gnn_list_of_pair_or_ndarray, \
+    validate_dataset_param_value, check_padding_options, \
     check_num_parallel_workers, check_columns, check_pos_int32, check_valid_str, check_dataset_num_shards_shard_id, \
-    check_valid_list_tuple, check_dict, check_feature_shape
+    check_valid_list_tuple
 
 from . import datasets
 from . import samplers
@@ -1308,8 +1307,8 @@ def check_batch(method):
         if num_parallel_workers is not None:
             check_num_parallel_workers(num_parallel_workers)
         type_check(drop_remainder, (bool,), "drop_remainder")
-        type_check(max_rowsize, (int,), "max_rowsize")
-        check_pos_int32(max_rowsize, "max_rowsize")
+
+        check_max_rowsize(max_rowsize)
 
         if (input_columns is not None) and (per_batch_map is None):
             # input_columns must be None when per_batch_map is not set
@@ -1404,6 +1403,20 @@ def get_map_kwargs_from_dict(param_dict):
     return python_multiprocessing, max_rowsize, cache, callbacks, offload
 
 
+def check_max_rowsize(max_rowsize):
+    """check the max_rowsize"""
+    type_check(max_rowsize, (int, list), "max_rowsize")
+    if isinstance(max_rowsize, int):
+        type_check(max_rowsize, (int,), "max_rowsize")
+        check_pos_int32(max_rowsize, "max_rowsize")
+    elif isinstance(max_rowsize, list) and len(max_rowsize) == 2:
+        for index, value in enumerate(max_rowsize):
+            type_check(value, (int,), "max_rowsize[{}]".format(index))
+            check_pos_int32(value, "max_rowsizei[{}]".format(index))
+    else:
+        raise TypeError("max_rowsize should be a single integer or a list[in_rowsize, out_rowsize] of length 2.")
+
+
 def check_map(method):
     """check the input arguments of map."""
 
@@ -1459,8 +1472,7 @@ def check_map(method):
             check_num_parallel_workers(num_parallel_workers)
         type_check(python_multiprocessing, (bool,), "python_multiprocessing")
         check_cache_option(cache)
-        type_check(max_rowsize, (int,), "max_rowsize")
-        check_pos_int32(max_rowsize, "max_rowsize")
+        check_max_rowsize(max_rowsize)
         if offload is not None:
             type_check(offload, (bool,), "offload")
 
@@ -1572,9 +1584,10 @@ def check_device_send(method):
 
     @wraps(method)
     def new_method(self, *args, **kwargs):
-        [send_epoch_end, create_data_info_queue], _ = parse_user_args(method, *args, **kwargs)
+        [send_epoch_end, create_data_info_queue, queue_name], _ = parse_user_args(method, *args, **kwargs)
         type_check(send_epoch_end, (bool,), "send_epoch_end")
         type_check(create_data_info_queue, (bool,), "create_data_info_queue")
+        type_check(queue_name, (str,), "queue_name")
 
         return method(self, *args, **kwargs)
 
@@ -1939,329 +1952,6 @@ def check_hostname(hostname):
     return all(allowed.match(x) for x in hostname.split("."))
 
 
-def check_gnn_graphdata(method):
-    """check the input arguments of graphdata."""
-
-    @wraps(method)
-    def new_method(self, *args, **kwargs):
-        [dataset_file, num_parallel_workers, working_mode, hostname,
-         port, num_client, auto_shutdown], _ = parse_user_args(method, *args, **kwargs)
-        check_file(dataset_file)
-        if num_parallel_workers is not None:
-            check_num_parallel_workers(num_parallel_workers)
-        type_check(hostname, (str,), "hostname")
-        if check_hostname(hostname) is False:
-            raise ValueError("The hostname is illegal.")
-        type_check(working_mode, (str,), "working_mode")
-        if working_mode not in {'local', 'client', 'server'}:
-            raise ValueError("Invalid working mode, please enter 'local', 'client' or 'server'.")
-        type_check(port, (int,), "port")
-        check_value(port, (1024, 65535), "port")
-        type_check(num_client, (int,), "num_client")
-        check_value(num_client, (1, 255), "num_client")
-        type_check(auto_shutdown, (bool,), "auto_shutdown")
-        return method(self, *args, **kwargs)
-
-    return new_method
-
-
-def check_gnn_graph(method):
-    """check the input arguments of Graph."""
-
-    @wraps(method)
-    def new_method(self, *args, **kwargs):
-        [edges, node_feat, edge_feat, graph_feat, node_type, edge_type, num_parallel_workers, working_mode,
-         hostname, port, num_client, auto_shutdown], _ = parse_user_args(method, *args, **kwargs)
-
-        type_check(edges, (list, np.ndarray), "edges")
-        edges = np.asarray(edges)
-        if len(edges.shape) != 2 or edges.shape[0] != 2:
-            raise ValueError("Input 'edges' should be of 2 dimension, here got {}."
-                             " And shape of first dimension should be 2, here got {}.".format(len(edges.shape),
-                                                                                              edges.shape[0]))
-        if not np.issubdtype(edges.dtype, np.integer):
-            raise TypeError("Each element in input 'edges' should be int, got value: {}.".format(edges))
-
-        check_dict(node_feat, str, np.ndarray, "node_feat")
-        check_dict(edge_feat, str, np.ndarray, "edge_feat")
-        check_dict(graph_feat, str, np.ndarray, "graph_feat")
-        if node_type is not None:
-            type_check(node_type, (list, np.ndarray), "node_type")
-            if not all(isinstance(item, str) for item in list(node_type)):
-                raise TypeError("Type of each element in 'node_type' should be str.")
-        if edge_type is not None:
-            type_check(edge_type, (list, np.ndarray), "edge_type")
-            if not all(isinstance(item, str) for item in list(edge_type)):
-                raise TypeError("Type of each element in 'edge_type' should be str.")
-            edge_type = np.asarray(edge_type)
-            if len(edge_type.shape) != 1 or edge_type.shape[0] != edges.shape[1]:
-                raise ValueError(
-                    "Input 'edge_type' should be of 1 dimension, and its length should be {}, but got {}.".format(
-                        edges.shape[1], edge_type.shape[0]))
-
-        # check shape of node_feat and edge_feat
-        num_nodes = np.max(edges) + 1
-        check_feature_shape(node_feat, num_nodes, "node_feat")
-        check_feature_shape(edge_feat, edges.shape[1], "edge_feat")
-
-        if num_parallel_workers is not None:
-            check_num_parallel_workers(num_parallel_workers)
-        type_check(hostname, (str,), "hostname")
-        if check_hostname(hostname) is False:
-            raise ValueError("The hostname is illegal.")
-        type_check(working_mode, (str,), "working_mode")
-        if working_mode not in {'local', 'client', 'server'}:
-            raise ValueError("Invalid working mode, please enter 'local', 'client' or 'server'.")
-        type_check(port, (int,), "port")
-        check_value(port, (1024, 65535), "port")
-        type_check(num_client, (int,), "num_client")
-        check_value(num_client, (1, 255), "num_client")
-        type_check(auto_shutdown, (bool,), "auto_shutdown")
-        return method(self, *args, **kwargs)
-
-    return new_method
-
-
-def check_gnn_get_all_nodes(method):
-    """A wrapper that wraps a parameter checker around the GNN `get_all_nodes` function."""
-
-    @wraps(method)
-    def new_method(self, *args, **kwargs):
-        [node_type], _ = parse_user_args(method, *args, **kwargs)
-        if "GraphData" in str(type(self)):
-            type_check(node_type, (int,), "node_type")
-        else:
-            type_check(node_type, (str,), "node_type")
-
-        return method(self, *args, **kwargs)
-
-    return new_method
-
-
-def check_gnn_get_all_edges(method):
-    """A wrapper that wraps a parameter checker around the GNN `get_all_edges` function."""
-
-    @wraps(method)
-    def new_method(self, *args, **kwargs):
-        [edge_type], _ = parse_user_args(method, *args, **kwargs)
-        if "GraphData" in str(type(self)):
-            type_check(edge_type, (int,), "edge_type")
-        else:
-            type_check(edge_type, (str,), "edge_type")
-
-        return method(self, *args, **kwargs)
-
-    return new_method
-
-
-def check_gnn_get_nodes_from_edges(method):
-    """A wrapper that wraps a parameter checker around the GNN `get_nodes_from_edges` function."""
-
-    @wraps(method)
-    def new_method(self, *args, **kwargs):
-        [edge_list], _ = parse_user_args(method, *args, **kwargs)
-        check_gnn_list_or_ndarray(edge_list, "edge_list")
-
-        return method(self, *args, **kwargs)
-
-    return new_method
-
-
-def check_gnn_get_edges_from_nodes(method):
-    """A wrapper that wraps a parameter checker around the GNN `get_edges_from_nodes` function."""
-
-    @wraps(method)
-    def new_method(self, *args, **kwargs):
-        [node_list], _ = parse_user_args(method, *args, **kwargs)
-        check_gnn_list_of_pair_or_ndarray(node_list, "node_list")
-
-        return method(self, *args, **kwargs)
-
-    return new_method
-
-
-def check_gnn_get_all_neighbors(method):
-    """A wrapper that wraps a parameter checker around the GNN `get_all_neighbors` function."""
-
-    @wraps(method)
-    def new_method(self, *args, **kwargs):
-        [node_list, neighbour_type, _], _ = parse_user_args(method, *args, **kwargs)
-
-        check_gnn_list_or_ndarray(node_list, 'node_list')
-        if "GraphData" in str(type(self)):
-            type_check(neighbour_type, (int,), "neighbour_type")
-        else:
-            type_check(neighbour_type, (str,), "neighbour_type")
-
-        return method(self, *args, **kwargs)
-
-    return new_method
-
-
-def check_gnn_get_sampled_neighbors(method):
-    """A wrapper that wraps a parameter checker around the GNN `get_sampled_neighbors` function."""
-
-    @wraps(method)
-    def new_method(self, *args, **kwargs):
-        [node_list, neighbor_nums, neighbor_types, _], _ = parse_user_args(method, *args, **kwargs)
-
-        check_gnn_list_or_ndarray(node_list, 'node_list')
-
-        check_gnn_list_or_ndarray(neighbor_nums, 'neighbor_nums')
-        neighbor_nums = list(neighbor_nums)
-        if not neighbor_nums or len(neighbor_nums) > 6:
-            raise ValueError("Wrong number of input members for {0}, should be between 1 and 6, got {1}.".format(
-                'neighbor_nums', len(neighbor_nums)))
-
-        if "GraphData" in str(type(self)):
-            check_gnn_list_or_ndarray(neighbor_types, 'neighbor_types')
-        else:
-            check_gnn_list_or_ndarray(neighbor_types, 'neighbor_types', str)
-        neighbor_types = list(neighbor_types)
-        if not neighbor_types or len(neighbor_types) > 6:
-            raise ValueError("Wrong number of input members for {0}, should be between 1 and 6, got {1}.".format(
-                'neighbor_types', len(neighbor_types)))
-
-        if len(neighbor_nums) != len(neighbor_types):
-            raise ValueError(
-                "The number of members of neighbor_nums and neighbor_types is inconsistent.")
-
-        return method(self, *args, **kwargs)
-
-    return new_method
-
-
-def check_gnn_get_neg_sampled_neighbors(method):
-    """A wrapper that wraps a parameter checker around the GNN `get_neg_sampled_neighbors` function."""
-
-    @wraps(method)
-    def new_method(self, *args, **kwargs):
-        [node_list, neg_neighbor_num, neg_neighbor_type], _ = parse_user_args(method, *args, **kwargs)
-
-        check_gnn_list_or_ndarray(node_list, 'node_list')
-        type_check(neg_neighbor_num, (int,), "neg_neighbor_num")
-
-        if "GraphData" in str(type(self)):
-            type_check(neg_neighbor_type, (int,), "neg_neighbor_type")
-        else:
-            type_check(neg_neighbor_type, (str,), "neg_neighbor_type")
-
-        return method(self, *args, **kwargs)
-
-    return new_method
-
-
-def check_gnn_random_walk(method):
-    """A wrapper that wraps a parameter checker around the GNN `random_walk` function."""
-
-    @wraps(method)
-    def new_method(self, *args, **kwargs):
-        [target_nodes, meta_path, step_home_param, step_away_param, default_node], _ = parse_user_args(method, *args,
-                                                                                                       **kwargs)
-        check_gnn_list_or_ndarray(target_nodes, 'target_nodes')
-        check_gnn_list_or_ndarray(meta_path, 'meta_path')
-        type_check(step_home_param, (float,), "step_home_param")
-        type_check(step_away_param, (float,), "step_away_param")
-        type_check(default_node, (int,), "default_node")
-        check_value(default_node, (-1, INT32_MAX), "default_node")
-
-        return method(self, *args, **kwargs)
-
-    return new_method
-
-
-def check_aligned_list(param, param_name, member_type):
-    """Check whether the structure of each member of the list is the same."""
-
-    type_check(param, (list,), "param")
-    if not param:
-        raise TypeError(
-            "Parameter {0} or its members are empty".format(param_name))
-    member_have_list = None
-    list_len = None
-    for member in param:
-        if isinstance(member, list):
-            check_aligned_list(member, param_name, member_type)
-
-            if member_have_list not in (None, True):
-                raise TypeError("The type of each member of the parameter {0} is inconsistent.".format(
-                    param_name))
-            if list_len is not None and len(member) != list_len:
-                raise TypeError("The size of each member of parameter {0} is inconsistent.".format(
-                    param_name))
-            member_have_list = True
-            list_len = len(member)
-        else:
-            type_check(member, (member_type,), param_name)
-            if member_have_list not in (None, False):
-                raise TypeError("The type of each member of the parameter {0} is inconsistent.".format(
-                    param_name))
-            member_have_list = False
-
-
-def check_gnn_get_node_feature(method):
-    """A wrapper that wraps a parameter checker around the GNN `get_node_feature` function."""
-
-    @wraps(method)
-    def new_method(self, *args, **kwargs):
-        [node_list, feature_types], _ = parse_user_args(method, *args, **kwargs)
-
-        type_check(node_list, (list, np.ndarray), "node_list")
-        if isinstance(node_list, list):
-            check_aligned_list(node_list, 'node_list', int)
-        elif isinstance(node_list, np.ndarray):
-            if not node_list.dtype == np.int32:
-                raise TypeError("Each member in {0} should be of type int32. Got {1}.".format(
-                    node_list, node_list.dtype))
-
-        if "GraphData" in str(type(self)):
-            check_gnn_list_or_ndarray(feature_types, 'feature_types')
-        else:
-            check_gnn_list_or_ndarray(feature_types, 'feature_types', data_type=str)
-
-        return method(self, *args, **kwargs)
-
-    return new_method
-
-
-def check_gnn_get_graph_feature(method):
-    """A wrapper that wraps a parameter checker around the GNN `get_graph_feature` function."""
-
-    @wraps(method)
-    def new_method(self, *args, **kwargs):
-        [feature_types], _ = parse_user_args(method, *args, **kwargs)
-        check_gnn_list_or_ndarray(feature_types, 'feature_types', str)
-
-        return method(self, *args, **kwargs)
-
-    return new_method
-
-
-def check_gnn_get_edge_feature(method):
-    """A wrapper that wraps a parameter checker around the GNN `get_edge_feature` function."""
-
-    @wraps(method)
-    def new_method(self, *args, **kwargs):
-        [edge_list, feature_types], _ = parse_user_args(method, *args, **kwargs)
-
-        type_check(edge_list, (list, np.ndarray), "edge_list")
-        if isinstance(edge_list, list):
-            check_aligned_list(edge_list, 'edge_list', int)
-        elif isinstance(edge_list, np.ndarray):
-            if not edge_list.dtype == np.int32:
-                raise TypeError("Each member in {0} should be of type int32. Got {1}.".format(
-                    edge_list, edge_list.dtype))
-
-        if "GraphData" in str(type(self)):
-            check_gnn_list_or_ndarray(feature_types, 'feature_types')
-        else:
-            check_gnn_list_or_ndarray(feature_types, 'feature_types', data_type=str)
-
-        return method(self, *args, **kwargs)
-
-    return new_method
-
-
 def check_numpyslicesdataset(method):
     """A wrapper that wraps a parameter checker around the original Dataset(NumpySlicesDataset)."""
 

mindspore/dataset/text/__init__.py

@@ -25,7 +25,7 @@ Common imported modules in corresponding API examples are as follows:
     import mindspore.dataset.text as text
 
 See `Text Transforms
-<https://www.mindspore.cn/tutorials/en/r2.0/beginner/transforms.html#text-transforms>`_ tutorial for more details.
+<https://www.mindspore.cn/tutorials/en/r2.2/beginner/transforms.html#text-transforms>`_ tutorial for more details.
 
 Descriptions of common data processing terms are as follows:
 
@@ -34,39 +34,11 @@ Descriptions of common data processing terms are as follows:
 
 The data transform operation can be executed in the data processing pipeline or in the eager mode:
 
-- Pipeline mode is generally used to process datasets. For examples, please refer to
-  `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/r2.0/api_python/
+- Pipeline mode is generally used to process big datasets. Examples refer to
+  `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/r2.2/api_python/
   mindspore.dataset.html#introduction-to-data-processing-pipeline>`_ .
-- Eager mode is generally used for scattered samples. Examples of text preprocessing are as follows:
-
-  .. code-block::
-
-      import mindspore.dataset.text as text
-      from mindspore.dataset.text import NormalizeForm
-
-      # construct vocab
-      vocab_list = {"music": 1, "Opera": 2, "form": 3, "theatre": 4, "which": 5, "in": 6,
-                    "fundamental": 7, "dramatic": 8, "component": 9, "taken": 10, "roles": 11, "singers": 12,
-                    "is": 13, "are": 14, "of": 15, "UNK": 16}
-      vocab = text.Vocab.from_dict(vocab_list)
-      tokenizer_op = text.BertTokenizer(vocab=vocab, suffix_indicator='##', max_bytes_per_token=100,
-                                        unknown_token='[UNK]', lower_case=False, keep_whitespace=False,
-                                        normalization_form=NormalizeForm.NONE, preserve_unused_token=True,
-                                        with_offsets=False)
-      # tokenizer
-      tokens = tokenizer_op("Opera is a form of theatre in which music is a fundamental "
-                            "component and dramatic roles are taken by singers.")
-      print("token: {}".format(tokens), flush=True)
-
-      # token to ids
-      ids = vocab.tokens_to_ids(tokens)
-      print("token to id: {}".format(ids), flush=True)
-
-      # ids to token
-      tokens_from_ids = vocab.ids_to_tokens([15, 3, 7])
-      print("token to id: {}".format(tokens_from_ids), flush=True)
-
-Note: In eager mode, non-NumPy input is implicitly converted to NumPy format and sent to MindSpore.
+- Eager mode is more like a function call to process data. Examples refer to
+  `Lightweight Data Processing <https://www.mindspore.cn/tutorials/en/r2.2/advanced/dataset/eager.html>`_ .
 """
 import platform