mindspore 1.10.0__cp39-cp39-win_amd64.whl → 2.0.0rc1__cp39-cp39-win_amd64.whl

mindspore/dataset/text/transforms.py

@@ -31,7 +31,7 @@ Examples:
     >>> tokenizer = text.UnicodeCharTokenizer()
     >>> # Load vocabulary from list
     >>> vocab = text.Vocab.from_list(word_list=['深', '圳', '欢', '迎', '您'])
-    >>> # Use Lookup operator to map tokens to ids
+    >>> # Use Lookup operation to map tokens to ids
     >>> lookup = text.Lookup(vocab=vocab)
     >>> text_file_dataset = text_file_dataset.map(operations=[tokenizer, lookup])
     >>> # if text line in dataset_file is:
@@ -49,11 +49,11 @@ import mindspore._c_dataengine as cde
 from mindspore.common import dtype as mstype
 
 from .utils import JiebaMode, NormalizeForm, to_str, SPieceTokenizerOutType, SPieceTokenizerLoadType, SentencePieceVocab
-from .validators import check_lookup, check_jieba_add_dict, check_to_vectors, \
+from .validators import check_add_token, check_lookup, check_jieba_add_dict, check_to_vectors, \
     check_jieba_add_word, check_jieba_init, check_with_offsets, check_unicode_script_tokenizer, \
     check_wordpiece_tokenizer, check_regex_replace, check_regex_tokenizer, check_basic_tokenizer, check_ngram, \
     check_pair_truncate, check_to_number, check_bert_tokenizer, check_python_tokenizer, check_slidingwindow, \
-    check_sentence_piece_tokenizer
+    check_sentence_piece_tokenizer, check_truncate
 from ..core.datatypes import mstype_to_detype
 from ..core.validator_helpers import replace_none
 from ..transforms.py_transforms_util import Implementation
@@ -91,6 +91,49 @@ DE_C_INTER_SENTENCEPIECE_OUTTYPE = {
 }
 
 
+class AddToken(TextTensorOperation):
+    """
+    Add token to beginning or end of sequence.
+
+    Args:
+        token (str): The token to be added.
+        begin (bool, optional): Choose the position where the token is inserted. If True,
+            the token will be inserted at the beginning of the sequence. Otherwise, it will
+            be inserted at the end of the sequence. Default: True.
+
+    Raises:
+        TypeError: If `token` is not of type string.
+        TypeError: If `begin` is not of type bool.
+
+    Supported Platforms:
+        ``CPU``
+
+    Examples:
+        >>> dataset = ds.NumpySlicesDataset(data={"text": [['a', 'b', 'c', 'd', 'e']]})
+        >>> # Data before
+        >>> # |           text            |
+        >>> # +---------------------------+
+        >>> # | ['a', 'b', 'c', 'd', 'e'] |
+        >>> # +---------------------------+
+        >>> add_token_op = text.AddToken(token='TOKEN', begin=True)
+        >>> dataset = dataset.map(operations=add_token_op)
+        >>> # Data after
+        >>> # |           text            |
+        >>> # +---------------------------+
+        >>> # | ['TOKEN', 'a', 'b', 'c', 'd', 'e'] |
+        >>> # +---------------------------+
+    """
+
+    @check_add_token
+    def __init__(self, token, begin=True):
+        super().__init__()
+        self.token = token
+        self.begin = begin
+
+    def parse(self):
+        return cde.AddTokenOperation(self.token, self.begin)
+
+
 class JiebaTokenizer(TextTensorOperation):
     """
     Tokenize Chinese string into words based on dictionary.
@@ -104,12 +147,15 @@ class JiebaTokenizer(TextTensorOperation):
         mp_path (str): Dictionary file is used by MPSegment algorithm.
             The dictionary can be obtained on the official website of cppjieba.
         mode (JiebaMode, optional): Valid values can be any of [JiebaMode.MP, JiebaMode.HMM,
-            JiebaMode.MIX](default=JiebaMode.MIX).
+            JiebaMode.MIX]. Default: JiebaMode.MIX.
 
             - JiebaMode.MP, tokenize with MPSegment algorithm.
+
             - JiebaMode.HMM, tokenize with Hidden Markov Model Segment algorithm.
+
             - JiebaMode.MIX, tokenize with a mix of MPSegment and HMMSegment algorithm.
-        with_offsets (bool, optional): Whether or not output offsets of tokens (default=False).
+
+        with_offsets (bool, optional): Whether or not output offsets of tokens. Default: False.
 
     Raises:
         ValueError: If path of HMMSegment dict is not provided.
@@ -121,6 +167,7 @@ class JiebaTokenizer(TextTensorOperation):
         ``CPU``
 
     Examples:
+        >>> import mindspore.dataset.text as text
         >>> from mindspore.dataset.text import JiebaMode
         >>> # If with_offsets=False, default output one column {["text", dtype=str]}
         >>> jieba_hmm_file = "/path/to/jieba/hmm/file"
@@ -131,8 +178,7 @@ class JiebaTokenizer(TextTensorOperation):
         >>> #                                                   ["offsets_limit", dtype=uint32]}
         >>> tokenizer_op = text.JiebaTokenizer(jieba_hmm_file, jieba_mp_file, mode=JiebaMode.MP, with_offsets=True)
         >>> text_file_dataset_1 = text_file_dataset_1.map(operations=tokenizer_op, input_columns=["text"],
-        ...                                               output_columns=["token", "offsets_start", "offsets_limit"],
-        ...                                               column_order=["token", "offsets_start", "offsets_limit"])
+        ...                                               output_columns=["token", "offsets_start", "offsets_limit"])
     """
 
     @check_jieba_init
@@ -173,9 +219,10 @@ class JiebaTokenizer(TextTensorOperation):
             word (str): The word to be added to the JiebaTokenizer instance.
                 The added word will not be written into the built-in dictionary on disk.
             freq (int, optional): The frequency of the word to be added. The higher the frequency,
-                the better chance the word will be tokenized (default=None, use default frequency).
+                the better chance the word will be tokenized. Default: None, use default frequency.
 
         Examples:
+            >>> import mindspore.dataset.text as text
             >>> from mindspore.dataset.text import JiebaMode
             >>> jieba_hmm_file = "/path/to/jieba/hmm/file"
             >>> jieba_mp_file = "/path/to/jieba/mp/file"
@@ -280,9 +327,9 @@ class Lookup(TextTensorOperation):
         vocab (Vocab): A vocabulary object.
         unknown_token (str, optional): Word is used for lookup. In case of the word is out of vocabulary (OOV),
             the result of lookup will be replaced with unknown_token. If the unknown_token is not specified or
-            it is OOV, runtime error will be thrown (default=None, means no unknown_token is specified).
+            it is OOV, runtime error will be thrown. Default: None, means no unknown_token is specified.
         data_type (mindspore.dtype, optional): The data type that lookup operation maps
-            string to(default=mindspore.int32).
+            string to. Default: mindspore.int32.
 
     Raises:
         TypeError: If `vocab` is not of type text.Vocab.
@@ -293,9 +340,10 @@ class Lookup(TextTensorOperation):
         ``CPU``
 
     Examples:
+        >>> import mindspore.dataset.text as text
         >>> # Load vocabulary from list
         >>> vocab = text.Vocab.from_list(['深', '圳', '欢', '迎', '您'])
-        >>> # Use Lookup operator to map tokens to ids
+        >>> # Use Lookup operation to map tokens to ids
         >>> lookup = text.Lookup(vocab)
         >>> text_file_dataset = text_file_dataset.map(operations=[lookup])
     """
@@ -325,13 +373,13 @@ class Ngram(TextTensorOperation):
             an empty string produced.
         left_pad (tuple, optional): Padding performed on left side of the sequence shaped like ("pad_token", pad_width).
             `pad_width` will be capped at n-1. For example, specifying left_pad=("_", 2) would pad left side of the
-            sequence with "__" (default=("", 0)).
+            sequence with "__". Default: ('', 0).
         right_pad (tuple, optional): Padding performed on right side of the sequence shaped like
             ("pad_token", pad_width). `pad_width` will be capped at n-1. For example, specifying right_pad=("_", 2)
-            would pad right side of the sequence with "__" (default=("", 0)).
+            would pad right side of the sequence with "__". Default: ('', 0).
         separator (str, optional): Symbol used to join strings together. For example, if 2-gram is
-            ["mindspore", "amazing"] with separator="-", the result would be ["mindspore-amazing"]
-            (default=" ", which will use whitespace as separator).
+            ["mindspore", "amazing"] with separator="-", the result would be ["mindspore-amazing"].
+            Default: ' ', which will use whitespace as separator.
 
     Raises:
         TypeError: If values of `n` not positive is not of type int.
@@ -344,6 +392,7 @@ class Ngram(TextTensorOperation):
         ``CPU``
 
     Examples:
+        >>> import mindspore.dataset.text as text
         >>> ngram_op = text.Ngram(3, separator="-")
         >>> output = ngram_op(["WildRose Country", "Canada's Ocean Playground", "Land of Living Skies"])
         >>> # output
@@ -429,6 +478,7 @@ class SentencePieceTokenizer(TextTensorOperation):
         ``CPU``
 
     Examples:
+        >>> import mindspore.dataset.text as text
         >>> from mindspore.dataset.text import SentencePieceModel, SPieceTokenizerOutType
         >>> sentence_piece_vocab_file = "/path/to/sentence/piece/vocab/file"
         >>> vocab = text.SentencePieceVocab.from_file([sentence_piece_vocab_file], 5000, 0.9995,
@@ -455,7 +505,7 @@ class SlidingWindow(TextTensorOperation):
 
     Args:
         width (int): The width of the window. It must be an integer and greater than zero.
-        axis (int, optional): The axis along which the sliding window is computed (default=0).
+        axis (int, optional): The axis along which the sliding window is computed. Default: 0.
 
     Raises:
         TypeError: If `width` is not of type int.
@@ -466,6 +516,7 @@ class SlidingWindow(TextTensorOperation):
         ``CPU``
 
     Examples:
+        >>> import mindspore.dataset as ds
         >>> dataset = ds.NumpySlicesDataset(data=[[1, 2, 3, 4, 5]], column_names="col1")
         >>> # Data before
         >>> # |     col1     |
@@ -512,6 +563,8 @@ class ToNumber(TextTensorOperation):
         ``CPU``
 
     Examples:
+        >>> import mindspore.dataset as ds
+        >>> import mindspore.dataset.text as text
         >>> from mindspore import dtype as mstype
         >>> data = [["1", "2", "3"]]
         >>> dataset = ds.NumpySlicesDataset(data)
@@ -535,11 +588,11 @@ class ToVectors(TextTensorOperation):
 
     Args:
         vectors (Vectors): A vectors object.
-        unk_init (sequence, optional): Sequence used to initialize out-of-vectors (OOV) token
-            (default=None, initialize with zero vectors).
+        unk_init (sequence, optional): Sequence used to initialize out-of-vectors (OOV) token.
+            Default: None, initialize with zero vectors.
         lower_case_backup (bool, optional): Whether to look up the token in the lower case. If False, each token in the
             original case will be looked up; if True, each token in the original case will be looked up first, if not
-            found in the keys of the property stoi, the token in the lower case will be looked up (default=False).
+            found in the keys of the property stoi, the token in the lower case will be looked up. Default: False.
 
     Raises:
         TypeError: If `unk_init` is not of type sequence.
@@ -550,9 +603,10 @@ class ToVectors(TextTensorOperation):
         ``CPU``
 
     Examples:
+        >>> import mindspore.dataset.text as text
         >>> # Load vectors from file
         >>> vectors = text.Vectors.from_file("/path/to/vectors/file")
-        >>> # Use ToVectors operator to map tokens to vectors
+        >>> # Use ToVectors operation to map tokens to vectors
         >>> to_vectors = text.ToVectors(vectors)
         >>> text_file_dataset = text_file_dataset.map(operations=[to_vectors])
     """
@@ -568,6 +622,46 @@ class ToVectors(TextTensorOperation):
         return cde.ToVectorsOperation(self.vectors, self.unk_init, self.lower_case_backup)
 
 
+class Truncate(TextTensorOperation):
+    """
+    Truncate the input sequence so that it does not exceed the maximum length.
+
+    Args:
+        max_seq_len (int): Maximum allowable length.
+
+    Raises:
+        TypeError: If `max_length_len` is not of type int.
+        ValueError: If value of `max_length_len` is not greater than or equal to 0.
+        RuntimeError: If the input tensor is not of dtype bool, int, float, double or str.
+
+    Supported Platforms:
+        ``CPU``
+
+    Examples:
+        >>> dataset = ds.NumpySlicesDataset(data=[['a', 'b', 'c', 'd', 'e']], column_names=["text"], shuffle=False)
+        >>> # Data before
+        >>> # |           col1            |
+        >>> # +---------------------------+
+        >>> # | ['a', 'b', 'c', 'd', 'e'] |
+        >>> # +---------------------------+
+        >>> truncate = text.Truncate(4)
+        >>> dataset = dataset.map(operations=truncate, input_columns=["text"])
+        >>> # Data after
+        >>> # |          col1          |
+        >>> # +------------------------+
+        >>> # |  ['a', 'b', 'c', 'd']  |
+        >>> # +------------------------+
+    """
+
+    @check_truncate
+    def __init__(self, max_seq_len):
+        super().__init__()
+        self.max_seq_len = max_seq_len
+
+    def parse(self):
+        return cde.TruncateOperation(self.max_seq_len)
+
+
 class TruncateSequencePair(TextTensorOperation):
     """
     Truncate a pair of rank-1 tensors such that the total length is less than max_length.
@@ -584,6 +678,7 @@ class TruncateSequencePair(TextTensorOperation):
         ``CPU``
 
     Examples:
+        >>> import mindspore.dataset.text as text
         >>> dataset = ds.NumpySlicesDataset(data={"col1": [[1, 2, 3]], "col2": [[4, 5]]})
         >>> # Data before
         >>> # |   col1    |   col2    |
@@ -613,7 +708,7 @@ class UnicodeCharTokenizer(TextTensorOperation):
     Tokenize a scalar tensor of UTF-8 string to Unicode characters.
 
     Args:
-        with_offsets (bool, optional): Whether or not output offsets of tokens (default=False).
+        with_offsets (bool, optional): Whether or not output offsets of tokens. Default: False.
 
     Raises:
         TypeError: If `with_offsets` is not of type bool.
@@ -622,6 +717,7 @@ class UnicodeCharTokenizer(TextTensorOperation):
         ``CPU``
 
     Examples:
+        >>> import mindspore.dataset.text as text
         >>> # If with_offsets=False, default output one column {["text", dtype=str]}
         >>> tokenizer_op = text.UnicodeCharTokenizer(with_offsets=False)
         >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op)
@@ -629,8 +725,7 @@ class UnicodeCharTokenizer(TextTensorOperation):
         >>> #                                                   ["offsets_limit", dtype=uint32]}
         >>> tokenizer_op = text.UnicodeCharTokenizer(with_offsets=True)
         >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op, input_columns=["text"],
-        ...                                           output_columns=["token", "offsets_start", "offsets_limit"],
-        ...                                           column_order=["token", "offsets_start", "offsets_limit"])
+        ...                                           output_columns=["token", "offsets_start", "offsets_limit"])
     """
 
     @check_with_offsets
@@ -657,7 +752,7 @@ class WordpieceTokenizer(TextTensorOperation):
         with_offsets (bool, optional): Whether to return the offsets of tokens. Default: False.
 
     Raises:
-        TypeError: If `vocab` is not of type :class:`mindspore.dataset.text.Vocab`.
+        TypeError: If `vocab` is not of type :class:`mindspore.dataset.text.Vocab` .
         TypeError: If `suffix_indicator` is not of type str.
         TypeError: If `max_bytes_per_token` is not of type int.
         TypeError: If `unknown_token` is not of type str.
@@ -668,6 +763,7 @@ class WordpieceTokenizer(TextTensorOperation):
         ``CPU``
 
     Examples:
+        >>> import mindspore.dataset.text as text
         >>> vocab_list = ["book", "cholera", "era", "favor", "##ite", "my", "is", "love", "dur", "##ing", "the"]
         >>> vocab = text.Vocab.from_list(vocab_list)
         >>> # If with_offsets=False, default output one column {["text", dtype=str]}
@@ -679,8 +775,7 @@ class WordpieceTokenizer(TextTensorOperation):
         >>> tokenizer_op = text.WordpieceTokenizer(vocab=vocab, unknown_token='[UNK]',
         ...                                       max_bytes_per_token=100, with_offsets=True)
         >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op, input_columns=["text"],
-        ...                                           output_columns=["token", "offsets_start", "offsets_limit"],
-        ...                                           column_order=["token", "offsets_start", "offsets_limit"])
+        ...                                           output_columns=["token", "offsets_start", "offsets_limit"])
     """
 
     @check_wordpiece_tokenizer
@@ -718,10 +813,10 @@ if platform.system().lower() != 'windows':
         Args:
             lower_case (bool, optional): Whether to perform lowercase processing on the text. If True, will fold the
                 text to lower case and strip accented characters. If False, will only perform normalization on the
-                text, with mode specified by `normalization_form`. Default: False.
+                text, with mode specified by `normalization_form` . Default: False.
             keep_whitespace (bool, optional): If True, the whitespace will be kept in the output. Default: False.
             normalization_form (NormalizeForm, optional):
-                `Unicode normalization forms <http://unicode.org/reports/tr15/>`_, only valid when `lower_case`
+                `Unicode normalization forms <http://unicode.org/reports/tr15/>`_ , only valid when `lower_case`
                 is False, can be NormalizeForm.NONE, NormalizeForm.NFC, NormalizeForm.NFKC, NormalizeForm.NFD or
                 NormalizeForm.NFKD. Default: NormalizeForm.NONE.
 
@@ -738,7 +833,7 @@ if platform.system().lower() != 'windows':
         Raises:
             TypeError: If `lower_case` is not of type bool.
             TypeError: If `keep_whitespace` is not of type bool.
-            TypeError: If `normalization_form` is not of type :class:`mindspore.dataset.text.NormalizeForm`.
+            TypeError: If `normalization_form` is not of type :class:`mindspore.dataset.text.NormalizeForm` .
             TypeError: If `preserve_unused_token` is not of type bool.
             TypeError: If `with_offsets` is not of type bool.
             RuntimeError: If dtype of input Tensor is not str.
@@ -747,6 +842,7 @@ if platform.system().lower() != 'windows':
             ``CPU``
 
         Examples:
+            >>> import mindspore.dataset.text as text
             >>> from mindspore.dataset.text import NormalizeForm
             >>>
             >>> # If with_offsets=False, default output one column {["text", dtype=str]}
@@ -766,9 +862,7 @@ if platform.system().lower() != 'windows':
             ...                                    with_offsets=True)
             >>> text_file_dataset_1 = text_file_dataset_1.map(operations=tokenizer_op, input_columns=["text"],
             ...                                               output_columns=["token", "offsets_start",
-            ...                                                               "offsets_limit"],
-            ...                                               column_order=["token", "offsets_start",
-            ...                                                             "offsets_limit"])
+            ...                                                               "offsets_limit"])
         """
 
         @check_basic_tokenizer
@@ -806,10 +900,10 @@ if platform.system().lower() != 'windows':
                 output. Default: '[UNK]'.
             lower_case (bool, optional): Whether to perform lowercase processing on the text. If True, will fold the
                 text to lower case and strip accented characters. If False, will only perform normalization on the
-                text, with mode specified by `normalization_form`. Default: False.
+                text, with mode specified by `normalization_form` . Default: False.
             keep_whitespace (bool, optional): If True, the whitespace will be kept in the output. Default: False.
             normalization_form (NormalizeForm, optional):
-                `Unicode normalization forms <http://unicode.org/reports/tr15/>`_, only valid when `lower_case`
+                `Unicode normalization forms <http://unicode.org/reports/tr15/>`_ , only valid when `lower_case`
                 is False, can be NormalizeForm.NONE, NormalizeForm.NFC, NormalizeForm.NFKC, NormalizeForm.NFD or
                 NormalizeForm.NFKD. Default: NormalizeForm.NONE.
 
@@ -824,14 +918,14 @@ if platform.system().lower() != 'windows':
             with_offsets (bool, optional): Whether to return the offsets of tokens. Default: False.
 
         Raises:
-            TypeError: If `vocab` is not of type :class:`mindspore.dataset.text.Vocab`.
+            TypeError: If `vocab` is not of type :class:`mindspore.dataset.text.Vocab` .
             TypeError: If `suffix_indicator` is not of type str.
             TypeError: If `max_bytes_per_token` is not of type int.
             ValueError: If `max_bytes_per_token` is negative.
             TypeError: If `unknown_token` is not of type str.
             TypeError: If `lower_case` is not of type bool.
             TypeError: If `keep_whitespace` is not of type bool.
-            TypeError: If `normalization_form` is not of type :class:`mindspore.dataset.text.NormalizeForm`.
+            TypeError: If `normalization_form` is not of type :class:`mindspore.dataset.text.NormalizeForm` .
             TypeError: If `preserve_unused_token` is not of type bool.
             TypeError: If `with_offsets` is not of type bool.
 
@@ -839,6 +933,7 @@ if platform.system().lower() != 'windows':
             ``CPU``
 
         Examples:
+            >>> import mindspore.dataset.text as text
             >>> from mindspore.dataset.text import NormalizeForm
             >>>
             >>> # If with_offsets=False, default output one column {["text", dtype=str]}
@@ -854,17 +949,15 @@ if platform.system().lower() != 'windows':
             ...                                   with_offsets=False)
             >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op)
             >>> # If with_offsets=True, then output three columns {["token", dtype=str],
-            >>> #                                                   ["offsets_start", dtype=uint32],
-            >>> #                                                   ["offsets_limit", dtype=uint32]}
+            >>> #                                                  ["offsets_start", dtype=uint32],
+            >>> #                                                  ["offsets_limit", dtype=uint32]}
             >>> tokenizer_op = text.BertTokenizer(vocab=vocab, suffix_indicator='##', max_bytes_per_token=100,
             ...                                   unknown_token='[UNK]', lower_case=False, keep_whitespace=False,
             ...                                   normalization_form=NormalizeForm.NONE, preserve_unused_token=True,
             ...                                   with_offsets=True)
             >>> text_file_dataset_1 = text_file_dataset_1.map(operations=tokenizer_op, input_columns=["text"],
             ...                                               output_columns=["token", "offsets_start",
-            ...                                                               "offsets_limit"],
-            ...                                               column_order=["token", "offsets_start",
-            ...                                                             "offsets_limit"])
+            ...                                                               "offsets_limit"])
         """
 
         @check_bert_tokenizer
@@ -894,7 +987,7 @@ if platform.system().lower() != 'windows':
     class CaseFold(TextTensorOperation):
         """
         Apply case fold operation on UTF-8 string tensor, which is aggressive that can convert more characters into
-        lower case than :func:`str.lower`. For supported normalization forms, please refer to
+        lower case than :code:`str.lower` . For supported normalization forms, please refer to
         `ICU_Normalizer2 <https://unicode-org.github.io/icu-docs/apidoc/released/icu4c/classicu_1_1Normalizer2.html>`_ .
 
         Note:
@@ -904,6 +997,7 @@ if platform.system().lower() != 'windows':
             ``CPU``
 
         Examples:
+            >>> import mindspore.dataset.text as text
             >>> case_op = text.CaseFold()
             >>> text_file_dataset = text_file_dataset.map(operations=case_op)
         """
@@ -924,7 +1018,7 @@ if platform.system().lower() != 'windows':
             ``CPU``
 
         Examples:
-            >>> import mindspore.dataset.text.transforms as text
+            >>> import mindspore.dataset.text as text
             >>>
             >>> replace_op = text.FilterWikipediaXML()
             >>> text_file_dataset = text_file_dataset.map(operations=replace_op)
@@ -944,7 +1038,7 @@ if platform.system().lower() != 'windows':
         Args:
             normalize_form (NormalizeForm, optional): Valid values can be [NormalizeForm.NONE, NormalizeForm.NFC,
                 NormalizeForm.NFKC, NormalizeForm.NFD, NormalizeForm.NFKD] any of the four unicode
-                normalized forms(default=NormalizeForm.NFKC).
+                normalized forms. Default: NormalizeForm.NFKC.
                 See http://unicode.org/reports/tr15/ for details.
 
                 - NormalizeForm.NONE, do nothing for input string tensor.
@@ -960,6 +1054,7 @@ if platform.system().lower() != 'windows':
             ``CPU``
 
         Examples:
+            >>> import mindspore.dataset.text as text
             >>> from mindspore.dataset.text import NormalizeForm
             >>> normalize_op = text.NormalizeUTF8(normalize_form=NormalizeForm.NFC)
             >>> text_file_dataset = text_file_dataset.map(operations=normalize_op)
@@ -990,7 +1085,7 @@ if platform.system().lower() != 'windows':
             pattern (str): the regex expression patterns.
             replace (str): the string to replace matched element.
             replace_all (bool, optional): If False, only replace first matched element;
-                if True, replace all matched elements (default=True).
+                if True, replace all matched elements. Default: True.
 
         Raises:
             TypeError: If `pattern` is not of type string.
@@ -1001,6 +1096,7 @@ if platform.system().lower() != 'windows':
             ``CPU``
 
         Examples:
+            >>> import mindspore.dataset.text as text
             >>> pattern = 'Canada'
             >>> replace = 'China'
             >>> replace_op = text.RegexReplace(pattern, replace)
@@ -1032,8 +1128,8 @@ if platform.system().lower() != 'windows':
                 The original string will be split by matched elements.
             keep_delim_pattern (str, optional): The string matched by 'delim_pattern' can be kept as a token
                 if it can be matched by 'keep_delim_pattern'. The default value is an empty str
-                which means that delimiters will not be kept as an output token (default='').
-            with_offsets (bool, optional): Whether or not output offsets of tokens(default=False).
+                which means that delimiters will not be kept as an output token. Default: ''.
+            with_offsets (bool, optional): Whether or not output offsets of tokens. Default: False.
 
         Raises:
             TypeError: If `delim_pattern` is not of type string.
@@ -1044,6 +1140,7 @@ if platform.system().lower() != 'windows':
             ``CPU``
 
         Examples:
+            >>> import mindspore.dataset.text as text
             >>> # If with_offsets=False, default output is one column {["text", dtype=str]}
             >>> delim_pattern = r"[ |,]"
             >>> tokenizer_op = text.RegexTokenizer(delim_pattern, with_offsets=False)
@@ -1054,9 +1151,7 @@ if platform.system().lower() != 'windows':
             >>> tokenizer_op = text.RegexTokenizer(delim_pattern, with_offsets=True)
             >>> text_file_dataset_1 = text_file_dataset_1.map(operations=tokenizer_op, input_columns=["text"],
             ...                                               output_columns=["token", "offsets_start",
-            ...                                                               "offsets_limit"],
-            ...                                               column_order=["token", "offsets_start",
-            ...                                                             "offsets_limit"])
+            ...                                                               "offsets_limit"])
         """
 
         @check_regex_tokenizer
@@ -1078,8 +1173,8 @@ if platform.system().lower() != 'windows':
             UnicodeScriptTokenizer is not supported on Windows platform yet.
 
         Args:
-            keep_whitespace (bool, optional): Whether or not emit whitespace tokens (default=False).
-            with_offsets (bool, optional): Whether or not output offsets of tokens (default=False).
+            keep_whitespace (bool, optional): Whether or not emit whitespace tokens. Default: False.
+            with_offsets (bool, optional): Whether or not output offsets of tokens. Default: False.
 
         Raises:
             TypeError: If `keep_whitespace` is not of type bool.
@@ -1089,6 +1184,7 @@ if platform.system().lower() != 'windows':
             ``CPU``
 
         Examples:
+            >>> import mindspore.dataset.text as text
             >>> # If with_offsets=False, default output one column {["text", dtype=str]}
             >>> tokenizer_op = text.UnicodeScriptTokenizer(keep_whitespace=True, with_offsets=False)
             >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op)
@@ -1097,8 +1193,7 @@ if platform.system().lower() != 'windows':
             >>> #                                                  ["offsets_limit", dtype=uint32]}
             >>> tokenizer_op = text.UnicodeScriptTokenizer(keep_whitespace=True, with_offsets=True)
             >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op, input_columns=["text"],
-            ...                                           output_columns=["token", "offsets_start", "offsets_limit"],
-            ...                                           column_order=["token", "offsets_start", "offsets_limit"])
+            ...                                           output_columns=["token", "offsets_start", "offsets_limit"])
 
         """
 
@@ -1122,7 +1217,7 @@ if platform.system().lower() != 'windows':
             WhitespaceTokenizer is not supported on Windows platform yet.
 
         Args:
-            with_offsets (bool, optional): Whether or not output offsets of tokens (default=False).
+            with_offsets (bool, optional): Whether or not output offsets of tokens. Default: False.
 
         Raises:
             TypeError: If `with_offsets` is not of type bool.
@@ -1131,6 +1226,7 @@ if platform.system().lower() != 'windows':
             ``CPU``
 
         Examples:
+            >>> import mindspore.dataset.text as text
             >>> # If with_offsets=False, default output one column {["text", dtype=str]}
             >>> tokenizer_op = text.WhitespaceTokenizer(with_offsets=False)
             >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op)
@@ -1139,8 +1235,7 @@ if platform.system().lower() != 'windows':
             >>> #                                                   ["offsets_limit", dtype=uint32]}
             >>> tokenizer_op = text.WhitespaceTokenizer(with_offsets=True)
             >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op, input_columns=["text"],
-            ...                                           output_columns=["token", "offsets_start", "offsets_limit"],
-            ...                                           column_order=["token", "offsets_start", "offsets_limit"])
+            ...                                           output_columns=["token", "offsets_start", "offsets_limit"])
         """
 
         @check_with_offsets