mindspore 2.0.0a0__cp39-cp39-win_amd64.whl → 2.0.0rc1__cp39-cp39-win_amd64.whl

mindspore/dataset/text/transforms.py

@@ -49,11 +49,11 @@ import mindspore._c_dataengine as cde
 from mindspore.common import dtype as mstype
 
 from .utils import JiebaMode, NormalizeForm, to_str, SPieceTokenizerOutType, SPieceTokenizerLoadType, SentencePieceVocab
-from .validators import check_lookup, check_jieba_add_dict, check_to_vectors, \
+from .validators import check_add_token, check_lookup, check_jieba_add_dict, check_to_vectors, \
     check_jieba_add_word, check_jieba_init, check_with_offsets, check_unicode_script_tokenizer, \
     check_wordpiece_tokenizer, check_regex_replace, check_regex_tokenizer, check_basic_tokenizer, check_ngram, \
     check_pair_truncate, check_to_number, check_bert_tokenizer, check_python_tokenizer, check_slidingwindow, \
-    check_sentence_piece_tokenizer
+    check_sentence_piece_tokenizer, check_truncate
 from ..core.datatypes import mstype_to_detype
 from ..core.validator_helpers import replace_none
 from ..transforms.py_transforms_util import Implementation
@@ -91,6 +91,49 @@ DE_C_INTER_SENTENCEPIECE_OUTTYPE = {
 }
 
 
+class AddToken(TextTensorOperation):
+    """
+    Add token to beginning or end of sequence.
+
+    Args:
+        token (str): The token to be added.
+        begin (bool, optional): Choose the position where the token is inserted. If True,
+            the token will be inserted at the beginning of the sequence. Otherwise, it will
+            be inserted at the end of the sequence. Default: True.
+
+    Raises:
+        TypeError: If `token` is not of type string.
+        TypeError: If `begin` is not of type bool.
+
+    Supported Platforms:
+        ``CPU``
+
+    Examples:
+        >>> dataset = ds.NumpySlicesDataset(data={"text": [['a', 'b', 'c', 'd', 'e']]})
+        >>> # Data before
+        >>> # |           text            |
+        >>> # +---------------------------+
+        >>> # | ['a', 'b', 'c', 'd', 'e'] |
+        >>> # +---------------------------+
+        >>> add_token_op = text.AddToken(token='TOKEN', begin=True)
+        >>> dataset = dataset.map(operations=add_token_op)
+        >>> # Data after
+        >>> # |           text            |
+        >>> # +---------------------------+
+        >>> # | ['TOKEN', 'a', 'b', 'c', 'd', 'e'] |
+        >>> # +---------------------------+
+    """
+
+    @check_add_token
+    def __init__(self, token, begin=True):
+        super().__init__()
+        self.token = token
+        self.begin = begin
+
+    def parse(self):
+        return cde.AddTokenOperation(self.token, self.begin)
+
+
 class JiebaTokenizer(TextTensorOperation):
     """
     Tokenize Chinese string into words based on dictionary.
@@ -579,6 +622,46 @@ class ToVectors(TextTensorOperation):
         return cde.ToVectorsOperation(self.vectors, self.unk_init, self.lower_case_backup)
 
 
+class Truncate(TextTensorOperation):
+    """
+    Truncate the input sequence so that it does not exceed the maximum length.
+
+    Args:
+        max_seq_len (int): Maximum allowable length.
+
+    Raises:
+        TypeError: If `max_length_len` is not of type int.
+        ValueError: If value of `max_length_len` is not greater than or equal to 0.
+        RuntimeError: If the input tensor is not of dtype bool, int, float, double or str.
+
+    Supported Platforms:
+        ``CPU``
+
+    Examples:
+        >>> dataset = ds.NumpySlicesDataset(data=[['a', 'b', 'c', 'd', 'e']], column_names=["text"], shuffle=False)
+        >>> # Data before
+        >>> # |           col1            |
+        >>> # +---------------------------+
+        >>> # | ['a', 'b', 'c', 'd', 'e'] |
+        >>> # +---------------------------+
+        >>> truncate = text.Truncate(4)
+        >>> dataset = dataset.map(operations=truncate, input_columns=["text"])
+        >>> # Data after
+        >>> # |          col1          |
+        >>> # +------------------------+
+        >>> # |  ['a', 'b', 'c', 'd']  |
+        >>> # +------------------------+
+    """
+
+    @check_truncate
+    def __init__(self, max_seq_len):
+        super().__init__()
+        self.max_seq_len = max_seq_len
+
+    def parse(self):
+        return cde.TruncateOperation(self.max_seq_len)
+
+
 class TruncateSequencePair(TextTensorOperation):
     """
     Truncate a pair of rank-1 tensors such that the total length is less than max_length.
@@ -904,7 +987,7 @@ if platform.system().lower() != 'windows':
     class CaseFold(TextTensorOperation):
         """
         Apply case fold operation on UTF-8 string tensor, which is aggressive that can convert more characters into
-        lower case than :func:`str.lower` . For supported normalization forms, please refer to
+        lower case than :code:`str.lower` . For supported normalization forms, please refer to
         `ICU_Normalizer2 <https://unicode-org.github.io/icu-docs/apidoc/released/icu4c/classicu_1_1Normalizer2.html>`_ .
 
         Note:

mindspore/dataset/text/utils.py

@@ -246,9 +246,10 @@ class SentencePieceVocab:
         """
 
         sentence_piece_vocab = cls()
-        sentence_piece_vocab.c_sentence_piece_vocab = dataset.build_sentencepiece_vocab(col_names, vocab_size,
-                                                                                        character_coverage,
-                                                                                        model_type, params)
+        # pylint: disable=protected-access
+        sentence_piece_vocab.c_sentence_piece_vocab = dataset._build_sentencepiece_vocab(col_names, vocab_size,
+                                                                                         character_coverage,
+                                                                                         model_type, params)
         return sentence_piece_vocab
 
     @classmethod
@@ -428,7 +429,8 @@ class Vocab:
         """
 
         vocab = cls()
-        vocab.c_vocab = dataset.build_vocab(columns, freq_range, top_k, special_tokens, special_first)
+        # pylint: disable=protected-access
+        vocab.c_vocab = dataset._build_vocab(columns, freq_range, top_k, special_tokens, special_first)
         return vocab
 
     @classmethod

mindspore/dataset/text/validators.py

@@ -27,6 +27,19 @@ from ..core.validator_helpers import parse_user_args, type_check, type_check_lis
     INT32_MAX, check_value, check_positive, check_pos_int32, check_filename, check_non_negative_int32
 
 
+def check_add_token(method):
+    """Wrapper method to check the parameters of add token."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        [token, begin], _ = parse_user_args(method, *args, **kwargs)
+        type_check(token, (str,), "token")
+        type_check(begin, (bool,), "begin")
+        return method(self, *args, **kwargs)
+
+    return new_method
+
+
 def check_unique_list_of_words(words, arg_name):
     """Check that words is a list and each element is a str without any duplication"""
 
@@ -443,6 +456,18 @@ def check_ngram(method):
     return new_method
 
 
+def check_truncate(method):
+    """Wrapper method to check the parameters of number of truncate."""
+
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        [max_seq_len], _ = parse_user_args(method, *args, **kwargs)
+        check_pos_int32(max_seq_len, "max_seq_len")
+        return method(self, *args, **kwargs)
+
+    return new_method
+
+
 def check_pair_truncate(method):
     """Wrapper method to check the parameters of number of pair truncate."""
 

mindspore/dataset/transforms/__init__.py

@@ -31,13 +31,14 @@ Note: Legacy c_transforms and py_transforms are deprecated but can still be impo
     from mindspore.dataset.transforms import py_transforms
 
 See `Common Transforms
-<https://www.mindspore.cn/tutorials/en/r2.0.0-alpha/beginner/transforms.html#common-transforms>`_ tutorial
-for more details.
+<https://www.mindspore.cn/tutorials/en/r2.0/beginner/transforms.html#common-transforms>`_ tutorial for more details.
 
 Descriptions of common data processing terms are as follows:
 
 - TensorOperation, the base class of all data processing operations implemented in C++.
 - PyTensorOperation, the base class of all data processing operations implemented in Python.
+
+Note: In eager mode, non-NumPy input is implicitly converted to NumPy format and sent to MindSpore.
 """
 from .. import vision
 from . import c_transforms

mindspore/dataset/transforms/c_transforms.py

@@ -142,7 +142,7 @@ class TypeCast(TensorOperation):
         TypeError: If `data_type` is not of type bool, int, float or string.
 
     Supported Platforms:
-        ``CPU`` ``Ascend`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> import numpy as np

mindspore/dataset/transforms/transforms.py

@@ -889,14 +889,14 @@ class TypeCast(TensorOperation):
         This operation supports running on Ascend or GPU platforms by Offload.
 
     Args:
-        data_type (Union[mindspore.dtype, numpy.dtype]): mindspore.dtype or numpy.dtype (e.g. :class:`numpy.float32`)
+        data_type (Union[mindspore.dtype, numpy.dtype]): mindspore.dtype or numpy.dtype (e.g. `numpy.float32`)
             to be cast to.
 
     Raises:
         TypeError: If `data_type` is not of MindSpore data type bool, int, float, string or type :class:`numpy.dtype` .
 
     Supported Platforms:
-        ``CPU`` ``Ascend`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> import numpy as np

mindspore/dataset/utils/__init__.py

@@ -14,5 +14,6 @@
 # ==============================================================================
 """init file for MindData utils"""
 from .browse_dataset import imshow_det_bbox
+from .line_reader import LineReader
 
-__all__ = ["imshow_det_bbox"]
+__all__ = ["imshow_det_bbox", "LineReader"]

mindspore/dataset/utils/line_reader.py

@@ -0,0 +1,121 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Efficient line based file reading.
+"""
+import os
+
+from mindspore import log as logger
+from ..core.validator_helpers import check_filename, check_uint64, check_value
+
+
+class LineReader:
+    """
+    Efficient file line reader.
+
+    This class is used to hold the line offsets of line based file.
+
+    The following functionality is provided:
+
+    - len() - return the number of lines in the file
+
+    - readline(line) - open file handle (if not opened yet), and read a line in the file
+
+    - close() - close the file handle opened in readline
+
+    Args:
+        filename (str): line based file to be loaded.
+
+    Raises:
+        TypeError: Parameter `filename` is wrong.
+        RuntimeError: The input file does not exist or is not a regular file.
+    """
+
+    def __init__(self, filename):
+        check_filename(filename)
+        self.filename = os.path.realpath(filename)
+
+        if not os.path.exists(self.filename):
+            raise RuntimeError("The input file [{}] does not exist.".format(filename))
+
+        if not os.path.isfile(self.filename):
+            raise RuntimeError("The input file [{}] is not a regular file.".format(filename))
+
+        # get the line offsets
+        self.offsets = [0]
+        with open(self.filename, mode='rb') as fo:
+            while fo.readline():
+                self.offsets.append(fo.tell())
+
+        # pop the last empty line offset
+        self.offsets.pop()
+        if not self.offsets:
+            logger.warning("The input file [{}] is empty.".format(filename))
+
+        # will be init in readline
+        self.fo_handle = None
+
+    def __getitem__(self, line):
+        """Read specified line content"""
+        return self.readline(line)
+
+    def __len__(self):
+        """Get the total number of lines in the file"""
+        return self.len()
+
+    def __del__(self):
+        """Close the file when object released"""
+        self.close()
+
+    def len(self):
+        """Get the total number of lines in the file"""
+        return len(self.offsets)
+
+    def readline(self, line):
+        """
+        Read specified line content.
+
+        Args:
+            line (int): the line number to be read, starting at 1.
+
+        Returns:
+            str, line content (until line break character).
+
+        Raises:
+            TypeError: Parameter `line` is the wrong type.
+            ValueError: Parameter `line` exceeds the file range.
+        """
+        check_uint64(line, "line")
+        check_value(line, [1, len(self.offsets)], "line")
+        if self.fo_handle is None:
+            self.fo_handle = open(self.filename, mode="rt")
+
+        self.fo_handle.seek(self.offsets[line - 1])
+        content = self.fo_handle.readline()
+
+        # remove the line break character
+        if content.endswith("\r\n"):
+            content = content[:-2]
+        elif content.endswith("\n"):
+            content = content[:-1]
+        elif content.endswith("\r"):
+            content = content[:-1]
+        return content
+
+    def close(self):
+        """Close the file"""
+        if self.fo_handle is None:
+            return
+        self.fo_handle.close()
+        self.fo_handle = None

mindspore/dataset/vision/__init__.py

@@ -32,8 +32,7 @@ Note: Legacy c_transforms and py_transforms are deprecated but can still be impo
     import mindspore.dataset.vision.py_transforms as py_vision
 
 See `Vision Transforms
-<https://www.mindspore.cn/tutorials/en/r2.0.0-alpha/beginner/transforms.html#vision-transforms>`_ tutorial
-for more details.
+<https://www.mindspore.cn/tutorials/en/r2.0/beginner/transforms.html#vision-transforms>`_ tutorial for more details.
 
 Descriptions of common data processing terms are as follows:
 
@@ -44,7 +43,7 @@ Descriptions of common data processing terms are as follows:
 The data transform operation can be executed in the data processing pipeline or in the eager mode:
 
 - Pipeline mode is generally used to process datasets. For examples, please refer to
-  `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/r2.0.0-alpha/api_python/
+  `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/r2.0/api_python/
   mindspore.dataset.html#introduction-to-data-processing-pipeline>`_ .
 - Eager mode is generally used for scattered samples. Examples of image preprocessing are as follows:
 

mindspore/dataset/vision/c_transforms.py

@@ -343,7 +343,7 @@ class CenterCrop(ImageTensorOperation):
     Raises:
         TypeError: If `size` is not of type int or sequence.
         ValueError: If `size` is less than or equal to 0.
-        RuntimeError: If given tensor shape is not <H, W> or <H, W, C>.
+        RuntimeError: If given tensor shape is not <H, W> or <..., H, W, C>.
 
     Supported Platforms:
         ``CPU``
@@ -716,7 +716,7 @@ class HWC2CHW(ImageTensorOperation):
         RuntimeError: If given tensor shape is not <H, W> or <H, W, C>.
 
     Supported Platforms:
-        ``CPU`` ``Ascend`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> transforms_list = [c_vision.Decode(),
@@ -822,7 +822,7 @@ class Normalize(ImageTensorOperation):
         RuntimeError: If given tensor shape is not <H, W> or <...,H, W, C>.
 
     Supported Platforms:
-        ``CPU`` ``Ascend`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> decode_op = c_vision.Decode()
@@ -1214,7 +1214,7 @@ class RandomColorAdjust(ImageTensorOperation):
         RuntimeError: If given tensor shape is not <H, W, C>.
 
     Supported Platforms:
-        ``CPU`` ``Ascend`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> decode_op = c_vision.Decode()
@@ -1307,7 +1307,7 @@ class RandomCrop(ImageTensorOperation):
         ValueError: If `size` is not positive.
         ValueError: If `padding` is negative.
         ValueError: If `fill_value` is not in range [0, 255].
-        RuntimeError: If given tensor shape is not <H, W> or <H, W, C>.
+        RuntimeError: If given tensor shape is not <H, W> or <..., H, W, C>.
 
     Supported Platforms:
         ``CPU``
@@ -1561,7 +1561,7 @@ class RandomHorizontalFlip(ImageTensorOperation):
         RuntimeError: If given tensor shape is not <H, W> or <H, W, C>.
 
     Supported Platforms:
-        ``CPU`` ``Ascend`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> transforms_list = [c_vision.Decode(), c_vision.RandomHorizontalFlip(0.75)]
@@ -2070,7 +2070,7 @@ class RandomSharpness(ImageTensorOperation):
         ValueError: If `degrees` is in (max, min) format instead of (min, max).
 
     Supported Platforms:
-        ``CPU`` ``Ascend`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> transforms_list = [c_vision.Decode(), c_vision.RandomSharpness(degrees=(0.2, 1.9))]
@@ -2136,7 +2136,7 @@ class RandomVerticalFlip(ImageTensorOperation):
         RuntimeError: If given tensor shape is not <H, W> or <H, W, C>.
 
     Supported Platforms:
-        ``CPU`` ``Ascend`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> transforms_list = [c_vision.Decode(), c_vision.RandomVerticalFlip(0.25)]
@@ -2200,7 +2200,7 @@ class Rescale(ImageTensorOperation):
         TypeError: If `shift` is not of type float.
 
     Supported Platforms:
-        ``CPU`` ``Ascend`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> transforms_list = [c_vision.Decode(), c_vision.Rescale(1.0 / 255.0, -1.0)]

mindspore/dataset/vision/py_transforms.py

@@ -1933,10 +1933,10 @@ class ToPIL(py_transforms.PyTensorOperation):
     Convert the input decoded numpy.ndarray image to PIL Image.
 
     Note:
-        The conversion mode will be determined by the data type using :class:`PIL.Image.fromarray` .
+        The conversion mode will be determined by the data type using `PIL.Image.fromarray` .
 
     Raises:
-        TypeError: If the input image is not of type :class:`numpy.ndarray` or :class:`PIL.Image.Image` .
+        TypeError: If the input image is not of type :class:`numpy.ndarray` or `PIL.Image.Image` .
 
     Supported Platforms:
         ``CPU``
@@ -1977,10 +1977,10 @@ class ToTensor(py_transforms.PyTensorOperation):
     from (H, W, C) to (C, H, W).
 
     Args:
-        output_type (numpy.dtype, optional): The desired dtype of the output image. Default: :class:`numpy.float32` .
+        output_type (numpy.dtype, optional): The desired dtype of the output image. Default: `numpy.float32` .
 
     Raises:
-        TypeError: If the input image is not of type :class:`PIL.Image.Image` or :class:`numpy.ndarray` .
+        TypeError: If the input image is not of type `PIL.Image.Image` or :class:`numpy.ndarray` .
         TypeError: If dimension of the input image is not 2 or 3.
 
     Supported Platforms:
@@ -2021,7 +2021,7 @@ class ToType(py_transforms.PyTensorOperation):
     Convert the input numpy.ndarray image to the desired dtype.
 
     Args:
-        output_type (numpy.dtype): The desired dtype of the output image, e.g. :class:`numpy.float32` .
+        output_type (numpy.dtype): The desired dtype of the output image, e.g. `numpy.float32` .
 
     Raises:
         TypeError: If the input image is not of type :class:`numpy.ndarray` .

mindspore/dataset/vision/py_transforms_util.py

@@ -964,6 +964,8 @@ def pad(img, padding, fill_value, padding_mode):
             image = ImageOps.expand(img, border=(left, top, right, bottom), fill=fill_value)
             image.putpalette(palette)
             return image
+        if isinstance(fill_value, tuple) and (img.mode == 'L' or img.mode == '1'):
+            fill_value = (fill_value[0],)
         return ImageOps.expand(img, border=(left, top, right, bottom), fill=fill_value)
 
     if img.mode == 'P':