mindspore 1.10.0__cp37-none-any.whl → 2.0.0rc1__cp37-none-any.whl

mindspore/include/dataset/text.h

@@ -234,6 +234,36 @@ class SentencePieceVocab {
 
 // Transform operations for text
 namespace text {
+/// \brief Add token to beginning or end of sequence.
+class DATASET_API AddToken final : public TensorTransform {
+ public:
+  /// \brief Constructor.
+  /// \param[in] token The token to be added.
+  /// \param[in] begin Whether to insert token at start or end of sequence. Default: true.
+  /// \par Example
+  /// \code
+  ///     /* Define operations */
+  ///     auto add_token_op = text::AddToken(token='TOKEN', begin=True);
+  ///
+  ///     /* dataset is an instance of Dataset object */
+  ///     dataset = dataset->Map({add_token_op},   // operations
+  ///                            {"text"});       // input columns
+  /// \endcode
+  explicit AddToken(const std::string &token, bool begin = true);
+
+  /// \brief Destructor.
+  ~AddToken() override = default;
+
+ protected:
+  /// \brief Function to convert TensorTransform object into a TensorOperation object.
+  /// \return Shared pointer to TensorOperation object.
+  std::shared_ptr<TensorOperation> Parse() override;
+
+ private:
+  struct Data;
+  std::shared_ptr<Data> data_;
+};
+
 #ifndef _WIN32
 /// \brief Tokenize a scalar tensor of UTF-8 string by specific rules.
 /// \note BasicTokenizer is not supported on the Windows platform yet.
@@ -864,6 +894,35 @@ class DATASET_API ToVectors final : public TensorTransform {
   std::shared_ptr<Data> data_;
 };
 
+/// \brief Truncate the input sequence so that it does not exceed the maximum length.
+class DATASET_API Truncate final : public TensorTransform {
+ public:
+  /// \brief Constructor.
+  /// \param[in] max_seq_len Maximum allowable length.
+  /// \par Example
+  /// \code
+  ///     /* Define operations */
+  ///     auto truncate_op = text::Truncate(5);
+  ///
+  ///     /* dataset is an instance of Dataset object */
+  ///     dataset = dataset->Map({truncate_op},   // operations
+  ///                            {"text"});       // input columns
+  /// \endcode
+  explicit Truncate(int32_t max_seq_len);
+
+  /// \brief Destructor.
+  ~Truncate() = default;
+
+ protected:
+  /// \brief The function to convert a TensorTransform object into a TensorOperation object.
+  /// \return Shared pointer to the TensorOperation object.
+  std::shared_ptr<TensorOperation> Parse() override;
+
+ private:
+  struct Data;
+  std::shared_ptr<Data> data_;
+};
+
 /// \brief Truncate a pair of rank-1 tensors such that the total length is less than max_length.
 class DATASET_API TruncateSequencePair final : public TensorTransform {
  public:

mindspore/include/dataset/vision.h

@@ -440,6 +440,20 @@ class DATASET_API CutOut final : public TensorTransform {
   std::shared_ptr<Data> data_;
 };
 
+/// \brief Encode the image as JPEG data.
+/// \param[in] image The image to be encoded.
+/// \param[out] output The Tensor data.
+/// \param[in] quality The quality for the output tensor, in range of [1, 100]. Default: 75.
+/// \return The status code.
+Status DATASET_API EncodeJpeg(const mindspore::MSTensor &image, mindspore::MSTensor *output, int quality = 75);
+
+/// \brief Encode the image as PNG data.
+/// \param[in] image The image to be encoded.
+/// \param[out] output The Tensor data.
+/// \param[in] compression_level The compression_level for encoding, in range of [0, 9]. Default: 6.
+/// \return The status code.
+Status DATASET_API EncodePng(const mindspore::MSTensor &image, mindspore::MSTensor *output, int compression_level = 6);
+
 /// \brief Apply histogram equalization on the input image.
 class DATASET_API Equalize final : public TensorTransform {
  public:
@@ -624,56 +638,6 @@ class DATASET_API NormalizePad final : public TensorTransform {
   std::shared_ptr<Data> data_;
 };
 
-/// \brief Pad the image according to padding parameters.
-class DATASET_API Pad final : public TensorTransform {
- public:
-  /// \brief Constructor.
-  /// \param[in] padding A vector representing the number of pixels to pad the image.
-  ///    If the vector has one value, it pads all sides of the image with that value.
-  ///    If the vector has two values, it pads left and top with the first and
-  ///    right and bottom with the second value.
-  ///    If the vector has four values, it pads left, top, right, and bottom with
-  ///    those values respectively.
-  /// \param[in] fill_value A vector representing the pixel intensity of the borders. Only valid if the
-  ///    padding_mode is BorderType.kConstant. If 1 value is provided, it is used for all RGB channels.
-  ///    If 3 values are provided, it is used to fill R, G, B channels respectively.
-  /// \param[in] padding_mode The method of padding (default=BorderType.kConstant).
-  ///    Can be any of
-  ///    [BorderType.kConstant, BorderType.kEdge, BorderType.kReflect, BorderType.kSymmetric]
-  ///    - BorderType.kConstant, means it fills the border with constant values
-  ///    - BorderType.kEdge, means it pads with the last value on the edge
-  ///    - BorderType.kReflect, means it reflects the values on the edge omitting the last value of edge
-  ///    - BorderType.kSymmetric, means it reflects the values on the edge repeating the last value of edge
-  /// \note The behavior when `padding` is a sequence of length 2 will change from padding left/top with
-  ///     the first value and right/bottom with the second, to padding left/right with the first one
-  ///     and top/bottom with the second in the future. Or you can pass in a 4-element sequence to specify
-  ///     left, top, right and bottom respectively.
-  /// \par Example
-  /// \code
-  ///     /* Define operations */
-  ///     auto decode_op = vision::Decode();
-  ///     auto pad_op = vision::Pad({10, 10, 10, 10}, {255, 255, 255});
-  ///
-  ///     /* dataset is an instance of Dataset object */
-  ///     dataset = dataset->Map({decode_op, pad_op},  // operations
-  ///                            {"image"});           // input columns
-  /// \endcode
-  explicit Pad(const std::vector<int32_t> &padding, const std::vector<uint8_t> &fill_value = {0},
-               BorderType padding_mode = BorderType::kConstant);
-
-  /// \brief Destructor.
-  ~Pad() = default;
-
- protected:
-  /// \brief The function to convert a TensorTransform object into a TensorOperation object.
-  /// \return Shared pointer to TensorOperation object.
-  std::shared_ptr<TensorOperation> Parse() override;
-
- private:
-  struct Data;
-  std::shared_ptr<Data> data_;
-};
-
 /// \brief Pad the image to a fixed size.
 class DATASET_API PadToSize final : public TensorTransform {
  public:
@@ -977,8 +941,8 @@ class DATASET_API RandomCrop final : public TensorTransform {
   ///     If the size has 2 values, it should be (height, width).
   /// \param[in] padding A vector representing the number of pixels to pad the image.
   ///    If the vector has one value, it pads all sides of the image with that value.
-  ///    If the vector has two values, it pads left and top with the first and
-  ///    right and bottom with the second value.
+  ///    If the vector has two values, it pads left and right with the first and
+  ///    top and bottom with the second value.
   ///    If the vector has four values, it pads left, top, right, and bottom with
   ///    those values respectively.
   /// \param[in] pad_if_needed A boolean indicating that whether to pad the image
@@ -993,10 +957,6 @@ class DATASET_API RandomCrop final : public TensorTransform {
   ///   - BorderType::kReflect, Reflect the values on the edge omitting the last value of edge.
   ///   - BorderType::kSymmetric, Reflect the values on the edge repeating the last value of edge.
   /// \note If the input image is more than one, then make sure that the image size is the same.
-  /// \note The behavior when `padding` is a sequence of length 2 will change from padding left/top with
-  ///     the first value and right/bottom with the second, to padding left/right with the first one
-  ///     and top/bottom with the second in the future. Or you can pass in a 4-element sequence to specify
-  ///     left, top, right and bottom respectively.
   /// \par Example
   /// \code
   ///     /* Define operations */
@@ -1080,8 +1040,8 @@ class DATASET_API RandomCropWithBBox final : public TensorTransform {
   ///     If the size has 2 values, it should be (height, width).
   /// \param[in] padding A vector representing the number of pixels to pad the image
   ///    If the vector has one value, it pads all sides of the image with that value.
-  ///    If the vector has two values, it pads left and top with the first and
-  ///    right and bottom with the second value.
+  ///    If the vector has two values, it pads left and right with the first and
+  ///    top and bottom with the second value.
   ///    If the vector has four values, it pads left, top, right, and bottom with
   ///    those values respectively.
   /// \param[in] pad_if_needed A boolean indicating that whether to pad the image
@@ -1095,10 +1055,6 @@ class DATASET_API RandomCropWithBBox final : public TensorTransform {
   ///   - BorderType::kEdge, Fill the border with the last value on the edge.
   ///   - BorderType::kReflect, Reflect the values on the edge omitting the last value of edge.
   ///   - BorderType::kSymmetric, Reflect the values on the edge repeating the last value of edge.
-  /// \note The behavior when `padding` is a sequence of length 2 will change from padding left/top with
-  ///     the first value and right/bottom with the second, to padding left/right with the first one
-  ///     and top/bottom with the second in the future. Or you can pass in a 4-element sequence to specify
-  ///     left, top, right and bottom respectively.
   /// \par Example
   /// \code
   ///     /* Define operations */
@@ -1711,36 +1667,24 @@ class DATASET_API RandomVerticalFlipWithBBox final : public TensorTransform {
   std::shared_ptr<Data> data_;
 };
 
-/// \brief Rescale the pixel value of input image.
-class DATASET_API Rescale final : public TensorTransform {
- public:
-  /// \brief Constructor.
-  /// \param[in] rescale Rescale factor.
-  /// \param[in] shift Shift factor.
-  /// \par Example
-  /// \code
-  ///     /* Define operations */
-  ///     auto decode_op = vision::Decode();
-  ///     auto rescale_op = vision::Rescale(1.0, 0.0);
-  ///
-  ///     /* dataset is an instance of Dataset object */
-  ///     dataset = dataset->Map({decode_op, rescale_op},  // operations
-  ///                            {"image"});               // input columns
-  /// \endcode
-  Rescale(float rescale, float shift);
-
-  /// \brief Destructor.
-  ~Rescale() = default;
-
- protected:
-  /// \brief The function to convert a TensorTransform object into a TensorOperation object.
-  /// \return Shared pointer to TensorOperation object.
-  std::shared_ptr<TensorOperation> Parse() override;
-
- private:
-  struct Data;
-  std::shared_ptr<Data> data_;
-};
+/// \brief Reads a file in binary mode.
+/// \param[in] filename The path to the file to be read.
+/// \param[out] output The binary data.
+/// \return The status code.
+Status DATASET_API ReadFile(const std::string &filename, mindspore::MSTensor *output);
+
+/// \brief Read a image file and decode it into one or three channels data.
+/// \param[in] filename The path to the file to be read.
+/// \param[out] output The Tensor data.
+/// \param[in] mode The read mode used for optionally converting the image, can be one of
+///    [ImageReadMode::kUNCHANGED, ImageReadMode::kGRAYSCALE, ImageReadMode::kCOLOR]. Default:
+///    ImageReadMode::kUNCHANGED.
+///    - ImageReadMode::kUNCHANGED, remain the output in the original format.
+///    - ImageReadMode::kGRAYSCALE, convert the output into one channel grayscale data.
+///    - ImageReadMode::kCOLOR, convert the output into three channels RGB color data.
+/// \return The status code.
+Status DATASET_API ReadImage(const std::string &filename, mindspore::MSTensor *output,
+                             ImageReadMode mode = ImageReadMode::kUNCHANGED);
 
 /// \brief Crop the given image and zoom to the specified size.
 class DATASET_API ResizedCrop final : public TensorTransform {
@@ -1943,31 +1887,6 @@ class DATASET_API Solarize final : public TensorTransform {
   std::shared_ptr<Data> data_;
 };
 
-/// \brief Swap the red and blue channels of the input image.
-class DATASET_API SwapRedBlue final : public TensorTransform {
- public:
-  /// \brief Constructor.
-  /// \par Example
-  /// \code
-  ///     /* Define operations */
-  ///     auto decode_op = vision::Decode();
-  ///     auto swap_red_blue_op = vision::SwapRedBlue();
-  ///
-  ///     /* dataset is an instance of Dataset object */
-  ///     dataset = dataset->Map({decode_op, swap_red_blue_op},  // operations
-  ///                            {"image"});                     // input columns
-  /// \endcode
-  SwapRedBlue();
-
-  /// \brief Destructor.
-  ~SwapRedBlue() = default;
-
- protected:
-  /// \brief The function to convert a TensorTransform object into a TensorOperation object.
-  /// \return Shared pointer to TensorOperation object.
-  std::shared_ptr<TensorOperation> Parse() override;
-};
-
 /// \brief Divide the pixel values by 255 and convert from HWC format to CHW format with required datatype.
 class DATASET_API ToTensor final : public TensorTransform {
  public:
@@ -2128,6 +2047,26 @@ class DATASET_API VerticalFlip final : public TensorTransform {
   /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 };
+
+/// \brief Write the one dimension uint8 data into a file using binary mode.
+/// \param[in] filename The path to the file to be written.
+/// \param[in] data The tensor data.
+/// \return The status code.
+Status DATASET_API WriteFile(const std::string &filename, const mindspore::MSTensor &data);
+
+/// \brief Write the image data into a JPEG file.
+/// \param[in] filename The path to the file to be written.
+/// \param[in] image The data tensor.
+/// \param[in] quality The quality for JPEG file, in range of [1, 100]. Default: 75.
+/// \return The status code.
+Status DATASET_API WriteJpeg(const std::string &filename, const mindspore::MSTensor &image, int quality = 75);
+
+/// \brief Write the image into a PNG file.
+/// \param[in] filename The path to the file to be written.
+/// \param[in] image The data tensor.
+/// \param[in] compression_level The compression level for PNG file, in range of [0, 9]. Default: 6.
+/// \return The status code.
+Status DATASET_API WritePng(const std::string &filename, const mindspore::MSTensor &image, int compression_level = 6);
 }  // namespace vision
 }  // namespace dataset
 }  // namespace mindspore

mindspore/include/dataset/vision_lite.h

@@ -268,6 +268,52 @@ class DATASET_API Normalize final : public TensorTransform {
   std::shared_ptr<Data> data_;
 };
 
+/// \brief Pad the image according to padding parameters.
+class DATASET_API Pad final : public TensorTransform {
+ public:
+  /// \brief Constructor.
+  /// \param[in] padding A vector representing the number of pixels to pad the image.
+  ///    If the vector has one value, it pads all sides of the image with that value.
+  ///    If the vector has two values, it pads left and right with the first and
+  ///    top and bottom with the second value.
+  ///    If the vector has four values, it pads left, top, right, and bottom with
+  ///    those values respectively.
+  /// \param[in] fill_value A vector representing the pixel intensity of the borders. Only valid if the
+  ///    padding_mode is BorderType.kConstant. If 1 value is provided, it is used for all RGB channels.
+  ///    If 3 values are provided, it is used to fill R, G, B channels respectively.
+  /// \param[in] padding_mode The method of padding (default=BorderType.kConstant).
+  ///    Can be any of
+  ///    [BorderType.kConstant, BorderType.kEdge, BorderType.kReflect, BorderType.kSymmetric]
+  ///    - BorderType.kConstant, means it fills the border with constant values
+  ///    - BorderType.kEdge, means it pads with the last value on the edge
+  ///    - BorderType.kReflect, means it reflects the values on the edge omitting the last value of edge
+  ///    - BorderType.kSymmetric, means it reflects the values on the edge repeating the last value of edge
+  /// \par Example
+  /// \code
+  ///     /* Define operations */
+  ///     auto decode_op = vision::Decode();
+  ///     auto pad_op = vision::Pad({10, 10, 10, 10}, {255, 255, 255});
+  ///
+  ///     /* dataset is an instance of Dataset object */
+  ///     dataset = dataset->Map({decode_op, pad_op},  // operations
+  ///                            {"image"});           // input columns
+  /// \endcode
+  explicit Pad(const std::vector<int32_t> &padding, const std::vector<uint8_t> &fill_value = {0},
+               BorderType padding_mode = BorderType::kConstant);
+
+  /// \brief Destructor.
+  ~Pad() = default;
+
+ protected:
+  /// \brief The function to convert a TensorTransform object into a TensorOperation object.
+  /// \return Shared pointer to TensorOperation object.
+  std::shared_ptr<TensorOperation> Parse() override;
+
+ private:
+  struct Data;
+  std::shared_ptr<Data> data_;
+};
+
 /// \brief Apply a Random Affine transformation on the input image in RGB or Greyscale mode.
 class DATASET_API RandomAffine final : public TensorTransform {
  public:
@@ -319,6 +365,37 @@ class DATASET_API RandomAffine final : public TensorTransform {
   std::shared_ptr<Data> data_;
 };
 
+/// \brief Rescale the pixel value of input image.
+class DATASET_API Rescale final : public TensorTransform {
+ public:
+  /// \brief Constructor.
+  /// \param[in] rescale Rescale factor.
+  /// \param[in] shift Shift factor.
+  /// \par Example
+  /// \code
+  ///     /* Define operations */
+  ///     auto decode_op = vision::Decode();
+  ///     auto rescale_op = vision::Rescale(1.0, 0.0);
+  ///
+  ///     /* dataset is an instance of Dataset object */
+  ///     dataset = dataset->Map({decode_op, rescale_op},  // operations
+  ///                            {"image"});               // input columns
+  /// \endcode
+  Rescale(float rescale, float shift);
+
+  /// \brief Destructor.
+  ~Rescale() = default;
+
+ protected:
+  /// \brief The function to convert a TensorTransform object into a TensorOperation object.
+  /// \return Shared pointer to TensorOperation object.
+  std::shared_ptr<TensorOperation> Parse() override;
+
+ private:
+  struct Data;
+  std::shared_ptr<Data> data_;
+};
+
 /// \brief Resize the input image to the given size.
 class DATASET_API Resize final : public TensorTransform {
  public:
@@ -517,6 +594,31 @@ class DATASET_API Rotate final : public TensorTransform {
   struct Data;
   std::shared_ptr<Data> data_;
 };
+
+/// \brief Swap the red and blue channels of the input image.
+class DATASET_API SwapRedBlue final : public TensorTransform {
+ public:
+  /// \brief Constructor.
+  /// \par Example
+  /// \code
+  ///     /* Define operations */
+  ///     auto decode_op = vision::Decode();
+  ///     auto swap_red_blue_op = vision::SwapRedBlue();
+  ///
+  ///     /* dataset is an instance of Dataset object */
+  ///     dataset = dataset->Map({decode_op, swap_red_blue_op},  // operations
+  ///                            {"image"});                     // input columns
+  /// \endcode
+  SwapRedBlue();
+
+  /// \brief Destructor.
+  ~SwapRedBlue() = default;
+
+ protected:
+  /// \brief The function to convert a TensorTransform object into a TensorOperation object.
+  /// \return Shared pointer to TensorOperation object.
+  std::shared_ptr<TensorOperation> Parse() override;
+};
 }  // namespace vision
 }  // namespace dataset
 }  // namespace mindspore

mindspore/include/mindapi/base/type_id.h

@@ -77,6 +77,7 @@ enum TypeId : int {
   kNumberTypeFloat16,
   kNumberTypeFloat32,
   kNumberTypeFloat64,
+  kNumberTypeDouble,
   kNumberTypeComplex,
   kNumberTypeComplex64,
   kNumberTypeComplex128,
@@ -94,12 +95,50 @@ enum TypeId : int {
   //
   // Sparse Types
   //
-  // Sparse types is placed at the end of enum,
-  // in order to keep fit with the type of existing model on the lite side.
   kSparseTypeBegin = kMonadTypeEnd,
   kObjectTypeCSRTensorType,
   kObjectTypeSparseTensorType,
-  kSparseTypeEnd
+  kObjectTypeMapTensorType,
+  kSparseTypeEnd,
+  // New types should placed at the end of enum,
+  // in order to keep fit with the type of existing model on the lite side.
+};
+
+/// \brief QuantDataType defines data type for quantization: int1-int16, uint1-uint16.
+enum QuantDataType : int {
+  kQuantDataTypeInt1 = 0,
+  kQuantDataTypeInt2,
+  kQuantDataTypeInt3,
+  kQuantDataTypeInt4,
+  kQuantDataTypeInt5,
+  kQuantDataTypeInt6,
+  kQuantDataTypeInt7,
+  kQuantDataTypeInt8,
+  kQuantDataTypeInt9,
+  kQuantDataTypeInt10,
+  kQuantDataTypeInt11,
+  kQuantDataTypeInt12,
+  kQuantDataTypeInt13,
+  kQuantDataTypeInt14,
+  kQuantDataTypeInt15,
+  kQuantDataTypeInt16,
+
+  kQuantDataTypeUInt1 = 100,
+  kQuantDataTypeUInt2,
+  kQuantDataTypeUInt3,
+  kQuantDataTypeUInt4,
+  kQuantDataTypeUInt5,
+  kQuantDataTypeUInt6,
+  kQuantDataTypeUInt7,
+  kQuantDataTypeUInt8,
+  kQuantDataTypeUInt9,
+  kQuantDataTypeUInt10,
+  kQuantDataTypeUInt11,
+  kQuantDataTypeUInt12,
+  kQuantDataTypeUInt13,
+  kQuantDataTypeUInt14,
+  kQuantDataTypeUInt15,
+  kQuantDataTypeUInt16,
 };
 }  // namespace mindspore
 #endif  // MINDSPORE_CORE_MINDAPI_BASE_TYPE_ID_H_