mindspore 2.0.0a0__cp37-none-any.whl → 2.0.0rc1__cp37-none-any.whl

mindspore/include/api/net.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 Huawei Technologies Co., Ltd
+ * Copyright 2022-2023 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -36,14 +36,14 @@ class NodeSet;
 class Graph;
 class NetData;
 
-class NetBase {
+class MS_API NetBase {
  public:
   NetBase() = default;
   virtual std::vector<Expr *> operator()(const std::vector<Expr *> &inputs) = 0;
   virtual uint32_t type() = 0;
 };
 
-class Node : public NetBase {
+class MS_API Node : public NetBase {
  public:
   Node();
   virtual ~Node();
@@ -65,7 +65,7 @@ class Node : public NetBase {
   std::shared_ptr<NodeImpl> impl_ = nullptr;
 };
 
-class Net : public NetBase, public std::enable_shared_from_this<Net> {
+class MS_API Net : public NetBase, public std::enable_shared_from_this<Net> {
  public:
   Net();
   virtual ~Net();
@@ -116,12 +116,12 @@ class Net : public NetBase, public std::enable_shared_from_this<Net> {
   std::shared_ptr<NetImpl> impl_;
 };
 
-class SoftMaxCrossEntropyCfg {
+class MS_API SoftMaxCrossEntropyCfg {
  public:
   std::string reduction = "mean"; /**<  Specifies reduction mode. The optional values are "none", "mean", "sum" */
 };
 
-class AdamConfig {
+class MS_API AdamConfig {
  public:
   float learning_rate_ = 1e-3;
   float beta1_ = 0.9;
@@ -131,11 +131,12 @@ class AdamConfig {
 };
 
 namespace NN {
-Net *NetWithLoss(Net *net, Node *loss);
-Graph *GraphWithLoss(Graph *g, Node *loss);
-Node *Adam(std::shared_ptr<NodeSet> learn, const AdamConfig &cfg);
-Node *SoftmaxCrossEntropy(const SoftMaxCrossEntropyCfg &cfg);
-std::unique_ptr<Node> Input(std::vector<int> dims, DataType data_type = DataType::kNumberTypeFloat32, int fmt = NHWC);
+MS_API Net *NetWithLoss(Net *net, Node *loss);
+MS_API Graph *GraphWithLoss(Graph *g, Node *loss);
+MS_API Node *Adam(std::shared_ptr<NodeSet> learn, const AdamConfig &cfg);
+MS_API Node *SoftmaxCrossEntropy(const SoftMaxCrossEntropyCfg &cfg);
+MS_API std::unique_ptr<Node> Input(std::vector<int> dims, DataType data_type = DataType::kNumberTypeFloat32,
+                                   int fmt = NHWC);
 };  // namespace NN
 }  // namespace mindspore
 #endif  // MINDSPORE_INCLUDE_API_NET_H

mindspore/include/api/serialization.h

@@ -79,16 +79,22 @@ class MS_API Serialization {
   ///
   /// \param[in] model The model data.
   /// \param[in] model_type The model file type.
-  /// \param[out] model_data The model parameter data.
+  /// \param[out] model_data The model buffer.
+  /// \param[in] quantization_type The quantification type.
+  /// \param[in] export_inference_only Whether to export a reasoning only model.
+  /// \param[in] output_tensor_name The set the name of the output tensor of the exported reasoning model, default as
+  /// empty, and export the complete reasoning model.
   ///
   /// \return Status.
-  static Status ExportModel(const Model &model, ModelType model_type, Buffer *model_data);
+  inline static Status ExportModel(const Model &model, ModelType model_type, Buffer *model_data,
+                                   QuantizationType quantization_type = kNoQuant, bool export_inference_only = true,
+                                   const std::vector<std::string> &output_tensor_name = {});
 
   /// \brief Export training model from file.
   ///
   /// \param[in] model The model data.
   /// \param[in] model_type The model file type.
-  /// \param[in] model_file The exported model file.
+  /// \param[in] model_file The path of exported model file.
   /// \param[in] quantization_type The quantification type.
   /// \param[in] export_inference_only Whether to export a reasoning only model.
   /// \param[in] output_tensor_name The set the name of the output tensor of the exported reasoning model, default as
@@ -111,6 +117,9 @@ class MS_API Serialization {
   static Status ExportModel(const Model &model, ModelType model_type, const std::vector<char> &model_file,
                             QuantizationType quantization_type, bool export_inference_only,
                             const std::vector<std::vector<char>> &output_tensor_name);
+  static Status ExportModel(const Model &model, ModelType model_type, Buffer *model_data,
+                            QuantizationType quantization_type, bool export_inference_only,
+                            const std::vector<std::vector<char>> &output_tensor_name);
 };
 
 Status Serialization::Load(const void *model_data, size_t data_size, ModelType model_type, Graph *graph,
@@ -139,5 +148,12 @@ Status Serialization::ExportModel(const Model &model, ModelType model_type, cons
                      VectorStringToChar(output_tensor_name));
 }
 
+Status Serialization::ExportModel(const Model &model, ModelType model_type, Buffer *model_data,
+                                  QuantizationType quantization_type, bool export_inference_only,
+                                  const std::vector<std::string> &output_tensor_name) {
+  return ExportModel(model, model_type, model_data, quantization_type, export_inference_only,
+                     VectorStringToChar(output_tensor_name));
+}
+
 }  // namespace mindspore
 #endif  // MINDSPORE_INCLUDE_API_SERIALIZATION_H

mindspore/include/api/types.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020-2022 Huawei Technologies Co., Ltd
+ * Copyright 2020-2023 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -368,7 +368,7 @@ std::string MSTensor::Name() const { return CharToString(CharName()); }
 
 void MSTensor::SetTensorName(const std::string &name) { SetTensorName(StringToChar(name)); }
 
-using Key = struct Key {
+using Key = struct MS_API Key {
   const size_t max_key_len = 32;
   size_t len = 0;
   unsigned char key[32] = {0};
@@ -390,7 +390,7 @@ using MSKernelCallBack =
   std::function<bool(const std::vector<MSTensor> & /* inputs */, const std::vector<MSTensor> & /* outputs */,
                      const MSCallBackParam &opInfo)>;
 
-std::vector<char> CharVersion();
+MS_API std::vector<char> CharVersion();
 inline std::string Version() { return CharToString(CharVersion()); }
 
 }  // namespace mindspore

mindspore/include/dataset/constants.h

@@ -291,6 +291,13 @@ enum class DATASET_API ResampleMethod {
   kKaiserWindow = 1,       ///< Resample audio by Kaiser window
 };
 
+/// \brief Possible configuration methods for processing error samples.
+enum class DATASET_API ErrorSamplesMode {
+  kReturn = 0,   ///< Erroneous sample results in error raised and returned
+  kReplace = 1,  ///< Erroneous sample is replaced with an internally determined sample
+  kSkip = 2      ///< Erroneous sample is skipped
+};
+
 /// \brief Convenience function to check bitmask for a 32bit int
 /// \param[in] bits a 32bit int to be tested
 /// \param[in] bitMask a 32bit int representing bit mask

mindspore/include/dataset/text.h

@@ -234,6 +234,36 @@ class SentencePieceVocab {
 
 // Transform operations for text
 namespace text {
+/// \brief Add token to beginning or end of sequence.
+class DATASET_API AddToken final : public TensorTransform {
+ public:
+  /// \brief Constructor.
+  /// \param[in] token The token to be added.
+  /// \param[in] begin Whether to insert token at start or end of sequence. Default: true.
+  /// \par Example
+  /// \code
+  ///     /* Define operations */
+  ///     auto add_token_op = text::AddToken(token='TOKEN', begin=True);
+  ///
+  ///     /* dataset is an instance of Dataset object */
+  ///     dataset = dataset->Map({add_token_op},   // operations
+  ///                            {"text"});       // input columns
+  /// \endcode
+  explicit AddToken(const std::string &token, bool begin = true);
+
+  /// \brief Destructor.
+  ~AddToken() override = default;
+
+ protected:
+  /// \brief Function to convert TensorTransform object into a TensorOperation object.
+  /// \return Shared pointer to TensorOperation object.
+  std::shared_ptr<TensorOperation> Parse() override;
+
+ private:
+  struct Data;
+  std::shared_ptr<Data> data_;
+};
+
 #ifndef _WIN32
 /// \brief Tokenize a scalar tensor of UTF-8 string by specific rules.
 /// \note BasicTokenizer is not supported on the Windows platform yet.
@@ -864,6 +894,35 @@ class DATASET_API ToVectors final : public TensorTransform {
   std::shared_ptr<Data> data_;
 };
 
+/// \brief Truncate the input sequence so that it does not exceed the maximum length.
+class DATASET_API Truncate final : public TensorTransform {
+ public:
+  /// \brief Constructor.
+  /// \param[in] max_seq_len Maximum allowable length.
+  /// \par Example
+  /// \code
+  ///     /* Define operations */
+  ///     auto truncate_op = text::Truncate(5);
+  ///
+  ///     /* dataset is an instance of Dataset object */
+  ///     dataset = dataset->Map({truncate_op},   // operations
+  ///                            {"text"});       // input columns
+  /// \endcode
+  explicit Truncate(int32_t max_seq_len);
+
+  /// \brief Destructor.
+  ~Truncate() = default;
+
+ protected:
+  /// \brief The function to convert a TensorTransform object into a TensorOperation object.
+  /// \return Shared pointer to the TensorOperation object.
+  std::shared_ptr<TensorOperation> Parse() override;
+
+ private:
+  struct Data;
+  std::shared_ptr<Data> data_;
+};
+
 /// \brief Truncate a pair of rank-1 tensors such that the total length is less than max_length.
 class DATASET_API TruncateSequencePair final : public TensorTransform {
  public:

mindspore/include/mindapi/base/type_id.h

@@ -77,6 +77,7 @@ enum TypeId : int {
   kNumberTypeFloat16,
   kNumberTypeFloat32,
   kNumberTypeFloat64,
+  kNumberTypeDouble,
   kNumberTypeComplex,
   kNumberTypeComplex64,
   kNumberTypeComplex128,

mindspore/log.py

@@ -279,7 +279,7 @@ def _check_directory_by_regular(target, reg=None, flag=re.ASCII, prim_name=None)
     if not isinstance(target, str):
         raise ValueError("The directory {} must be string, but got {}, please check it".format(target, type(target)))
     if reg is None:
-        reg = r"^[\/0-9a-zA-Z\_\-\.\:\\]+$"
+        reg = r"^[\/0-9a-zA-Z@\_\-\.\:\\]+$"
     if re.match(reg, target, flag) is None:
         prim_name = f'in `{prim_name}`' if prim_name else ""
         raise ValueError("'{}' {} is illegal, it should be match regular'{}' by flag'{}'".format(

mindspore/mindrecord/filereader.py

@@ -100,3 +100,21 @@ class FileReader:
     def close(self):
         """Stop reader worker and close file."""
         self._reader.close()
+
+    def schema(self):
+        """
+        Get the schema of the MindRecord.
+
+        Returns:
+            dict, the schema info.
+        """
+        return self._header.schema
+
+    def len(self):
+        """
+        Get the number of the samples in MindRecord.
+
+        Returns:
+            int, the number of the samples in MindRecord.
+        """
+        return self._reader.len()

mindspore/mindrecord/filewriter.py

@@ -17,8 +17,11 @@ This module is to write data into mindrecord.
 """
 import os
 import platform
+import queue
 import re
 import stat
+import time
+import multiprocessing as mp
 import numpy as np
 from mindspore import log as logger
 from .shardwriter import ShardWriter
@@ -26,7 +29,7 @@ from .shardreader import ShardReader
 from .shardheader import ShardHeader
 from .shardindexgenerator import ShardIndexGenerator
 from .shardutils import MIN_SHARD_COUNT, MAX_SHARD_COUNT, VALID_ATTRIBUTES, VALID_ARRAY_ATTRIBUTES, \
-    check_filename, VALUE_TYPE_MAP
+    check_filename, VALUE_TYPE_MAP, SUCCESS
 from .common.exceptions import ParamValueError, ParamTypeError, MRMInvalidSchemaError, MRMDefineIndexError
 
 __all__ = ['FileWriter']
@@ -103,6 +106,13 @@ class FileWriter:
         self._writer = ShardWriter()
         self._generator = None
 
+        # parallel write mode
+        self._parallel_writer = None
+        self._writers = None
+        self._queue = None
+        self._workers = None
+        self._index_workers = None
+
     @classmethod
     def open_for_append(cls, file_name):
         r"""
@@ -258,23 +268,72 @@ class FileWriter:
             MRMValidateDataError: If data does not match blob fields.
             MRMSetHeaderError: If failed to set header.
             MRMWriteDatasetError: If failed to write dataset.
+            TypeError: If parallel_writer is not bool.
         """
-        if not self._writer.is_open:
-            self._writer.open(self._paths, self._overwrite)
-        if not self._writer.get_shard_header():
-            self._writer.set_shard_header(self._header)
-        if not isinstance(raw_data, list):
-            raise ParamTypeError('raw_data', 'list')
-        if self._flush and not self._append:
-            raise RuntimeError("Not allowed to call `write_raw_data` on flushed MindRecord files." \
-                               "When creating new Mindrecord files, please remove `commit` before `write_raw_data`." \
-                               "In other cases, when appending to existing MindRecord files, " \
-                               "please call `open_for_append` first and then `write_raw_data`.")
-        for each_raw in raw_data:
-            if not isinstance(each_raw, dict):
-                raise ParamTypeError('raw_data item', 'dict')
-        self._verify_based_on_schema(raw_data)
-        return self._writer.write_raw_data(raw_data, True, parallel_writer)
+        if not isinstance(parallel_writer, bool):
+            raise TypeError("The parameter `parallel_writer` must be bool.")
+
+        if self._parallel_writer is None:
+            self._parallel_writer = parallel_writer
+        if self._parallel_writer != parallel_writer:
+            raise RuntimeError("The parameter `parallel_writer` must be consistent during use.")
+        if not self._parallel_writer:
+            if not self._writer.is_open:
+                self._writer.open(self._paths, self._overwrite)
+            if not self._writer.get_shard_header():
+                self._writer.set_shard_header(self._header)
+            if not isinstance(raw_data, list):
+                raise ParamTypeError('raw_data', 'list')
+            if self._flush and not self._append:
+                raise RuntimeError("Not allowed to call `write_raw_data` on flushed MindRecord files." \
+                                   "When creating new Mindrecord files, please remove `commit` before " \
+                                   "`write_raw_data`. In other cases, when appending to existing MindRecord files, " \
+                                   "please call `open_for_append` first and then `write_raw_data`.")
+            for each_raw in raw_data:
+                if not isinstance(each_raw, dict):
+                    raise ParamTypeError('raw_data item', 'dict')
+            self._verify_based_on_schema(raw_data)
+            return self._writer.write_raw_data(raw_data, True, parallel_writer)
+
+        ## parallel write mode
+        # init the _writers and launch the workers
+        if self._writers is None:
+            self._writers = [None] * len(self._paths)  # writers used by worker
+            self._queue = mp.Queue(len(self._paths) * 2)  # queue for worker
+            self._workers = [None] * len(self._paths)  # worker process
+            for i, path in enumerate(self._paths):
+                self._writers[i] = ShardWriter()
+                self._writers[i].open([path], self._overwrite)
+                self._writers[i].set_shard_header(self._header)
+
+                # launch the workers for parallel write
+                self._queue._joincancelled = True  # pylint: disable=W0212
+                p = mp.Process(target=self._write_worker, args=(i, self._queue))
+                p.daemon = True
+                p.start()
+                logger.info("Start worker process(pid:{}) to parallel write.".format(p.pid))
+                self._workers[i] = p
+
+        # fill the self._queue
+        check_interval = 0.5  # 0.5s
+        start_time = time.time()
+        while True:
+            try:
+                self._queue.put(raw_data, block=False)
+            except queue.Full:
+                if time.time() - start_time > check_interval:
+                    start_time = time.time()
+                    logger.warning("Because there are too few MindRecord file shards, the efficiency of parallel " \
+                                   "writing is too low. You can stop the current task and add the parameter " \
+                                   "`shard_num` of `FileWriter` to upgrade the task.")
+
+                # check the status of worker process
+                for i in range(len(self._paths)):
+                    if not self._workers[i].is_alive():
+                        raise RuntimeError("Worker process(pid:{}) has stopped abnormal. Please check " \
+                                           "the above log".format(self._workers[i].pid))
+                continue
+            return SUCCESS
 
     def set_header_size(self, header_size):
         """
@@ -326,7 +385,7 @@ class FileWriter:
         """
         return self._writer.set_page_size(page_size)
 
-    def commit(self):
+    def commit(self):  # pylint: disable=W0212
         """
         Flush data in memory to disk and generate the corresponding database files.
 
@@ -342,25 +401,37 @@ class FileWriter:
             MRMIndexGeneratorError: If failed to create index generator.
             MRMGenerateIndexError: If failed to write to database.
             MRMCommitError: If failed to flush data to disk.
+            RuntimeError: Parallel write failed.
         """
-        self._flush = True
-        if not self._writer.is_open:
-            self._writer.open(self._paths, self._overwrite)
-        # permit commit without data
-        if not self._writer.get_shard_header():
-            self._writer.set_shard_header(self._header)
-        ret = self._writer.commit()
-        if self._index_generator:
-            if self._append:
-                self._generator = ShardIndexGenerator(self._file_name, self._append)
-            elif len(self._paths) >= 1:
-                self._generator = ShardIndexGenerator(os.path.realpath(self._paths[0]), self._append)
-            self._generator.build()
-            self._generator.write_to_db()
+        if not self._parallel_writer:
+            self._flush = True
+            if not self._writer.is_open:
+                self._writer.open(self._paths, self._overwrite)
+            # permit commit without data
+            if not self._writer.get_shard_header():
+                self._writer.set_shard_header(self._header)
+            self._writer.commit()
+            if self._index_generator:
+                if self._append:
+                    self._generator = ShardIndexGenerator(self._file_name, self._append)
+                elif len(self._paths) >= 1:
+                    self._generator = ShardIndexGenerator(os.path.realpath(self._paths[0]), self._append)
+                self._generator.build()
+                self._generator.write_to_db()
+        else:
+            # maybe a empty mindrecord, so need check _writers
+            if self._writers is None:
+                self._writers = [None] * len(self._paths)
+                for i, path in enumerate(self._paths):
+                    self._writers[i] = ShardWriter()
+                    self._writers[i].open(path, self._overwrite)
+                    self._writers[i].set_shard_header(self._header)
+
+            self._parallel_commit()
 
+        # change the file mode to 600
         mindrecord_files = []
         index_files = []
-        # change the file mode to 600
         for item in self._paths:
             if os.path.exists(item):
                 os.chmod(item, stat.S_IRUSR | stat.S_IWUSR)
@@ -373,7 +444,73 @@ class FileWriter:
         logger.info("The list of mindrecord files created are: {}, and the list of index files are: {}".format(
             mindrecord_files, index_files))
 
-        return ret
+        return SUCCESS
+
+    def _index_worker(self, i):
+        """The worker do the index generator"""
+        generator = ShardIndexGenerator(os.path.realpath(self._paths[i]), False)
+        generator.build()
+        generator.write_to_db()
+
+    def _parallel_commit(self):
+        """Parallel commit"""
+        # if some workers stopped, error may occur
+        alive_count = 0
+        for i in range(len(self._paths)):
+            if self._workers[i].is_alive():
+                alive_count += 1
+        if alive_count != len(self._paths):
+            raise RuntimeError("Parallel write worker error, please check the log file.")
+
+        # send EOF to worker process
+        for _ in range(len(self._paths)):
+            while True:
+                try:
+                    self._queue.put("EOF", block=False)
+                except queue.Full:
+                    time.sleep(1)
+                    continue
+                break
+
+        # wait the worker processing
+        while True:
+            alive_count = 0
+            for i in range(len(self._paths)):
+                if self._workers[i].is_alive():
+                    alive_count += 1
+            if alive_count == 0:
+                break
+            time.sleep(1)
+            logger.info("Waiting for all the parallel workers to finish.")
+
+        del self._queue
+
+        # wait for worker process stop
+        for index in range(len(self._paths)):
+            while True:
+                logger.info("Waiting for the worker process(pid:{}) to process all the data.".format(
+                    self._workers[index].pid))
+                if self._workers[index].is_alive():
+                    time.sleep(1)
+                    continue
+                elif self._workers[index].exitcode != 0:
+                    raise RuntimeError("Worker process(pid:{}) has stopped abnormal. Please check " \
+                                       "the above log".format(self._workers[index].pid))
+                break
+
+        if self._index_generator:
+            # use parallel index workers to generator index
+            self._index_workers = [None] * len(self._paths)
+            for index in range(len(self._paths)):
+                p = mp.Process(target=self._index_worker, args=(index,))
+                p.daemon = True
+                p.start()
+                logger.info("Start worker process(pid:{}) to generate index.".format(p.pid))
+                self._index_workers[index] = p
+
+            # wait the index workers stop
+            for index in range(len(self._paths)):
+                self._index_workers[index].join()
 
     def _validate_array(self, k, v):
         """
@@ -487,3 +624,29 @@ class FileWriter:
                 error = "Field '{}' should be dict.".format(k)
                 return False, error
         return True, error
+
+    def _write_worker(self, i, in_queue):
+        """The worker do the data check and write to disk for parallel mode"""
+        while True:
+            # try to get new raw_data from master
+            try:
+                raw_data = in_queue.get(block=False)
+            except queue.Empty:
+                continue
+
+            # get EOF from master, worker should commit and stop
+            if raw_data == "EOF":
+                ret = self._writers[i].commit()
+                if ret != SUCCESS:
+                    raise RuntimeError("Commit the {}th shard of MindRecord file failed.".format(i))
+                break
+
+            # check the raw_data
+            if not isinstance(raw_data, list):
+                raise ParamTypeError('raw_data', 'list')
+            for each_raw in raw_data:
+                if not isinstance(each_raw, dict):
+                    raise ParamTypeError('raw_data item', 'dict')
+
+            self._verify_based_on_schema(raw_data)
+            self._writers[i].write_raw_data(raw_data, True, False)

mindspore/mindrecord/shardreader.py

@@ -106,3 +106,12 @@ class ShardReader:
     def close(self):
         """close MindRecord File."""
         self._reader.close()
+
+    def len(self):
+        """
+        Get the number of the samples in MindRecord.
+
+        Returns:
+            int, the number of the samples in MindRecord.
+        """
+        return self._reader.len()

mindspore/mindrecord/shardwriter.py

@@ -173,7 +173,7 @@ class ShardWriter:
         for item in data:
             row_blob = self._merge_blob({field: item[field] for field in self._header.blob_fields})
             if row_blob:
-                blob_data.append(list(row_blob))
+                blob_data.append(row_blob)
             # filter raw data according to schema
             row_raw = {field: self.convert_np_types(item[field])
                        for field in self._header.schema.keys() - self._header.blob_fields if field in item}