mindspore 1.10.0__cp38-cp38-win_amd64.whl → 2.0.0rc1__cp38-cp38-win_amd64.whl

mindspore/mindrecord/filewriter.py

@@ -17,8 +17,11 @@ This module is to write data into mindrecord.
 """
 import os
 import platform
+import queue
 import re
 import stat
+import time
+import multiprocessing as mp
 import numpy as np
 from mindspore import log as logger
 from .shardwriter import ShardWriter
@@ -26,7 +29,7 @@ from .shardreader import ShardReader
 from .shardheader import ShardHeader
 from .shardindexgenerator import ShardIndexGenerator
 from .shardutils import MIN_SHARD_COUNT, MAX_SHARD_COUNT, VALID_ATTRIBUTES, VALID_ARRAY_ATTRIBUTES, \
-    check_filename, VALUE_TYPE_MAP
+    check_filename, VALUE_TYPE_MAP, SUCCESS
 from .common.exceptions import ParamValueError, ParamTypeError, MRMInvalidSchemaError, MRMDefineIndexError
 
 __all__ = ['FileWriter']
@@ -103,6 +106,13 @@ class FileWriter:
         self._writer = ShardWriter()
         self._generator = None
 
+        # parallel write mode
+        self._parallel_writer = None
+        self._writers = None
+        self._queue = None
+        self._workers = None
+        self._index_workers = None
+
     @classmethod
     def open_for_append(cls, file_name):
         r"""
@@ -163,7 +173,7 @@ class FileWriter:
         The schema is added to describe the raw data to be written.
 
         Note:
-            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter`.
+            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter` .
 
         Args:
             content (dict): Dictionary of schema content.
@@ -186,14 +196,14 @@ class FileWriter:
     def add_index(self, index_fields):
         """
         Select index fields from schema to accelerate reading.
-        schema is added through `add_schema`.
+        schema is added through `add_schema` .
 
         Note:
             The index fields should be primitive type. e.g. int/float/str.
             If the function is not called, the fields of the primitive type
             in schema are set as indexes by default.
 
-            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter`.
+            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter` .
 
         Args:
             index_fields (list[str]): fields from schema.
@@ -217,56 +227,10 @@ class FileWriter:
                 raise ParamTypeError('index field', 'str')
         return self._header.add_index_fields(index_fields)
 
-    def _verify_based_on_schema(self, raw_data):
-        """
-        Verify data according to schema and remove invalid data if validation failed.
-
-        1) allowed data type contains: "int32", "int64", "float32", "float64", "string", "bytes".
-
-        Args:
-           raw_data (list[dict]): List of raw data.
-        """
-        error_data_dic = {}
-        schema_content = self._header.schema
-        for field in schema_content:
-            for i, v in enumerate(raw_data):
-                if i in error_data_dic:
-                    continue
-
-                if field not in v:
-                    error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                        "there is not '{}' object in the raw data.".format(i, field)
-                    continue
-                field_type = type(v[field]).__name__
-                if field_type not in VALUE_TYPE_MAP:
-                    error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                        "data type for '{}' is not matched.".format(i, field)
-                    continue
-
-                if schema_content[field]["type"] not in VALUE_TYPE_MAP[field_type]:
-                    error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                        "data type for '{}' is not matched.".format(i, field)
-                    continue
-
-                if field_type == 'ndarray':
-                    if 'shape' not in schema_content[field]:
-                        error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                            "data type for '{}' is not matched.".format(i, field)
-                    else:
-                        try:
-                            np.reshape(v[field], schema_content[field]['shape'])
-                        except ValueError:
-                            error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                                "data type for '{}' is not matched.".format(i, field)
-        error_data_dic = sorted(error_data_dic.items(), reverse=True)
-        for i, v in error_data_dic:
-            raw_data.pop(i)
-            logger.warning(v)
-
     def open_and_set_header(self):
         """
         Open writer and set header which stores meta information. The function is only used for parallel \
-        writing and is called before the `write_raw_data`.
+        writing and is called before the `write_raw_data` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
@@ -289,7 +253,7 @@ class FileWriter:
         files after the raw data is verified against the schema.
 
         Note:
-            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter`.
+            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter` .
 
         Args:
            raw_data (list[dict]): List of raw data.
@@ -304,23 +268,72 @@ class FileWriter:
             MRMValidateDataError: If data does not match blob fields.
             MRMSetHeaderError: If failed to set header.
             MRMWriteDatasetError: If failed to write dataset.
+            TypeError: If parallel_writer is not bool.
         """
-        if not self._writer.is_open:
-            self._writer.open(self._paths, self._overwrite)
-        if not self._writer.get_shard_header():
-            self._writer.set_shard_header(self._header)
-        if not isinstance(raw_data, list):
-            raise ParamTypeError('raw_data', 'list')
-        if self._flush and not self._append:
-            raise RuntimeError("Unexpected error. Not allow to call `write_raw_data` on flushed MindRecord files." \
-                               "When creating new Mindrecord files, please remove `commit` before `write_raw_data`." \
-                               "In other cases, when appending to existing MindRecord files, " \
-                               "please call `open_for_append` first and then `write_raw_data`.")
-        for each_raw in raw_data:
-            if not isinstance(each_raw, dict):
-                raise ParamTypeError('raw_data item', 'dict')
-        self._verify_based_on_schema(raw_data)
-        return self._writer.write_raw_data(raw_data, True, parallel_writer)
+        if not isinstance(parallel_writer, bool):
+            raise TypeError("The parameter `parallel_writer` must be bool.")
+
+        if self._parallel_writer is None:
+            self._parallel_writer = parallel_writer
+        if self._parallel_writer != parallel_writer:
+            raise RuntimeError("The parameter `parallel_writer` must be consistent during use.")
+        if not self._parallel_writer:
+            if not self._writer.is_open:
+                self._writer.open(self._paths, self._overwrite)
+            if not self._writer.get_shard_header():
+                self._writer.set_shard_header(self._header)
+            if not isinstance(raw_data, list):
+                raise ParamTypeError('raw_data', 'list')
+            if self._flush and not self._append:
+                raise RuntimeError("Not allowed to call `write_raw_data` on flushed MindRecord files." \
+                                   "When creating new Mindrecord files, please remove `commit` before " \
+                                   "`write_raw_data`. In other cases, when appending to existing MindRecord files, " \
+                                   "please call `open_for_append` first and then `write_raw_data`.")
+            for each_raw in raw_data:
+                if not isinstance(each_raw, dict):
+                    raise ParamTypeError('raw_data item', 'dict')
+            self._verify_based_on_schema(raw_data)
+            return self._writer.write_raw_data(raw_data, True, parallel_writer)
+
+        ## parallel write mode
+        # init the _writers and launch the workers
+        if self._writers is None:
+            self._writers = [None] * len(self._paths)  # writers used by worker
+            self._queue = mp.Queue(len(self._paths) * 2)  # queue for worker
+            self._workers = [None] * len(self._paths)  # worker process
+            for i, path in enumerate(self._paths):
+                self._writers[i] = ShardWriter()
+                self._writers[i].open([path], self._overwrite)
+                self._writers[i].set_shard_header(self._header)
+
+                # launch the workers for parallel write
+                self._queue._joincancelled = True  # pylint: disable=W0212
+                p = mp.Process(target=self._write_worker, args=(i, self._queue))
+                p.daemon = True
+                p.start()
+                logger.info("Start worker process(pid:{}) to parallel write.".format(p.pid))
+                self._workers[i] = p
+
+        # fill the self._queue
+        check_interval = 0.5  # 0.5s
+        start_time = time.time()
+        while True:
+            try:
+                self._queue.put(raw_data, block=False)
+            except queue.Full:
+                if time.time() - start_time > check_interval:
+                    start_time = time.time()
+                    logger.warning("Because there are too few MindRecord file shards, the efficiency of parallel " \
+                                   "writing is too low. You can stop the current task and add the parameter " \
+                                   "`shard_num` of `FileWriter` to upgrade the task.")
+
+                # check the status of worker process
+                for i in range(len(self._paths)):
+                    if not self._workers[i].is_alive():
+                        raise RuntimeError("Worker process(pid:{}) has stopped abnormal. Please check " \
+                                           "the above log".format(self._workers[i].pid))
+                continue
+            return SUCCESS
 
     def set_header_size(self, header_size):
         """
@@ -329,7 +342,6 @@ class FileWriter:
         the MindRecord file can store. If the size of header is larger than \
         the default size (16MB), users need to call the API to set a proper size.
 
-
         Args:
             header_size (int): Size of header, between 16*1024(16KB) and
                 128*1024*1024(128MB).
@@ -369,16 +381,16 @@ class FileWriter:
         Examples:
             >>> from mindspore.mindrecord import FileWriter
             >>> writer = FileWriter(file_name="test.mindrecord", shard_num=1)
-            >>> status = writer.set_page_size(1 << 26) # 128MB
+            >>> status = writer.set_page_size(1 << 26)  # 64MB
         """
         return self._writer.set_page_size(page_size)
 
-    def commit(self):
+    def commit(self):  # pylint: disable=W0212
         """
         Flush data in memory to disk and generate the corresponding database files.
 
         Note:
-            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter`.
+            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
@@ -389,25 +401,37 @@ class FileWriter:
             MRMIndexGeneratorError: If failed to create index generator.
             MRMGenerateIndexError: If failed to write to database.
             MRMCommitError: If failed to flush data to disk.
+            RuntimeError: Parallel write failed.
         """
-        self._flush = True
-        if not self._writer.is_open:
-            self._writer.open(self._paths, self._overwrite)
-        # permit commit without data
-        if not self._writer.get_shard_header():
-            self._writer.set_shard_header(self._header)
-        ret = self._writer.commit()
-        if self._index_generator:
-            if self._append:
-                self._generator = ShardIndexGenerator(self._file_name, self._append)
-            elif len(self._paths) >= 1:
-                self._generator = ShardIndexGenerator(os.path.realpath(self._paths[0]), self._append)
-            self._generator.build()
-            self._generator.write_to_db()
+        if not self._parallel_writer:
+            self._flush = True
+            if not self._writer.is_open:
+                self._writer.open(self._paths, self._overwrite)
+            # permit commit without data
+            if not self._writer.get_shard_header():
+                self._writer.set_shard_header(self._header)
+            self._writer.commit()
+            if self._index_generator:
+                if self._append:
+                    self._generator = ShardIndexGenerator(self._file_name, self._append)
+                elif len(self._paths) >= 1:
+                    self._generator = ShardIndexGenerator(os.path.realpath(self._paths[0]), self._append)
+                self._generator.build()
+                self._generator.write_to_db()
+        else:
+            # maybe a empty mindrecord, so need check _writers
+            if self._writers is None:
+                self._writers = [None] * len(self._paths)
+                for i, path in enumerate(self._paths):
+                    self._writers[i] = ShardWriter()
+                    self._writers[i].open(path, self._overwrite)
+                    self._writers[i].set_shard_header(self._header)
+
+            self._parallel_commit()
 
+        # change the file mode to 600
         mindrecord_files = []
         index_files = []
-        # change the file mode to 600
         for item in self._paths:
             if os.path.exists(item):
                 os.chmod(item, stat.S_IRUSR | stat.S_IWUSR)
@@ -420,7 +444,73 @@ class FileWriter:
         logger.info("The list of mindrecord files created are: {}, and the list of index files are: {}".format(
             mindrecord_files, index_files))
 
-        return ret
+        return SUCCESS
+
+    def _index_worker(self, i):
+        """The worker do the index generator"""
+        generator = ShardIndexGenerator(os.path.realpath(self._paths[i]), False)
+        generator.build()
+        generator.write_to_db()
+
+    def _parallel_commit(self):
+        """Parallel commit"""
+        # if some workers stopped, error may occur
+        alive_count = 0
+        for i in range(len(self._paths)):
+            if self._workers[i].is_alive():
+                alive_count += 1
+        if alive_count != len(self._paths):
+            raise RuntimeError("Parallel write worker error, please check the log file.")
+
+        # send EOF to worker process
+        for _ in range(len(self._paths)):
+            while True:
+                try:
+                    self._queue.put("EOF", block=False)
+                except queue.Full:
+                    time.sleep(1)
+                    continue
+                break
+
+        # wait the worker processing
+        while True:
+            alive_count = 0
+            for i in range(len(self._paths)):
+                if self._workers[i].is_alive():
+                    alive_count += 1
+            if alive_count == 0:
+                break
+            time.sleep(1)
+            logger.info("Waiting for all the parallel workers to finish.")
+
+        del self._queue
+
+        # wait for worker process stop
+        for index in range(len(self._paths)):
+            while True:
+                logger.info("Waiting for the worker process(pid:{}) to process all the data.".format(
+                    self._workers[index].pid))
+                if self._workers[index].is_alive():
+                    time.sleep(1)
+                    continue
+                elif self._workers[index].exitcode != 0:
+                    raise RuntimeError("Worker process(pid:{}) has stopped abnormal. Please check " \
+                                       "the above log".format(self._workers[index].pid))
+                break
+
+        if self._index_generator:
+            # use parallel index workers to generator index
+            self._index_workers = [None] * len(self._paths)
+            for index in range(len(self._paths)):
+                p = mp.Process(target=self._index_worker, args=(index,))
+                p.daemon = True
+                p.start()
+                logger.info("Start worker process(pid:{}) to generate index.".format(p.pid))
+                self._index_workers[index] = p
+
+            # wait the index workers stop
+            for index in range(len(self._paths)):
+                self._index_workers[index].join()
 
     def _validate_array(self, k, v):
         """
@@ -448,6 +538,52 @@ class FileWriter:
             return False, error
         return True, ''
 
+    def _verify_based_on_schema(self, raw_data):
+        """
+        Verify data according to schema and remove invalid data if validation failed.
+
+        1) allowed data type contains: "int32", "int64", "float32", "float64", "string", "bytes".
+
+        Args:
+           raw_data (list[dict]): List of raw data.
+        """
+        error_data_dic = {}
+        schema_content = self._header.schema
+        for field in schema_content:
+            for i, v in enumerate(raw_data):
+                if i in error_data_dic:
+                    continue
+
+                if field not in v:
+                    error_data_dic[i] = "for schema, {} th data is wrong, " \
+                                        "there is not '{}' object in the raw data.".format(i, field)
+                    continue
+                field_type = type(v[field]).__name__
+                if field_type not in VALUE_TYPE_MAP:
+                    error_data_dic[i] = "for schema, {} th data is wrong, " \
+                                        "data type for '{}' is not matched.".format(i, field)
+                    continue
+
+                if schema_content[field]["type"] not in VALUE_TYPE_MAP[field_type]:
+                    error_data_dic[i] = "for schema, {} th data is wrong, " \
+                                        "data type for '{}' is not matched.".format(i, field)
+                    continue
+
+                if field_type == 'ndarray':
+                    if 'shape' not in schema_content[field]:
+                        error_data_dic[i] = "for schema, {} th data is wrong, " \
+                                            "data type for '{}' is not matched.".format(i, field)
+                    else:
+                        try:
+                            np.reshape(v[field], schema_content[field]['shape'])
+                        except ValueError:
+                            error_data_dic[i] = "for schema, {} th data is wrong, " \
+                                                "data type for '{}' is not matched.".format(i, field)
+        error_data_dic = sorted(error_data_dic.items(), reverse=True)
+        for i, v in error_data_dic:
+            raw_data.pop(i)
+            logger.warning(v)
+
     def _validate_schema(self, content):
         """
         Validate schema and return validation result and error message.
@@ -488,3 +624,29 @@ class FileWriter:
                 error = "Field '{}' should be dict.".format(k)
                 return False, error
         return True, error
+
+    def _write_worker(self, i, in_queue):
+        """The worker do the data check and write to disk for parallel mode"""
+        while True:
+            # try to get new raw_data from master
+            try:
+                raw_data = in_queue.get(block=False)
+            except queue.Empty:
+                continue
+
+            # get EOF from master, worker should commit and stop
+            if raw_data == "EOF":
+                ret = self._writers[i].commit()
+                if ret != SUCCESS:
+                    raise RuntimeError("Commit the {}th shard of MindRecord file failed.".format(i))
+                break
+
+            # check the raw_data
+            if not isinstance(raw_data, list):
+                raise ParamTypeError('raw_data', 'list')
+            for each_raw in raw_data:
+                if not isinstance(each_raw, dict):
+                    raise ParamTypeError('raw_data item', 'dict')
+
+            self._verify_based_on_schema(raw_data)
+            self._writers[i].write_raw_data(raw_data, True, False)

mindspore/mindrecord/mindpage.py

@@ -34,7 +34,7 @@ class MindPage:
             It should not be smaller than 1 or larger than the number of processor cores.
 
     Raises:
-        ParamValueError: If `file_name`, `num_consumer` or columns is invalid.
+        ParamValueError: If `file_name` , `num_consumer` or columns is invalid.
         MRMInitSegmentError: If failed to initialize ShardSegment.
     """
     @check_parameter
@@ -54,6 +54,16 @@ class MindPage:
         """
         return self._candidate_fields
 
+    @property
+    def category_field(self):
+        """
+        Getter function for category fields.
+
+        Returns:
+            list[str], by which data could be grouped.
+        """
+        return self._category_field
+
     def get_category_fields(self):
         """
         Return candidate category fields.
@@ -86,16 +96,6 @@ class MindPage:
             raise MRMDefineCategoryError("Field '{}' is not a candidate category field.".format(category_field))
         return self._segment.set_category_field(category_field)
 
-    @property
-    def category_field(self):
-        """
-        Getter function for category fields.
-
-        Returns:
-            list[str], by which data could be grouped.
-        """
-        return self._category_field
-
     @category_field.setter
     def category_field(self, category_field):
         """
@@ -128,7 +128,7 @@ class MindPage:
         Query by category id in pagination.
 
         Args:
-             category_id (int): Category id, referred to the return of `read_category_info`.
+             category_id (int): Category id, referred to the return of `read_category_info` .
              page (int): Index of page.
              num_row (int): Number of rows in a page.
 
@@ -154,7 +154,7 @@ class MindPage:
 
         Args:
             category_name (str): String of category field's value,
-                referred to the return of `read_category_info`.
+                referred to the return of `read_category_info` .
             page (int): Index of page.
             num_row (int): Number of row in a page.
 

mindspore/mindrecord/shardheader.py

@@ -34,6 +34,21 @@ class ShardHeader:
         else:
             self._header = ms.ShardHeader()
 
+    @property
+    def header(self):
+        """Getter of header"""
+        return self._header
+
+    @property
+    def blob_fields(self):
+        """Getter of blob fields"""
+        return self._get_blob_fields()
+
+    @property
+    def schema(self):
+        """Getter of schema"""
+        return self._get_schema()
+
     def add_schema(self, schema):
         """
         Add object of ShardSchema.
@@ -93,11 +108,6 @@ class ShardHeader:
             raise MRMBuildSchemaError
         return schema
 
-    @property
-    def header(self):
-        """Getter of header"""
-        return self._header
-
     def _get_schema(self):
         """
         Get schema info.
@@ -129,13 +139,3 @@ class ShardHeader:
 
         logger.critical("Failed to get meta info.")
         raise MRMGetMetaError
-
-    @property
-    def blob_fields(self):
-        """Getter of blob fields"""
-        return self._get_blob_fields()
-
-    @property
-    def schema(self):
-        """Getter of schema"""
-        return self._get_schema()

mindspore/mindrecord/shardreader.py

@@ -106,3 +106,12 @@ class ShardReader:
     def close(self):
         """close MindRecord File."""
         self._reader.close()
+
+    def len(self):
+        """
+        Get the number of the samples in MindRecord.
+
+        Returns:
+            int, the number of the samples in MindRecord.
+        """
+        return self._reader.len()

mindspore/mindrecord/shardwriter.py

@@ -36,6 +36,18 @@ class ShardWriter:
         self._header = None
         self._is_open = False
 
+    @property
+    def is_open(self):
+        """getter function"""
+        return self._is_open
+
+    @staticmethod
+    def convert_np_types(val):
+        """convert numpy type to python primitive type"""
+        if isinstance(val, (np.int32, np.int64, np.float32, np.float64)):
+            return val.item()
+        return val
+
     def open(self, paths, override):
         """
         Open a new MindRecord File and prepare to write raw data.
@@ -137,13 +149,6 @@ class ShardWriter:
     def get_shard_header(self):
         return self._header
 
-    @staticmethod
-    def convert_np_types(val):
-        """convert numpy type to python primitive type"""
-        if isinstance(val, (np.int32, np.int64, np.float32, np.float64)):
-            return val.item()
-        return val
-
     def write_raw_data(self, data, validate=True, parallel_writer=False):
         """
         Write raw data of cv dataset.
@@ -168,7 +173,7 @@ class ShardWriter:
         for item in data:
             row_blob = self._merge_blob({field: item[field] for field in self._header.blob_fields})
             if row_blob:
-                blob_data.append(list(row_blob))
+                blob_data.append(row_blob)
             # filter raw data according to schema
             row_raw = {field: self.convert_np_types(item[field])
                        for field in self._header.schema.keys() - self._header.blob_fields if field in item}
@@ -181,6 +186,22 @@ class ShardWriter:
             raise MRMWriteDatasetError
         return ret
 
+    def commit(self):
+        """
+        Flush data to disk.
+
+        Returns:
+            MSRStatus, SUCCESS or FAILED.
+
+        Raises:
+            MRMCommitError: If failed to flush data to disk.
+        """
+        ret = self._writer.commit()
+        if ret != ms.MSRStatus.SUCCESS:
+            logger.critical("Failed to commit.")
+            raise MRMCommitError
+        return ret
+
     def _merge_blob(self, blob_data):
         """
         Merge multiple blob data whose type is bytes or ndarray
@@ -207,24 +228,3 @@ class ShardWriter:
             merged += int_to_bytes(len(v))
             merged += v
         return merged
-
-    def commit(self):
-        """
-        Flush data to disk.
-
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
-        Raises:
-            MRMCommitError: If failed to flush data to disk.
-        """
-        ret = self._writer.commit()
-        if ret != ms.MSRStatus.SUCCESS:
-            logger.critical("Failed to commit.")
-            raise MRMCommitError
-        return ret
-
-    @property
-    def is_open(self):
-        """getter function"""
-        return self._is_open