mindspore 1.10.0__cp37-none-any.whl → 2.0.0rc1__cp37-none-any.whl

mindspore/log.py

@@ -37,18 +37,18 @@ _setup_logger_lock = threading.Lock()
 
 # When getting the logger, Used to check whether
 # the logger already exists
-_global_logger = None
+GLOBAL_LOGGER = None
 
 # The flag for enable console output
-_std_on = '1'
+STD_ON = '1'
 # The flag for disable console output
-_std_off = '0'
+STD_OFF = '0'
 # Rotating max bytes, default is 50M
-_logger_def_max_bytes = '52428800'
+MAX_BYTES = '52428800'
 # Rotating backup count, default is 30
-_logger_def_backup_count = '30'
+BACKUP_COUNT = '30'
 # The default log level
-_logger_def_level = '2'
+LOGGER_LEVEL = '2'
 
 # Log level name and level mapping
 _name_to_level = {
@@ -155,8 +155,8 @@ def _get_logger():
     Returns:
         Logger, a logger.
     """
-    if _global_logger:
-        return _global_logger
+    if GLOBAL_LOGGER:
+        return GLOBAL_LOGGER
 
     kwargs = _get_env_config()
     _verify_config(kwargs)
@@ -180,11 +180,11 @@ def _adapt_cfg(kwargs):
     Returns:
         Dict, the input parameter dictionary.
     """
-    kwargs['level'] = _gloglevel_to_name.get(kwargs.get('level', _logger_def_level))
-    kwargs['stderr_level'] = _gloglevel_to_name.get(kwargs.get('stderr_level', _logger_def_level))
-    kwargs['console'] = not kwargs.get('console') == _std_off
-    kwargs['maxBytes'] = int(kwargs.get('maxBytes', _logger_def_max_bytes))
-    kwargs['backupCount'] = int(kwargs.get('backupCount', _logger_def_backup_count))
+    kwargs['level'] = _gloglevel_to_name.get(kwargs.get('level', LOGGER_LEVEL))
+    kwargs['stderr_level'] = _gloglevel_to_name.get(kwargs.get('stderr_level', LOGGER_LEVEL))
+    kwargs['console'] = not kwargs.get('console') == STD_OFF
+    kwargs['maxBytes'] = int(kwargs.get('maxBytes', MAX_BYTES))
+    kwargs['backupCount'] = int(kwargs.get('backupCount', BACKUP_COUNT))
     return kwargs
 
 
@@ -279,7 +279,7 @@ def _check_directory_by_regular(target, reg=None, flag=re.ASCII, prim_name=None)
     if not isinstance(target, str):
         raise ValueError("The directory {} must be string, but got {}, please check it".format(target, type(target)))
     if reg is None:
-        reg = r"^[\/0-9a-zA-Z\_\-\.\:\\]+$"
+        reg = r"^[\/0-9a-zA-Z@\_\-\.\:\\]+$"
     if re.match(reg, target, flag) is None:
         prim_name = f'in `{prim_name}`' if prim_name else ""
         raise ValueError("'{}' {} is illegal, it should be match regular'{}' by flag'{}'".format(
@@ -335,29 +335,29 @@ def _verify_config(kwargs):
     file_path = kwargs.get('filepath', None)
 
     if console is not None:
-        if not console.isdigit() or console not in (_std_off, _std_on):
+        if not console.isdigit() or console not in (STD_OFF, STD_ON):
             raise ValueError(f'Incorrect value, the value of {_confmap_dict["console"]} must be 0 or 1, '
                              f'but got {console}.')
 
-        if console == _std_off and not file_path:
+        if console == STD_OFF and not file_path:
             raise ValueError(f'When {_confmap_dict["console"]} is set to 0, the directory of saving log '
                              f'{_confmap_dict["filepath"]} must be set, but got it empty.')
 
         # Check the input value of filepath
-        if console == _std_off and file_path is not None:
+        if console == STD_OFF and file_path is not None:
             file_real_path = os.path.realpath(file_path)
             if not os.path.exists(file_real_path):
                 _make_directory(file_real_path)
         # Check the input value of maxBytes
         max_bytes = kwargs.get('maxBytes', None)
-        if console == _std_off and max_bytes is not None:
+        if console == STD_OFF and max_bytes is not None:
             if not max_bytes.isdigit():
                 raise ValueError(f'Incorrect value, the value of {_confmap_dict["maxBytes"]} must be positive integer. '
                                  f'But got {_confmap_dict["maxBytes"]}:{max_bytes}.')
 
         # Check the input value of backupCount
         backup_count = kwargs.get('backupCount', None)
-        if console == _std_off and backup_count is not None:
+        if console == STD_OFF and backup_count is not None:
             if not backup_count.isdigit():
                 raise ValueError(f'Incorrect value, the value of {_confmap_dict["backupCount"]} must be positive '
                                  f'integer. But got {_confmap_dict["backupCount"]}:{backup_count}')
@@ -404,10 +404,10 @@ def get_log_config():
     handler = logger.handlers[0]
     config_dict = {}
     config_dict['GLOG_v'] = get_level()
-    config_dict['GLOG_logtostderr'] = _std_on
+    config_dict['GLOG_logtostderr'] = STD_ON
 
     if handler.name == 'FileHandler':
-        config_dict['GLOG_logtostderr'] = _std_off
+        config_dict['GLOG_logtostderr'] = STD_OFF
         # Separating file path and name
         file_path_and_name = os.path.split(handler.baseFilename)
         config_dict['GLOG_log_dir'] = file_path_and_name[0]
@@ -532,12 +532,12 @@ def _setup_logger(kwargs):
     pid = str(os.getpid())
     log_name = 'mindspore.log.' + pid
 
-    global _global_logger
+    global GLOBAL_LOGGER
 
     _setup_logger_lock.acquire()
     try:
-        if _global_logger:
-            return _global_logger
+        if GLOBAL_LOGGER:
+            return GLOBAL_LOGGER
 
         logger = logging.getLogger(name=f'{sub_module}.{log_name}')
         # Override findCaller on the logger, Support for getting log record
@@ -569,9 +569,9 @@ def _setup_logger(kwargs):
             logfile_handler = _MultiCompatibleRotatingFileHandler(
                 filename=file_name,
                 # Rotating max bytes, default is 50M
-                maxBytes=kwargs.get('maxBytes', _logger_def_max_bytes),
+                maxBytes=kwargs.get('maxBytes', MAX_BYTES),
                 # Rotating backup count, default is 30
-                backupCount=kwargs.get('backupCount', _logger_def_backup_count),
+                backupCount=kwargs.get('backupCount', BACKUP_COUNT),
                 encoding='utf8'
             )
             logfile_handler.name = 'FileHandler'
@@ -589,11 +589,11 @@ def _setup_logger(kwargs):
             console_handler.setLevel(kwargs.get('stderr_level', logging.WARNING))
             logger.addHandler(console_handler)
 
-        _global_logger = logger
+        GLOBAL_LOGGER = logger
 
     finally:
         _setup_logger_lock.release()
-    return _global_logger
+    return GLOBAL_LOGGER
 
 
 class _LogActionOnce:

mindspore/mindrecord/common/exceptions.py

@@ -80,8 +80,7 @@ class MindRecordException(Exception):
         """Initialize an error which may occurs in mindrecord."""
         super(MindRecordException, self).__init__()
         class_name = self.__class__.__name__
-        error_item = EXCEPTIONS[class_name] if class_name in EXCEPTIONS \
-            else EXCEPTIONS['UnknownError']
+        error_item = EXCEPTIONS.get(class_name) if class_name in EXCEPTIONS else EXCEPTIONS.get('UnknownError')
         self._error_msg = error_item[1]
         self._error_code = MindRecordException.transform_error_code(error_item[0])
 
@@ -123,8 +122,7 @@ class MindRecordException(Exception):
         runtime = LogRuntime.RT_HOST
         error_code_type = ErrorCodeType.ERROR_CODE
         error_level = ErrorLevel.COMMON_LEVEL
-        exception_no_range_per_module = 100
-        mod_id = int(exception_no / exception_no_range_per_module) + 1
+        mod_id = int(exception_no / 100) + 1
         error_code = (((0xFF & runtime) << 30)
                       | ((0xFF & error_code_type) << 28)
                       | ((0xFF & error_level) << 25)

mindspore/mindrecord/filereader.py

@@ -43,7 +43,7 @@ class FileReader:
         operator (int, optional): Reserved parameter for operators. Default: None.
 
     Raises:
-        ParamValueError: If `file_name`, `num_consumer` or `columns` is invalid.
+        ParamValueError: If `file_name` , `num_consumer` or `columns` is invalid.
 
     Examples:
         >>> from mindspore.mindrecord import FileReader
@@ -100,3 +100,21 @@ class FileReader:
     def close(self):
         """Stop reader worker and close file."""
         self._reader.close()
+
+    def schema(self):
+        """
+        Get the schema of the MindRecord.
+
+        Returns:
+            dict, the schema info.
+        """
+        return self._header.schema
+
+    def len(self):
+        """
+        Get the number of the samples in MindRecord.
+
+        Returns:
+            int, the number of the samples in MindRecord.
+        """
+        return self._reader.len()

mindspore/mindrecord/filewriter.py

@@ -17,8 +17,11 @@ This module is to write data into mindrecord.
 """
 import os
 import platform
+import queue
 import re
 import stat
+import time
+import multiprocessing as mp
 import numpy as np
 from mindspore import log as logger
 from .shardwriter import ShardWriter
@@ -26,7 +29,7 @@ from .shardreader import ShardReader
 from .shardheader import ShardHeader
 from .shardindexgenerator import ShardIndexGenerator
 from .shardutils import MIN_SHARD_COUNT, MAX_SHARD_COUNT, VALID_ATTRIBUTES, VALID_ARRAY_ATTRIBUTES, \
-    check_filename, VALUE_TYPE_MAP
+    check_filename, VALUE_TYPE_MAP, SUCCESS
 from .common.exceptions import ParamValueError, ParamTypeError, MRMInvalidSchemaError, MRMDefineIndexError
 
 __all__ = ['FileWriter']
@@ -103,6 +106,13 @@ class FileWriter:
         self._writer = ShardWriter()
         self._generator = None
 
+        # parallel write mode
+        self._parallel_writer = None
+        self._writers = None
+        self._queue = None
+        self._workers = None
+        self._index_workers = None
+
     @classmethod
     def open_for_append(cls, file_name):
         r"""
@@ -163,7 +173,7 @@ class FileWriter:
         The schema is added to describe the raw data to be written.
 
         Note:
-            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter`.
+            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter` .
 
         Args:
             content (dict): Dictionary of schema content.
@@ -186,14 +196,14 @@ class FileWriter:
     def add_index(self, index_fields):
         """
         Select index fields from schema to accelerate reading.
-        schema is added through `add_schema`.
+        schema is added through `add_schema` .
 
         Note:
             The index fields should be primitive type. e.g. int/float/str.
             If the function is not called, the fields of the primitive type
             in schema are set as indexes by default.
 
-            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter`.
+            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter` .
 
         Args:
             index_fields (list[str]): fields from schema.
@@ -217,56 +227,10 @@ class FileWriter:
                 raise ParamTypeError('index field', 'str')
         return self._header.add_index_fields(index_fields)
 
-    def _verify_based_on_schema(self, raw_data):
-        """
-        Verify data according to schema and remove invalid data if validation failed.
-
-        1) allowed data type contains: "int32", "int64", "float32", "float64", "string", "bytes".
-
-        Args:
-           raw_data (list[dict]): List of raw data.
-        """
-        error_data_dic = {}
-        schema_content = self._header.schema
-        for field in schema_content:
-            for i, v in enumerate(raw_data):
-                if i in error_data_dic:
-                    continue
-
-                if field not in v:
-                    error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                        "there is not '{}' object in the raw data.".format(i, field)
-                    continue
-                field_type = type(v[field]).__name__
-                if field_type not in VALUE_TYPE_MAP:
-                    error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                        "data type for '{}' is not matched.".format(i, field)
-                    continue
-
-                if schema_content[field]["type"] not in VALUE_TYPE_MAP[field_type]:
-                    error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                        "data type for '{}' is not matched.".format(i, field)
-                    continue
-
-                if field_type == 'ndarray':
-                    if 'shape' not in schema_content[field]:
-                        error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                            "data type for '{}' is not matched.".format(i, field)
-                    else:
-                        try:
-                            np.reshape(v[field], schema_content[field]['shape'])
-                        except ValueError:
-                            error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                                "data type for '{}' is not matched.".format(i, field)
-        error_data_dic = sorted(error_data_dic.items(), reverse=True)
-        for i, v in error_data_dic:
-            raw_data.pop(i)
-            logger.warning(v)
-
     def open_and_set_header(self):
         """
         Open writer and set header which stores meta information. The function is only used for parallel \
-        writing and is called before the `write_raw_data`.
+        writing and is called before the `write_raw_data` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
@@ -289,7 +253,7 @@ class FileWriter:
         files after the raw data is verified against the schema.
 
         Note:
-            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter`.
+            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter` .
 
         Args:
            raw_data (list[dict]): List of raw data.
@@ -304,23 +268,72 @@ class FileWriter:
             MRMValidateDataError: If data does not match blob fields.
             MRMSetHeaderError: If failed to set header.
             MRMWriteDatasetError: If failed to write dataset.
+            TypeError: If parallel_writer is not bool.
         """
-        if not self._writer.is_open:
-            self._writer.open(self._paths, self._overwrite)
-        if not self._writer.get_shard_header():
-            self._writer.set_shard_header(self._header)
-        if not isinstance(raw_data, list):
-            raise ParamTypeError('raw_data', 'list')
-        if self._flush and not self._append:
-            raise RuntimeError("Unexpected error. Not allow to call `write_raw_data` on flushed MindRecord files." \
-                               "When creating new Mindrecord files, please remove `commit` before `write_raw_data`." \
-                               "In other cases, when appending to existing MindRecord files, " \
-                               "please call `open_for_append` first and then `write_raw_data`.")
-        for each_raw in raw_data:
-            if not isinstance(each_raw, dict):
-                raise ParamTypeError('raw_data item', 'dict')
-        self._verify_based_on_schema(raw_data)
-        return self._writer.write_raw_data(raw_data, True, parallel_writer)
+        if not isinstance(parallel_writer, bool):
+            raise TypeError("The parameter `parallel_writer` must be bool.")
+
+        if self._parallel_writer is None:
+            self._parallel_writer = parallel_writer
+        if self._parallel_writer != parallel_writer:
+            raise RuntimeError("The parameter `parallel_writer` must be consistent during use.")
+        if not self._parallel_writer:
+            if not self._writer.is_open:
+                self._writer.open(self._paths, self._overwrite)
+            if not self._writer.get_shard_header():
+                self._writer.set_shard_header(self._header)
+            if not isinstance(raw_data, list):
+                raise ParamTypeError('raw_data', 'list')
+            if self._flush and not self._append:
+                raise RuntimeError("Not allowed to call `write_raw_data` on flushed MindRecord files." \
+                                   "When creating new Mindrecord files, please remove `commit` before " \
+                                   "`write_raw_data`. In other cases, when appending to existing MindRecord files, " \
+                                   "please call `open_for_append` first and then `write_raw_data`.")
+            for each_raw in raw_data:
+                if not isinstance(each_raw, dict):
+                    raise ParamTypeError('raw_data item', 'dict')
+            self._verify_based_on_schema(raw_data)
+            return self._writer.write_raw_data(raw_data, True, parallel_writer)
+
+        ## parallel write mode
+        # init the _writers and launch the workers
+        if self._writers is None:
+            self._writers = [None] * len(self._paths)  # writers used by worker
+            self._queue = mp.Queue(len(self._paths) * 2)  # queue for worker
+            self._workers = [None] * len(self._paths)  # worker process
+            for i, path in enumerate(self._paths):
+                self._writers[i] = ShardWriter()
+                self._writers[i].open([path], self._overwrite)
+                self._writers[i].set_shard_header(self._header)
+
+                # launch the workers for parallel write
+                self._queue._joincancelled = True  # pylint: disable=W0212
+                p = mp.Process(target=self._write_worker, args=(i, self._queue))
+                p.daemon = True
+                p.start()
+                logger.info("Start worker process(pid:{}) to parallel write.".format(p.pid))
+                self._workers[i] = p
+
+        # fill the self._queue
+        check_interval = 0.5  # 0.5s
+        start_time = time.time()
+        while True:
+            try:
+                self._queue.put(raw_data, block=False)
+            except queue.Full:
+                if time.time() - start_time > check_interval:
+                    start_time = time.time()
+                    logger.warning("Because there are too few MindRecord file shards, the efficiency of parallel " \
+                                   "writing is too low. You can stop the current task and add the parameter " \
+                                   "`shard_num` of `FileWriter` to upgrade the task.")
+
+                # check the status of worker process
+                for i in range(len(self._paths)):
+                    if not self._workers[i].is_alive():
+                        raise RuntimeError("Worker process(pid:{}) has stopped abnormal. Please check " \
+                                           "the above log".format(self._workers[i].pid))
+                continue
+            return SUCCESS
 
     def set_header_size(self, header_size):
         """
@@ -329,7 +342,6 @@ class FileWriter:
         the MindRecord file can store. If the size of header is larger than \
         the default size (16MB), users need to call the API to set a proper size.
 
-
         Args:
             header_size (int): Size of header, between 16*1024(16KB) and
                 128*1024*1024(128MB).
@@ -369,16 +381,16 @@ class FileWriter:
         Examples:
             >>> from mindspore.mindrecord import FileWriter
             >>> writer = FileWriter(file_name="test.mindrecord", shard_num=1)
-            >>> status = writer.set_page_size(1 << 26) # 128MB
+            >>> status = writer.set_page_size(1 << 26)  # 64MB
         """
         return self._writer.set_page_size(page_size)
 
-    def commit(self):
+    def commit(self):  # pylint: disable=W0212
         """
         Flush data in memory to disk and generate the corresponding database files.
 
         Note:
-            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter`.
+            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
@@ -389,25 +401,37 @@ class FileWriter:
             MRMIndexGeneratorError: If failed to create index generator.
             MRMGenerateIndexError: If failed to write to database.
             MRMCommitError: If failed to flush data to disk.
+            RuntimeError: Parallel write failed.
         """
-        self._flush = True
-        if not self._writer.is_open:
-            self._writer.open(self._paths, self._overwrite)
-        # permit commit without data
-        if not self._writer.get_shard_header():
-            self._writer.set_shard_header(self._header)
-        ret = self._writer.commit()
-        if self._index_generator:
-            if self._append:
-                self._generator = ShardIndexGenerator(self._file_name, self._append)
-            elif len(self._paths) >= 1:
-                self._generator = ShardIndexGenerator(os.path.realpath(self._paths[0]), self._append)
-            self._generator.build()
-            self._generator.write_to_db()
+        if not self._parallel_writer:
+            self._flush = True
+            if not self._writer.is_open:
+                self._writer.open(self._paths, self._overwrite)
+            # permit commit without data
+            if not self._writer.get_shard_header():
+                self._writer.set_shard_header(self._header)
+            self._writer.commit()
+            if self._index_generator:
+                if self._append:
+                    self._generator = ShardIndexGenerator(self._file_name, self._append)
+                elif len(self._paths) >= 1:
+                    self._generator = ShardIndexGenerator(os.path.realpath(self._paths[0]), self._append)
+                self._generator.build()
+                self._generator.write_to_db()
+        else:
+            # maybe a empty mindrecord, so need check _writers
+            if self._writers is None:
+                self._writers = [None] * len(self._paths)
+                for i, path in enumerate(self._paths):
+                    self._writers[i] = ShardWriter()
+                    self._writers[i].open(path, self._overwrite)
+                    self._writers[i].set_shard_header(self._header)
+
+            self._parallel_commit()
 
+        # change the file mode to 600
         mindrecord_files = []
         index_files = []
-        # change the file mode to 600
         for item in self._paths:
             if os.path.exists(item):
                 os.chmod(item, stat.S_IRUSR | stat.S_IWUSR)
@@ -420,7 +444,73 @@ class FileWriter:
         logger.info("The list of mindrecord files created are: {}, and the list of index files are: {}".format(
             mindrecord_files, index_files))
 
-        return ret
+        return SUCCESS
+
+    def _index_worker(self, i):
+        """The worker do the index generator"""
+        generator = ShardIndexGenerator(os.path.realpath(self._paths[i]), False)
+        generator.build()
+        generator.write_to_db()
+
+    def _parallel_commit(self):
+        """Parallel commit"""
+        # if some workers stopped, error may occur
+        alive_count = 0
+        for i in range(len(self._paths)):
+            if self._workers[i].is_alive():
+                alive_count += 1
+        if alive_count != len(self._paths):
+            raise RuntimeError("Parallel write worker error, please check the log file.")
+
+        # send EOF to worker process
+        for _ in range(len(self._paths)):
+            while True:
+                try:
+                    self._queue.put("EOF", block=False)
+                except queue.Full:
+                    time.sleep(1)
+                    continue
+                break
+
+        # wait the worker processing
+        while True:
+            alive_count = 0
+            for i in range(len(self._paths)):
+                if self._workers[i].is_alive():
+                    alive_count += 1
+            if alive_count == 0:
+                break
+            time.sleep(1)
+            logger.info("Waiting for all the parallel workers to finish.")
+
+        del self._queue
+
+        # wait for worker process stop
+        for index in range(len(self._paths)):
+            while True:
+                logger.info("Waiting for the worker process(pid:{}) to process all the data.".format(
+                    self._workers[index].pid))
+                if self._workers[index].is_alive():
+                    time.sleep(1)
+                    continue
+                elif self._workers[index].exitcode != 0:
+                    raise RuntimeError("Worker process(pid:{}) has stopped abnormal. Please check " \
+                                       "the above log".format(self._workers[index].pid))
+                break
+
+        if self._index_generator:
+            # use parallel index workers to generator index
+            self._index_workers = [None] * len(self._paths)
+            for index in range(len(self._paths)):
+                p = mp.Process(target=self._index_worker, args=(index,))
+                p.daemon = True
+                p.start()
+                logger.info("Start worker process(pid:{}) to generate index.".format(p.pid))
+                self._index_workers[index] = p
+
+            # wait the index workers stop
+            for index in range(len(self._paths)):
+                self._index_workers[index].join()
 
     def _validate_array(self, k, v):
         """
@@ -448,6 +538,52 @@ class FileWriter:
             return False, error
         return True, ''
 
+    def _verify_based_on_schema(self, raw_data):
+        """
+        Verify data according to schema and remove invalid data if validation failed.
+
+        1) allowed data type contains: "int32", "int64", "float32", "float64", "string", "bytes".
+
+        Args:
+           raw_data (list[dict]): List of raw data.
+        """
+        error_data_dic = {}
+        schema_content = self._header.schema
+        for field in schema_content:
+            for i, v in enumerate(raw_data):
+                if i in error_data_dic:
+                    continue
+
+                if field not in v:
+                    error_data_dic[i] = "for schema, {} th data is wrong, " \
+                                        "there is not '{}' object in the raw data.".format(i, field)
+                    continue
+                field_type = type(v[field]).__name__
+                if field_type not in VALUE_TYPE_MAP:
+                    error_data_dic[i] = "for schema, {} th data is wrong, " \
+                                        "data type for '{}' is not matched.".format(i, field)
+                    continue
+
+                if schema_content[field]["type"] not in VALUE_TYPE_MAP[field_type]:
+                    error_data_dic[i] = "for schema, {} th data is wrong, " \
+                                        "data type for '{}' is not matched.".format(i, field)
+                    continue
+
+                if field_type == 'ndarray':
+                    if 'shape' not in schema_content[field]:
+                        error_data_dic[i] = "for schema, {} th data is wrong, " \
+                                            "data type for '{}' is not matched.".format(i, field)
+                    else:
+                        try:
+                            np.reshape(v[field], schema_content[field]['shape'])
+                        except ValueError:
+                            error_data_dic[i] = "for schema, {} th data is wrong, " \
+                                                "data type for '{}' is not matched.".format(i, field)
+        error_data_dic = sorted(error_data_dic.items(), reverse=True)
+        for i, v in error_data_dic:
+            raw_data.pop(i)
+            logger.warning(v)
+
     def _validate_schema(self, content):
         """
         Validate schema and return validation result and error message.
@@ -488,3 +624,29 @@ class FileWriter:
                 error = "Field '{}' should be dict.".format(k)
                 return False, error
         return True, error
+
+    def _write_worker(self, i, in_queue):
+        """The worker do the data check and write to disk for parallel mode"""
+        while True:
+            # try to get new raw_data from master
+            try:
+                raw_data = in_queue.get(block=False)
+            except queue.Empty:
+                continue
+
+            # get EOF from master, worker should commit and stop
+            if raw_data == "EOF":
+                ret = self._writers[i].commit()
+                if ret != SUCCESS:
+                    raise RuntimeError("Commit the {}th shard of MindRecord file failed.".format(i))
+                break
+
+            # check the raw_data
+            if not isinstance(raw_data, list):
+                raise ParamTypeError('raw_data', 'list')
+            for each_raw in raw_data:
+                if not isinstance(each_raw, dict):
+                    raise ParamTypeError('raw_data item', 'dict')
+
+            self._verify_based_on_schema(raw_data)
+            self._writers[i].write_raw_data(raw_data, True, False)