mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/mindrecord/filereader.py

@@ -23,6 +23,7 @@ from .shardheader import ShardHeader
 from .shardutils import populate_data
 from .shardutils import check_parameter
 from .common.exceptions import ParamTypeError
+from .config import _get_enc_key, _get_dec_mode, decrypt, verify_file_hash
 
 __all__ = ['FileReader']
 
@@ -82,6 +83,30 @@ class FileReader:
         else:
             self._file_name = file_name
 
+        if isinstance(self._file_name, str):
+            # decrypt the data file and index file
+            index_file_name = self._file_name + ".db"
+            decrypt_filename = decrypt(self._file_name, _get_enc_key(), _get_dec_mode())
+            self._file_name = decrypt_filename
+            decrypt(index_file_name, _get_enc_key(), _get_dec_mode())
+
+            # verify integrity check
+            verify_file_hash(self._file_name)
+            verify_file_hash(self._file_name + ".db")
+        else:
+            file_names_decrypted = []
+            for item in self._file_name:
+                # decrypt the data file and index file
+                index_file_name = item + ".db"
+                decrypt_filename = decrypt(item, _get_enc_key(), _get_dec_mode())
+                file_names_decrypted.append(decrypt_filename)
+                decrypt(index_file_name, _get_enc_key(), _get_dec_mode())
+
+                # verify integrity check
+                verify_file_hash(decrypt_filename)
+                verify_file_hash(decrypt_filename + ".db")
+            self._file_name = file_names_decrypted
+
         self._reader.open(self._file_name, num_consumer, columns, operator)
         self._header = ShardHeader(self._reader.get_header())
         self._reader.launch()

mindspore/mindrecord/filewriter.py

@@ -19,6 +19,7 @@ import os
 import platform
 import queue
 import re
+import shutil
 import stat
 import time
 import multiprocessing as mp
@@ -31,6 +32,8 @@ from .shardindexgenerator import ShardIndexGenerator
 from .shardutils import MIN_SHARD_COUNT, MAX_SHARD_COUNT, VALID_ATTRIBUTES, VALID_ARRAY_ATTRIBUTES, \
     check_filename, VALUE_TYPE_MAP, SUCCESS
 from .common.exceptions import ParamValueError, ParamTypeError, MRMInvalidSchemaError, MRMDefineIndexError
+from .config import _get_enc_key, _get_enc_mode, _get_dec_mode, _get_hash_mode, encrypt, decrypt, append_hash_to_file, \
+    verify_file_hash
 
 __all__ = ['FileWriter']
 
@@ -57,14 +60,14 @@ class FileWriter:
         >>>
         >>> writer = FileWriter(file_name="test.mindrecord", shard_num=1, overwrite=True)
         >>> schema_json = {"file_name": {"type": "string"}, "label": {"type": "int32"}, "data": {"type": "bytes"}}
-        >>> schema_id = writer.add_schema(schema_json, "test_schema")
+        >>> writer.add_schema(schema_json, "test_schema")
         >>> indexes = ["file_name", "label"]
-        >>> status = writer.add_index(indexes)
+        >>> writer.add_index(indexes)
         >>> for i in range(10):
         ...     data = [{"file_name": str(i) + ".jpg", "label": i,
         ...              "data": b"\x10c\xb3w\xa8\xee$o&<q\x8c\x8e(\xa2\x90\x90\x96\xbc\xb1\x1e\xd4QER\x13?\xff"}]
-        ...     status = writer.write_raw_data(data)
-        >>> status = writer.commit()
+        ...     writer.write_raw_data(data)
+        >>> writer.commit()
     """
 
     def __init__(self, file_name, shard_num=1, overwrite=False):
@@ -93,6 +96,9 @@ class FileWriter:
         if self._shard_num == 1:
             self._paths = [self._file_name]
         else:
+            if _get_enc_key() is not None or _get_hash_mode() is not None:
+                raise RuntimeError("When encode mode or hash check is enabled, " +
+                                   "the automatic sharding function is unavailable.")
             self._paths = ["{}{}".format(self._file_name,
                                          str(x).rjust(suffix_shard_size, '0'))
                            for x in range(self._shard_num)]
@@ -107,8 +113,10 @@ class FileWriter:
         # parallel write mode
         self._parallel_writer = None
         self._writers = None
-        self._queue = None
+        self._queues = []      # the data queue from main process to worker process
+        self._msg_queues = []  # the err msg from worker process to main process
         self._workers = None
+        self._workers_loop_index = 0
         self._index_workers = None
 
     @classmethod
@@ -135,21 +143,35 @@ class FileWriter:
             ...          "data": b"\x10c\xb3w\xa8\xee$o&<q\x8c\x8e(\xa2\x90\x90\x96\xbc\xb1\x1e\xd4QER\x13?\xff"}]
             >>> writer = FileWriter(file_name="test.mindrecord", shard_num=1, overwrite=True)
             >>> schema_json = {"file_name": {"type": "string"}, "label": {"type": "int32"}, "data": {"type": "bytes"}}
-            >>> schema_id = writer.add_schema(schema_json, "test_schema")
-            >>> status = writer.write_raw_data(data)
-            >>> status = writer.commit()
+            >>> writer.add_schema(schema_json, "test_schema")
+            >>> writer.write_raw_data(data)
+            >>> writer.commit()
             >>>
             >>> write_append = FileWriter.open_for_append("test.mindrecord")
             >>> for i in range(9):
             ...     data = [{"file_name": str(i+1) + ".jpg", "label": i,
             ...              "data": b"\x10c\xb3w\xa8\xee$o&<q\x8c\x8e(\xa2\x90\x90\x96\xbc\xb1\x1e\xd4QER\x13?\xff"}]
-            ...     status = write_append.write_raw_data(data)
-            >>> status = write_append.commit()
+            ...     write_append.write_raw_data(data)
+            >>> write_append.commit()
         """
         if platform.system().lower() == "windows":
             file_name = file_name.replace("\\", "/")
         check_filename(file_name)
 
+        # decrypt the data file and index file
+        index_file_name = file_name + ".db"
+        decrypt_filename = decrypt(file_name, _get_enc_key(), _get_dec_mode())
+        decrypt_index_filename = decrypt(index_file_name, _get_enc_key(), _get_dec_mode())
+
+        # verify integrity check
+        verify_file_hash(decrypt_filename)
+        verify_file_hash(decrypt_index_filename)
+
+        # move after decrypt and hash check all success
+        if decrypt_filename != file_name:
+            shutil.move(decrypt_filename, file_name)
+            shutil.move(decrypt_index_filename, index_file_name)
+
         # construct ShardHeader
         reader = ShardReader()
         reader.open(file_name, False)
@@ -160,6 +182,7 @@ class FileWriter:
         instance.init_append(file_name, header)
         return instance
 
+    # pylint: disable=missing-docstring
     def init_append(self, file_name, header):
         self._append = True
 
@@ -170,6 +193,7 @@ class FileWriter:
 
         self._header = header
         self._writer.open_for_append(self._file_name)
+        self._paths = [self._file_name]
 
     def add_schema(self, content, desc=None):
         """
@@ -220,9 +244,6 @@ class FileWriter:
             content (dict): Dictionary of schema content.
             desc (str, optional): String of schema description, Default: ``None`` .
 
-        Returns:
-            int, schema id.
-
         Raises:
             MRMInvalidSchemaError: If schema is invalid.
             MRMBuildSchemaError: If failed to build schema.
@@ -238,7 +259,7 @@ class FileWriter:
         if ret is False:
             raise MRMInvalidSchemaError(error_msg)
         schema = self._header.build_schema(content, desc)
-        return self._header.add_schema(schema)
+        self._header.add_schema(schema)
 
     def add_index(self, index_fields):
         """
@@ -255,9 +276,6 @@ class FileWriter:
         Args:
             index_fields (list[str]): fields from schema.
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             ParamTypeError: If index field is invalid.
             MRMDefineIndexError: If index field is not primitive type.
@@ -272,16 +290,7 @@ class FileWriter:
                 raise MRMDefineIndexError("Failed to set field {} since it's not primitive type.".format(field))
             if not isinstance(field, str):
                 raise ParamTypeError('index field', 'str')
-        return self._header.add_index_fields(index_fields)
-
-    def open_and_set_header(self):
-        logger.warning("This interface will be deleted or invisible in the future.")
-
-        if not self._writer.is_open:
-            ret = self._writer.open(self._paths, self._overwrite)
-        if not self._writer.get_shard_header():
-            return self._writer.set_shard_header(self._header)
-        return ret
+        self._header.add_index_fields(index_fields)
 
     def write_raw_data(self, raw_data, parallel_writer=False):
         """
@@ -295,9 +304,6 @@ class FileWriter:
            raw_data (list[dict]): List of raw data.
            parallel_writer (bool, optional): Write raw data in parallel if it equals to True. Default: ``False`` .
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             ParamTypeError: If index field is invalid.
             MRMOpenError: If failed to open MindRecord file.
@@ -329,47 +335,59 @@ class FileWriter:
                 if not isinstance(each_raw, dict):
                     raise ParamTypeError('raw_data item', 'dict')
             self._verify_based_on_schema(raw_data)
-            return self._writer.write_raw_data(raw_data, True, parallel_writer)
+            self._writer.write_raw_data(raw_data, True, parallel_writer)
+            return
 
         ## parallel write mode
         # init the _writers and launch the workers
         if self._writers is None:
             self._writers = [None] * len(self._paths)  # writers used by worker
-            self._queue = mp.Queue(len(self._paths) * 2)  # queue for worker
+            self._queues = [mp.Queue(2) for _ in range(len(self._paths))]  # data queue for worker
+            self._msg_queues = [mp.Queue(2) for _ in range(len(self._paths))]  # msg queue for worker
             self._workers = [None] * len(self._paths)  # worker process
+            worker_pid_list = []
             for i, path in enumerate(self._paths):
                 self._writers[i] = ShardWriter()
                 self._writers[i].open([path], self._overwrite)
                 self._writers[i].set_shard_header(self._header)
 
                 # launch the workers for parallel write
-                self._queue._joincancelled = True  # pylint: disable=W0212
-                p = mp.Process(target=self._write_worker, args=(i, self._queue))
+                self._queues[i]._joincancelled = True  # pylint: disable=W0212
+                self._msg_queues[i]._joincancelled = True  # pylint: disable=W0212
+                p = mp.Process(target=self._write_worker, name="WriterWorker" + str(i),
+                               args=(i, self._queues[i], self._msg_queues[i]))
                 p.daemon = True
                 p.start()
                 logger.info("Start worker process(pid:{}) to parallel write.".format(p.pid))
                 self._workers[i] = p
+                worker_pid_list.append(self._workers[i].pid)
 
-        # fill the self._queue
+        # fill the self._queues
         check_interval = 0.5  # 0.5s
         start_time = time.time()
         while True:
             try:
-                self._queue.put(raw_data, block=False)
+                self._queues[self._workers_loop_index].put(raw_data, block=False)
             except queue.Full:
                 if time.time() - start_time > check_interval:
                     start_time = time.time()
                     logger.warning("Because there are too few MindRecord file shards, the efficiency of parallel " \
                                    "writing is too low. You can stop the current task and add the parameter " \
                                    "`shard_num` of `FileWriter` to upgrade the task.")
-
-                # check the status of worker process
-                for i in range(len(self._paths)):
-                    if not self._workers[i].is_alive():
-                        raise RuntimeError("Worker process(pid:{}) has stopped abnormally. Please check " \
-                                           "the above log".format(self._workers[i].pid))
+                self.check_worker_status(self._workers_loop_index)
                 continue
-            return SUCCESS
+            self._workers_loop_index = (self._workers_loop_index + 1) % len(self._paths)
+            return
+
+    def check_all_worker_status(self):
+        for index in range(len(self._msg_queues)):
+            self.check_worker_status(index)
+
+    def check_worker_status(self, index):
+        if not self._msg_queues[index].empty():
+            if self._msg_queues[index].get() == "Error":
+                raise RuntimeError("Worker process(pid:{}) has stopped abnormally. Please check " \
+                                   "the above log".format(self._workers[index].pid))
 
     def set_header_size(self, header_size):
         """
@@ -379,21 +397,18 @@ class FileWriter:
         the default size (16MB), users need to call the API to set a proper size.
 
         Args:
-            header_size (int): Size of header, between 16*1024(16KB) and
+            header_size (int): Size of header, in bytes, which between 16*1024(16KB) and
                 128*1024*1024(128MB).
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             MRMInvalidHeaderSizeError: If failed to set header size.
 
         Examples:
             >>> from mindspore.mindrecord import FileWriter
             >>> writer = FileWriter(file_name="test.mindrecord", shard_num=1)
-            >>> status = writer.set_header_size(1 << 25) # 32MB
+            >>> writer.set_header_size(1 << 25) # 32MB
         """
-        return self._writer.set_header_size(header_size)
+        self._writer.set_header_size(header_size)
 
     def set_page_size(self, page_size):
         """
@@ -404,21 +419,18 @@ class FileWriter:
         to set a proper size.
 
         Args:
-           page_size (int): Size of page, between 32*1024(32KB) and
+           page_size (int): Size of page, in bytes, which between 32*1024(32KB) and
                256*1024*1024(256MB).
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             MRMInvalidPageSizeError: If failed to set page size.
 
         Examples:
             >>> from mindspore.mindrecord import FileWriter
             >>> writer = FileWriter(file_name="test.mindrecord", shard_num=1)
-            >>> status = writer.set_page_size(1 << 26)  # 64MB
+            >>> writer.set_page_size(1 << 26)  # 64MB
         """
-        return self._writer.set_page_size(page_size)
+        self._writer.set_page_size(page_size)
 
     def commit(self):  # pylint: disable=W0212
         """
@@ -427,9 +439,6 @@ class FileWriter:
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.FileWriter` .
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             MRMOpenError: If failed to open MindRecord file.
             MRMSetHeaderError: If failed to set header.
@@ -464,7 +473,7 @@ class FileWriter:
 
             self._parallel_commit()
 
-        # change the file mode to 600
+        # change file mode first, because encrypt / hash check may failed
         mindrecord_files = []
         index_files = []
         for item in self._paths:
@@ -476,11 +485,21 @@ class FileWriter:
                 os.chmod(index_file, stat.S_IRUSR | stat.S_IWUSR)
                 index_files.append(index_file)
 
+        for item in self._paths:
+            if os.path.exists(item):
+                # add the integrity check string
+                if _get_hash_mode() is not None:
+                    append_hash_to_file(item)
+                    append_hash_to_file(item + ".db")
+
+                # encrypt the mindrecord file
+                if _get_enc_key() is not None:
+                    encrypt(item, _get_enc_key(), _get_enc_mode())
+                    encrypt(item + ".db", _get_enc_key(), _get_enc_mode())
+
         logger.info("The list of mindrecord files created are: {}, and the list of index files are: {}".format(
             mindrecord_files, index_files))
 
-        return SUCCESS
-
     def _index_worker(self, i):
         """The worker do the index generator"""
         generator = ShardIndexGenerator(os.path.realpath(self._paths[i]), False)
@@ -489,55 +508,51 @@ class FileWriter:
 
     def _parallel_commit(self):
         """Parallel commit"""
-        # if some workers stopped, error may occur
-        alive_count = 0
-        for i in range(len(self._paths)):
-            if self._workers[i].is_alive():
-                alive_count += 1
-        if alive_count != len(self._paths):
-            raise RuntimeError("Parallel write worker error, please check the log file.")
+        # waiting for all the self._queues had been writed by workers
+        for index in range(len(self._paths)):
+            while not self._queues[index].empty():
+                logger.info("Waiting for the data is writed by worker: {}.".format(self._workers[index].pid))
+                time.sleep(1)
+
+                # check the workers status
+                self.check_worker_status(index)
 
         # send EOF to worker process
-        for _ in range(len(self._paths)):
+        for index in range(len(self._paths)):
+            logger.info("Send EOF flag to worker: {}.".format(self._workers[index].pid))
             while True:
                 try:
-                    self._queue.put("EOF", block=False)
+                    self._queues[index].put("EOF", block=False)
                 except queue.Full:
+                    # check the workers status
+                    self.check_worker_status(index)
+
                     time.sleep(1)
                     continue
                 break
 
-        # wait the worker processing
-        while True:
-            alive_count = 0
-            for i in range(len(self._paths)):
-                if self._workers[i].is_alive():
-                    alive_count += 1
-            if alive_count == 0:
-                break
-            time.sleep(1)
-            logger.info("Waiting for all the parallel workers to finish.")
-
-        del self._queue
-
-        # wait for worker process stop
-        for index in range(len(self._paths)):
+        worker_sucess = 0
+        for index in range(len(self._msg_queues)):
             while True:
-                logger.info("Waiting for the worker process(pid:{}) to process all the data.".format(
-                    self._workers[index].pid))
-                if self._workers[index].is_alive():
-                    time.sleep(1)
-                    continue
-                elif self._workers[index].exitcode != 0:
-                    raise RuntimeError("Worker process(pid:{}) has stopped abnormally. Please check " \
-                                       "the above log".format(self._workers[index].pid))
-                break
+                logger.info("Waiting for the worker: {} to exit.".format(self._workers[index].pid))
+                if not self._msg_queues[index].empty():
+                    ret = self._msg_queues[index].get()
+                    if ret == "Success":
+                        worker_sucess += 1
+                        break
+                    else:
+                        raise RuntimeError("Worker process(pid:{}) has stopped abnormally. Please check " \
+                                           "the above log".format(self._workers[index].pid))
+                time.sleep(1)
+
+        del self._queues
+        del self._msg_queues
 
         if self._index_generator:
             # use parallel index workers to generator index
             self._index_workers = [None] * len(self._paths)
             for index in range(len(self._paths)):
-                p = mp.Process(target=self._index_worker, args=(index,))
+                p = mp.Process(target=self._index_worker, name="IndexWorker" + str(index), args=(index,))
                 p.daemon = True
                 p.start()
                 logger.info("Start worker process(pid:{}) to generate index.".format(p.pid))
@@ -591,29 +606,38 @@ class FileWriter:
 
                 if field not in v:
                     error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                        "there is not '{}' object in the raw data.".format(i, field)
+                                        "there is not field: '{}' in the raw data.".format(i, field)
                     continue
                 field_type = type(v[field]).__name__
                 if field_type not in VALUE_TYPE_MAP:
                     error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                        "data type for '{}' is not matched.".format(i, field)
+                                        "data type: '{}' for field: '{}' is not matched.".format(i, field_type, field)
                     continue
 
                 if schema_content[field]["type"] not in VALUE_TYPE_MAP[field_type]:
                     error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                        "data type for '{}' is not matched.".format(i, field)
+                                        "data type: '{}' for field: '{}' is not matched." \
+                                        .format(i, schema_content[field]["type"], field)
                     continue
 
                 if field_type == 'ndarray':
                     if 'shape' not in schema_content[field]:
                         error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                            "data type for '{}' is not matched.".format(i, field)
+                                            "data shape for field: '{}' is not specified.".format(i, field)
+                    elif 'type' not in schema_content[field]:
+                        error_data_dic[i] = "for schema, {} th data is wrong, " \
+                                            "data type for field: '{}' is not specified.".format(i, field)
+                    elif schema_content[field]['type'] != str(v[field].dtype):
+                        error_data_dic[i] = "for schema, {} th data is wrong, " \
+                                            "data type: '{}' for field: '{}' is not matched." \
+                                            .format(i, str(v[field].dtype), field)
                     else:
                         try:
                             np.reshape(v[field], schema_content[field]['shape'])
                         except ValueError:
                             error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                                "data type for '{}' is not matched.".format(i, field)
+                                                "data shape: '{}' for field: '{}' is not matched." \
+                                                .format(i, str(v[field].shape), field)
         error_data_dic = sorted(error_data_dic.items(), reverse=True)
         for i, v in error_data_dic:
             raw_data.pop(i)
@@ -660,7 +684,7 @@ class FileWriter:
                 return False, error
         return True, error
 
-    def _write_worker(self, i, in_queue):
+    def _write_worker(self, i, in_queue, msg_queue):
         """The worker do the data check and write to disk for parallel mode"""
         while True:
             # try to get new raw_data from master
@@ -673,15 +697,26 @@ class FileWriter:
             if raw_data == "EOF":
                 ret = self._writers[i].commit()
                 if ret != SUCCESS:
+                    msg_queue.put("Error")
                     raise RuntimeError("Commit the {}th shard of MindRecord file failed.".format(i))
+
+                logger.info("Send Success flag from worker: {} to master: {}.".format(os.getpid(), os.getppid()))
+                msg_queue.put("Success")
+
                 break
 
             # check the raw_data
             if not isinstance(raw_data, list):
+                msg_queue.put("Error")
                 raise ParamTypeError('raw_data', 'list')
             for each_raw in raw_data:
                 if not isinstance(each_raw, dict):
+                    msg_queue.put("Error")
                     raise ParamTypeError('raw_data item', 'dict')
 
-            self._verify_based_on_schema(raw_data)
-            self._writers[i].write_raw_data(raw_data, True, False)
+            try:
+                self._verify_based_on_schema(raw_data)
+                self._writers[i].write_raw_data(raw_data, True, False)
+            except Exception as e:
+                msg_queue.put("Error")
+                raise e

mindspore/mindrecord/mindpage.py

@@ -19,6 +19,7 @@ This module is to support reading page from MindRecord.
 from .shardsegment import ShardSegment
 from .shardutils import check_parameter
 from .common.exceptions import ParamValueError, ParamTypeError, MRMDefineCategoryError
+from .config import _get_enc_key, _get_dec_mode, decrypt, verify_file_hash
 
 __all__ = ['MindPage']
 
@@ -33,8 +34,8 @@ class MindPage:
             It should not be smaller than 1 or larger than the number of processor cores.
 
     Raises:
-        ParamValueError: If `file_name` , `num_consumer` or columns is invalid.
-        MRMInitSegmentError: If failed to initialize ShardSegment.
+        ParamValueError: If `file_name` is not type str or list[str].
+        ParamValueError: If `num_consumer` is not type int.
 
     Examples:
         >>> from mindspore.mindrecord import MindPage
@@ -60,6 +61,31 @@ class MindPage:
     @check_parameter
     def __init__(self, file_name, num_consumer=4):
         self._segment = ShardSegment()
+
+        if isinstance(file_name, str):
+            # decrypt the data file and index file
+            index_file_name = file_name + ".db"
+            decrypt_filename = decrypt(file_name, _get_enc_key(), _get_dec_mode())
+            file_name = decrypt_filename
+            decrypt(index_file_name, _get_enc_key(), _get_dec_mode())
+
+            # verify integrity check
+            verify_file_hash(file_name)
+            verify_file_hash(file_name + ".db")
+        else:
+            file_names_decrypted = []
+            for item in file_name:
+                # decrypt the data file and index file
+                index_file_name = item + ".db"
+                decrypt_filename = decrypt(item, _get_enc_key(), _get_dec_mode())
+                file_names_decrypted.append(decrypt_filename)
+                decrypt(index_file_name, _get_enc_key(), _get_dec_mode())
+
+                # verify integrity check
+                verify_file_hash(decrypt_filename)
+                verify_file_hash(decrypt_filename + ".db")
+            file_name = file_names_decrypted
+
         self._segment.open(file_name, num_consumer)
         self._category_field = None
         self._candidate_fields = [field[:field.rfind('_')] for field in self._segment.get_category_fields()]
@@ -80,7 +106,7 @@ class MindPage:
     @property
     def category_field(self):
         """
-        Getter function for category fields.
+        Setter / Getter function for category fields.
 
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
@@ -97,9 +123,6 @@ class MindPage:
 
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
-
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
         """
         if not category_field or not isinstance(category_field, str):
             raise ParamTypeError('category_fields', 'str')
@@ -112,6 +135,17 @@ class MindPage:
         """
         Return category information when data is grouped by indicated category field.
 
+        The result is similar to the following, where `key` represents the index field and `categories` represent
+        statistical information for the index.
+
+        .. code-block::
+
+            {"categories":[{"count":1,"id":0,"name":"0.jpg"},
+                           {"count":1,"id":1,"name":"1.jpg"},
+                           {"count":1,"id":2,"name":"2.jpg"},
+                           {"count":1,"id":3,"name":"3.jpg"}],
+             "key":"file_name_0"}
+
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
 

mindspore/mindrecord/shardutils.py

@@ -40,6 +40,7 @@ MIN_SHARD_COUNT = ms.MIN_SHARD_COUNT
 MAX_SHARD_COUNT = ms.MAX_SHARD_COUNT
 MIN_CONSUMER_COUNT = ms.MIN_CONSUMER_COUNT
 MAX_CONSUMER_COUNT = ms.get_max_thread_num
+MIN_FILE_SIZE = ms.MIN_FILE_SIZE
 
 VALUE_TYPE_MAP = {"int": ["int32", "int64"], "float": ["float32", "float64"], "str": "string", "bytes": "bytes",
                   "int32": "int32", "int64": "int64", "float32": "float32", "float64": "float64",
@@ -179,6 +180,6 @@ def populate_data(raw, blob, columns, blob_fields, schema):
         else:
             raw[field] = blob_data
 
-    for i, blob_field in enumerate(loaded_columns):
-        _render_raw(blob_field, bytes(blob[i]))
+    for blob_field in loaded_columns:
+        _render_raw(blob_field, bytes(blob[blob_field]))
     return raw

mindspore/mindrecord/shardwriter.py

@@ -167,8 +167,14 @@ class ShardWriter:
         Raises:
             MRMWriteCVError: If failed to write cv type dataset.
         """
+
+        if data == []:
+            raise RuntimeError("There is no valid data which can be written by 'write_raw_data' to mindrecord file. " +
+                               "Please check the abnormal input data indicated in the warning log above.")
+
         blob_data = []
         raw_data = []
+
         # slice data to blob data and raw data
         for item in data:
             row_blob = self._merge_blob({field: item[field] for field in self._header.blob_fields})
@@ -179,6 +185,7 @@ class ShardWriter:
                        for field in self._header.schema.keys() - self._header.blob_fields if field in item}
             if row_raw:
                 raw_data.append(row_raw)
+
         raw_data = {0: raw_data} if raw_data else {}
         ret = self._writer.write_raw_data(raw_data, blob_data, validate, parallel_writer)
         if ret != ms.MSRStatus.SUCCESS: