mindspore 2.2.11__cp37-cp37m-manylinux1_x86_64.whl → 2.3.0rc1__cp37-cp37m-manylinux1_x86_64.whl

mindspore/mindrecord/filereader.py

@@ -23,6 +23,7 @@ from .shardheader import ShardHeader
 from .shardutils import populate_data
 from .shardutils import check_parameter
 from .common.exceptions import ParamTypeError
+from .config import _get_enc_key, _get_dec_mode, decrypt, verify_file_hash
 
 __all__ = ['FileReader']
 
@@ -82,6 +83,30 @@ class FileReader:
         else:
             self._file_name = file_name
 
+        if isinstance(self._file_name, str):
+            # decrypt the data file and index file
+            index_file_name = self._file_name + ".db"
+            decrypt_filename = decrypt(self._file_name, _get_enc_key(), _get_dec_mode())
+            self._file_name = decrypt_filename
+            decrypt(index_file_name, _get_enc_key(), _get_dec_mode())
+
+            # verify integrity check
+            verify_file_hash(self._file_name)
+            verify_file_hash(self._file_name + ".db")
+        else:
+            file_names_decrypted = []
+            for item in self._file_name:
+                # decrypt the data file and index file
+                index_file_name = item + ".db"
+                decrypt_filename = decrypt(item, _get_enc_key(), _get_dec_mode())
+                file_names_decrypted.append(decrypt_filename)
+                decrypt(index_file_name, _get_enc_key(), _get_dec_mode())
+
+                # verify integrity check
+                verify_file_hash(decrypt_filename)
+                verify_file_hash(decrypt_filename + ".db")
+            self._file_name = file_names_decrypted
+
         self._reader.open(self._file_name, num_consumer, columns, operator)
         self._header = ShardHeader(self._reader.get_header())
         self._reader.launch()

mindspore/mindrecord/filewriter.py

@@ -19,6 +19,7 @@ import os
 import platform
 import queue
 import re
+import shutil
 import stat
 import time
 import multiprocessing as mp
@@ -31,6 +32,8 @@ from .shardindexgenerator import ShardIndexGenerator
 from .shardutils import MIN_SHARD_COUNT, MAX_SHARD_COUNT, VALID_ATTRIBUTES, VALID_ARRAY_ATTRIBUTES, \
     check_filename, VALUE_TYPE_MAP, SUCCESS
 from .common.exceptions import ParamValueError, ParamTypeError, MRMInvalidSchemaError, MRMDefineIndexError
+from .config import _get_enc_key, _get_enc_mode, _get_dec_mode, _get_hash_mode, encrypt, decrypt, append_hash_to_file, \
+    verify_file_hash
 
 __all__ = ['FileWriter']
 
@@ -57,14 +60,14 @@ class FileWriter:
         >>>
         >>> writer = FileWriter(file_name="test.mindrecord", shard_num=1, overwrite=True)
         >>> schema_json = {"file_name": {"type": "string"}, "label": {"type": "int32"}, "data": {"type": "bytes"}}
-        >>> schema_id = writer.add_schema(schema_json, "test_schema")
+        >>> writer.add_schema(schema_json, "test_schema")
         >>> indexes = ["file_name", "label"]
-        >>> status = writer.add_index(indexes)
+        >>> writer.add_index(indexes)
         >>> for i in range(10):
         ...     data = [{"file_name": str(i) + ".jpg", "label": i,
         ...              "data": b"\x10c\xb3w\xa8\xee$o&<q\x8c\x8e(\xa2\x90\x90\x96\xbc\xb1\x1e\xd4QER\x13?\xff"}]
-        ...     status = writer.write_raw_data(data)
-        >>> status = writer.commit()
+        ...     writer.write_raw_data(data)
+        >>> writer.commit()
     """
 
     def __init__(self, file_name, shard_num=1, overwrite=False):
@@ -93,6 +96,9 @@ class FileWriter:
         if self._shard_num == 1:
             self._paths = [self._file_name]
         else:
+            if _get_enc_key() is not None or _get_hash_mode() is not None:
+                raise RuntimeError("When encode mode or hash check is enabled, " +
+                                   "the automatic sharding function is unavailable.")
             self._paths = ["{}{}".format(self._file_name,
                                          str(x).rjust(suffix_shard_size, '0'))
                            for x in range(self._shard_num)]
@@ -135,21 +141,35 @@ class FileWriter:
             ...          "data": b"\x10c\xb3w\xa8\xee$o&<q\x8c\x8e(\xa2\x90\x90\x96\xbc\xb1\x1e\xd4QER\x13?\xff"}]
             >>> writer = FileWriter(file_name="test.mindrecord", shard_num=1, overwrite=True)
             >>> schema_json = {"file_name": {"type": "string"}, "label": {"type": "int32"}, "data": {"type": "bytes"}}
-            >>> schema_id = writer.add_schema(schema_json, "test_schema")
-            >>> status = writer.write_raw_data(data)
-            >>> status = writer.commit()
+            >>> writer.add_schema(schema_json, "test_schema")
+            >>> writer.write_raw_data(data)
+            >>> writer.commit()
             >>>
             >>> write_append = FileWriter.open_for_append("test.mindrecord")
             >>> for i in range(9):
             ...     data = [{"file_name": str(i+1) + ".jpg", "label": i,
             ...              "data": b"\x10c\xb3w\xa8\xee$o&<q\x8c\x8e(\xa2\x90\x90\x96\xbc\xb1\x1e\xd4QER\x13?\xff"}]
-            ...     status = write_append.write_raw_data(data)
-            >>> status = write_append.commit()
+            ...     write_append.write_raw_data(data)
+            >>> write_append.commit()
         """
         if platform.system().lower() == "windows":
             file_name = file_name.replace("\\", "/")
         check_filename(file_name)
 
+        # decrypt the data file and index file
+        index_file_name = file_name + ".db"
+        decrypt_filename = decrypt(file_name, _get_enc_key(), _get_dec_mode())
+        decrypt_index_filename = decrypt(index_file_name, _get_enc_key(), _get_dec_mode())
+
+        # verify integrity check
+        verify_file_hash(decrypt_filename)
+        verify_file_hash(decrypt_index_filename)
+
+        # move after decrypt and hash check all success
+        if decrypt_filename != file_name:
+            shutil.move(decrypt_filename, file_name)
+            shutil.move(decrypt_index_filename, index_file_name)
+
         # construct ShardHeader
         reader = ShardReader()
         reader.open(file_name, False)
@@ -160,6 +180,7 @@ class FileWriter:
         instance.init_append(file_name, header)
         return instance
 
+    # pylint: disable=missing-docstring
     def init_append(self, file_name, header):
         self._append = True
 
@@ -170,6 +191,7 @@ class FileWriter:
 
         self._header = header
         self._writer.open_for_append(self._file_name)
+        self._paths = [self._file_name]
 
     def add_schema(self, content, desc=None):
         """
@@ -220,9 +242,6 @@ class FileWriter:
             content (dict): Dictionary of schema content.
             desc (str, optional): String of schema description, Default: ``None`` .
 
-        Returns:
-            int, schema id.
-
         Raises:
             MRMInvalidSchemaError: If schema is invalid.
             MRMBuildSchemaError: If failed to build schema.
@@ -238,7 +257,7 @@ class FileWriter:
         if ret is False:
             raise MRMInvalidSchemaError(error_msg)
         schema = self._header.build_schema(content, desc)
-        return self._header.add_schema(schema)
+        self._header.add_schema(schema)
 
     def add_index(self, index_fields):
         """
@@ -255,9 +274,6 @@ class FileWriter:
         Args:
             index_fields (list[str]): fields from schema.
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             ParamTypeError: If index field is invalid.
             MRMDefineIndexError: If index field is not primitive type.
@@ -272,16 +288,7 @@ class FileWriter:
                 raise MRMDefineIndexError("Failed to set field {} since it's not primitive type.".format(field))
             if not isinstance(field, str):
                 raise ParamTypeError('index field', 'str')
-        return self._header.add_index_fields(index_fields)
-
-    def open_and_set_header(self):
-        logger.warning("This interface will be deleted or invisible in the future.")
-
-        if not self._writer.is_open:
-            ret = self._writer.open(self._paths, self._overwrite)
-        if not self._writer.get_shard_header():
-            return self._writer.set_shard_header(self._header)
-        return ret
+        self._header.add_index_fields(index_fields)
 
     def write_raw_data(self, raw_data, parallel_writer=False):
         """
@@ -295,9 +302,6 @@ class FileWriter:
            raw_data (list[dict]): List of raw data.
            parallel_writer (bool, optional): Write raw data in parallel if it equals to True. Default: ``False`` .
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             ParamTypeError: If index field is invalid.
             MRMOpenError: If failed to open MindRecord file.
@@ -329,7 +333,8 @@ class FileWriter:
                 if not isinstance(each_raw, dict):
                     raise ParamTypeError('raw_data item', 'dict')
             self._verify_based_on_schema(raw_data)
-            return self._writer.write_raw_data(raw_data, True, parallel_writer)
+            self._writer.write_raw_data(raw_data, True, parallel_writer)
+            return
 
         ## parallel write mode
         # init the _writers and launch the workers
@@ -369,7 +374,7 @@ class FileWriter:
                         raise RuntimeError("Worker process(pid:{}) has stopped abnormally. Please check " \
                                            "the above log".format(self._workers[i].pid))
                 continue
-            return SUCCESS
+            return
 
     def set_header_size(self, header_size):
         """
@@ -379,21 +384,18 @@ class FileWriter:
         the default size (16MB), users need to call the API to set a proper size.
 
         Args:
-            header_size (int): Size of header, between 16*1024(16KB) and
+            header_size (int): Size of header, in bytes, which between 16*1024(16KB) and
                 128*1024*1024(128MB).
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             MRMInvalidHeaderSizeError: If failed to set header size.
 
         Examples:
             >>> from mindspore.mindrecord import FileWriter
             >>> writer = FileWriter(file_name="test.mindrecord", shard_num=1)
-            >>> status = writer.set_header_size(1 << 25) # 32MB
+            >>> writer.set_header_size(1 << 25) # 32MB
         """
-        return self._writer.set_header_size(header_size)
+        self._writer.set_header_size(header_size)
 
     def set_page_size(self, page_size):
         """
@@ -404,21 +406,18 @@ class FileWriter:
         to set a proper size.
 
         Args:
-           page_size (int): Size of page, between 32*1024(32KB) and
+           page_size (int): Size of page, in bytes, which between 32*1024(32KB) and
                256*1024*1024(256MB).
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             MRMInvalidPageSizeError: If failed to set page size.
 
         Examples:
             >>> from mindspore.mindrecord import FileWriter
             >>> writer = FileWriter(file_name="test.mindrecord", shard_num=1)
-            >>> status = writer.set_page_size(1 << 26)  # 64MB
+            >>> writer.set_page_size(1 << 26)  # 64MB
         """
-        return self._writer.set_page_size(page_size)
+        self._writer.set_page_size(page_size)
 
     def commit(self):  # pylint: disable=W0212
         """
@@ -427,9 +426,6 @@ class FileWriter:
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.FileWriter` .
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             MRMOpenError: If failed to open MindRecord file.
             MRMSetHeaderError: If failed to set header.
@@ -464,7 +460,7 @@ class FileWriter:
 
             self._parallel_commit()
 
-        # change the file mode to 600
+        # change file mode first, because encrypt / hash check may failed
         mindrecord_files = []
         index_files = []
         for item in self._paths:
@@ -476,11 +472,21 @@ class FileWriter:
                 os.chmod(index_file, stat.S_IRUSR | stat.S_IWUSR)
                 index_files.append(index_file)
 
+        for item in self._paths:
+            if os.path.exists(item):
+                # add the integrity check string
+                if _get_hash_mode() is not None:
+                    append_hash_to_file(item)
+                    append_hash_to_file(item + ".db")
+
+                # encrypt the mindrecord file
+                if _get_enc_key() is not None:
+                    encrypt(item, _get_enc_key(), _get_enc_mode())
+                    encrypt(item + ".db", _get_enc_key(), _get_enc_mode())
+
         logger.info("The list of mindrecord files created are: {}, and the list of index files are: {}".format(
             mindrecord_files, index_files))
 
-        return SUCCESS
-
     def _index_worker(self, i):
         """The worker do the index generator"""
         generator = ShardIndexGenerator(os.path.realpath(self._paths[i]), False)
@@ -495,15 +501,18 @@ class FileWriter:
             if self._workers[i].is_alive():
                 alive_count += 1
         if alive_count != len(self._paths):
-            raise RuntimeError("Parallel write worker error, please check the log file.")
+            raise RuntimeError("Parallel write worker error, please check the above log.")
 
         # send EOF to worker process
-        for _ in range(len(self._paths)):
+        for i in range(len(self._paths)):
             while True:
                 try:
                     self._queue.put("EOF", block=False)
                 except queue.Full:
                     time.sleep(1)
+                    if not self._workers[i].is_alive():
+                        raise RuntimeError("Worker process(pid:{}) has stopped abnormally. Please check " \
+                                           "the above log".format(self._workers[i].pid))
                     continue
                 break
 
@@ -591,29 +600,38 @@ class FileWriter:
 
                 if field not in v:
                     error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                        "there is not '{}' object in the raw data.".format(i, field)
+                                        "there is not field: '{}' in the raw data.".format(i, field)
                     continue
                 field_type = type(v[field]).__name__
                 if field_type not in VALUE_TYPE_MAP:
                     error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                        "data type for '{}' is not matched.".format(i, field)
+                                        "data type: '{}' for field: '{}' is not matched.".format(i, field_type, field)
                     continue
 
                 if schema_content[field]["type"] not in VALUE_TYPE_MAP[field_type]:
                     error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                        "data type for '{}' is not matched.".format(i, field)
+                                        "data type: '{}' for field: '{}' is not matched." \
+                                        .format(i, schema_content[field]["type"], field)
                     continue
 
                 if field_type == 'ndarray':
                     if 'shape' not in schema_content[field]:
                         error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                            "data type for '{}' is not matched.".format(i, field)
+                                            "data shape for field: '{}' is not specified.".format(i, field)
+                    elif 'type' not in schema_content[field]:
+                        error_data_dic[i] = "for schema, {} th data is wrong, " \
+                                            "data type for field: '{}' is not specified.".format(i, field)
+                    elif schema_content[field]['type'] != str(v[field].dtype):
+                        error_data_dic[i] = "for schema, {} th data is wrong, " \
+                                            "data type: '{}' for field: '{}' is not matched." \
+                                            .format(i, str(v[field].dtype), field)
                     else:
                         try:
                             np.reshape(v[field], schema_content[field]['shape'])
                         except ValueError:
                             error_data_dic[i] = "for schema, {} th data is wrong, " \
-                                                "data type for '{}' is not matched.".format(i, field)
+                                                "data shape: '{}' for field: '{}' is not matched." \
+                                                .format(i, str(v[field].shape), field)
         error_data_dic = sorted(error_data_dic.items(), reverse=True)
         for i, v in error_data_dic:
             raw_data.pop(i)

mindspore/mindrecord/mindpage.py

@@ -19,6 +19,7 @@ This module is to support reading page from MindRecord.
 from .shardsegment import ShardSegment
 from .shardutils import check_parameter
 from .common.exceptions import ParamValueError, ParamTypeError, MRMDefineCategoryError
+from .config import _get_enc_key, _get_dec_mode, decrypt, verify_file_hash
 
 __all__ = ['MindPage']
 
@@ -33,8 +34,8 @@ class MindPage:
             It should not be smaller than 1 or larger than the number of processor cores.
 
     Raises:
-        ParamValueError: If `file_name` , `num_consumer` or columns is invalid.
-        MRMInitSegmentError: If failed to initialize ShardSegment.
+        ParamValueError: If `file_name` is not type str or list[str].
+        ParamValueError: If `num_consumer` is not type int.
 
     Examples:
         >>> from mindspore.mindrecord import MindPage
@@ -60,6 +61,31 @@ class MindPage:
     @check_parameter
     def __init__(self, file_name, num_consumer=4):
         self._segment = ShardSegment()
+
+        if isinstance(file_name, str):
+            # decrypt the data file and index file
+            index_file_name = file_name + ".db"
+            decrypt_filename = decrypt(file_name, _get_enc_key(), _get_dec_mode())
+            file_name = decrypt_filename
+            decrypt(index_file_name, _get_enc_key(), _get_dec_mode())
+
+            # verify integrity check
+            verify_file_hash(file_name)
+            verify_file_hash(file_name + ".db")
+        else:
+            file_names_decrypted = []
+            for item in file_name:
+                # decrypt the data file and index file
+                index_file_name = item + ".db"
+                decrypt_filename = decrypt(item, _get_enc_key(), _get_dec_mode())
+                file_names_decrypted.append(decrypt_filename)
+                decrypt(index_file_name, _get_enc_key(), _get_dec_mode())
+
+                # verify integrity check
+                verify_file_hash(decrypt_filename)
+                verify_file_hash(decrypt_filename + ".db")
+            file_name = file_names_decrypted
+
         self._segment.open(file_name, num_consumer)
         self._category_field = None
         self._candidate_fields = [field[:field.rfind('_')] for field in self._segment.get_category_fields()]
@@ -80,7 +106,7 @@ class MindPage:
     @property
     def category_field(self):
         """
-        Getter function for category fields.
+        Setter / Getter function for category fields.
 
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
@@ -97,9 +123,6 @@ class MindPage:
 
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
-
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
         """
         if not category_field or not isinstance(category_field, str):
             raise ParamTypeError('category_fields', 'str')
@@ -112,6 +135,17 @@ class MindPage:
         """
         Return category information when data is grouped by indicated category field.
 
+        The result is similar to the following, where `key` represents the index field and `categories` represent
+        statistical information for the index.
+
+        .. code-block::
+
+            {"categories":[{"count":1,"id":0,"name":"0.jpg"},
+                           {"count":1,"id":1,"name":"1.jpg"},
+                           {"count":1,"id":2,"name":"2.jpg"},
+                           {"count":1,"id":3,"name":"3.jpg"}],
+             "key":"file_name_0"}
+
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
 

mindspore/mindrecord/shardutils.py

@@ -40,6 +40,7 @@ MIN_SHARD_COUNT = ms.MIN_SHARD_COUNT
 MAX_SHARD_COUNT = ms.MAX_SHARD_COUNT
 MIN_CONSUMER_COUNT = ms.MIN_CONSUMER_COUNT
 MAX_CONSUMER_COUNT = ms.get_max_thread_num
+MIN_FILE_SIZE = ms.MIN_FILE_SIZE
 
 VALUE_TYPE_MAP = {"int": ["int32", "int64"], "float": ["float32", "float64"], "str": "string", "bytes": "bytes",
                   "int32": "int32", "int64": "int64", "float32": "float32", "float64": "float64",
@@ -179,6 +180,6 @@ def populate_data(raw, blob, columns, blob_fields, schema):
         else:
             raw[field] = blob_data
 
-    for i, blob_field in enumerate(loaded_columns):
-        _render_raw(blob_field, bytes(blob[i]))
+    for blob_field in loaded_columns:
+        _render_raw(blob_field, bytes(blob[blob_field]))
     return raw

mindspore/mindrecord/shardwriter.py

@@ -167,8 +167,14 @@ class ShardWriter:
         Raises:
             MRMWriteCVError: If failed to write cv type dataset.
         """
+
+        if data == []:
+            raise RuntimeError("There is no valid data which can be written by 'write_raw_data' to mindrecord file. " +
+                               "Please check the abnormal input data indicated in the warning log above.")
+
         blob_data = []
         raw_data = []
+
         # slice data to blob data and raw data
         for item in data:
             row_blob = self._merge_blob({field: item[field] for field in self._header.blob_fields})
@@ -179,6 +185,7 @@ class ShardWriter:
                        for field in self._header.schema.keys() - self._header.blob_fields if field in item}
             if row_raw:
                 raw_data.append(row_raw)
+
         raw_data = {0: raw_data} if raw_data else {}
         ret = self._writer.write_raw_data(raw_data, blob_data, validate, parallel_writer)
         if ret != ms.MSRStatus.SUCCESS:

mindspore/mindrecord/tools/cifar100_to_mr.py

@@ -18,18 +18,13 @@ Cifar100 convert tool for MindRecord.
 
 from importlib import import_module
 import os
-import numpy as np
 
 from mindspore import log as logger
 from .cifar100 import Cifar100
 from ..common.exceptions import PathNotExistsError
 from ..filewriter import FileWriter
-from ..shardutils import check_filename, ExceptionThread, SUCCESS, FAILED
+from ..shardutils import check_filename, ExceptionThread, SUCCESS
 
-try:
-    cv_import = import_module("cv2")
-except ModuleNotFoundError:
-    cv_import = None
 
 __all__ = ['Cifar100ToMR']
 
@@ -53,10 +48,12 @@ class Cifar100ToMR:
         >>> cifar100_dir = "/path/to/cifar100"
         >>> mindrecord_file = "/path/to/mindrecord/file"
         >>> cifar100_to_mr = Cifar100ToMR(cifar100_dir, mindrecord_file)
-        >>> status = cifar100_to_mr.transform()
+        >>> cifar100_to_mr.transform()
     """
 
     def __init__(self, source, destination):
+        self.cv_import = import_module("cv2")
+
         check_filename(source)
         self.source = source
 
@@ -100,13 +97,11 @@ class Cifar100ToMR:
         test_coarse_labels = cifar100_data.Test.coarse_labels
         logger.info("test images coarse label: {}".format(coarse_labels.shape))
 
-        data_list = _construct_raw_data(images, fine_labels, coarse_labels)
-        test_data_list = _construct_raw_data(test_images, test_fine_labels, test_coarse_labels)
+        data_list = self._construct_raw_data(images, fine_labels, coarse_labels)
+        test_data_list = self._construct_raw_data(test_images, test_fine_labels, test_coarse_labels)
 
-        if _generate_mindrecord(self.destination, data_list, fields, "img_train") != SUCCESS:
-            return FAILED
-        if _generate_mindrecord(self.destination + "_test", test_data_list, fields, "img_test") != SUCCESS:
-            return FAILED
+        self._generate_mindrecord(self.destination, data_list, fields, "img_train")
+        self._generate_mindrecord(self.destination + "_test", test_data_list, fields, "img_test")
         return SUCCESS
 
     def transform(self, fields=None):
@@ -121,16 +116,12 @@ class Cifar100ToMR:
                 A list of index field, e.g.["fine_label", "coarse_label"]. Default: ``None`` . For index
                 field settings, please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             ParamTypeError: If index field is invalid.
             MRMOpenError: If failed to open MindRecord file.
             MRMValidateDataError: If data does not match blob fields.
             MRMSetHeaderError: If failed to set header.
             MRMWriteDatasetError: If failed to write dataset.
-            TypeError: If `parallel_writer` is not bool.
             ValueError: If parameter `fields` is invalid.
         """
 
@@ -140,59 +131,55 @@ class Cifar100ToMR:
         t.join()
         if t.exitcode != 0:
             raise t.exception
-        return t.res
 
 
-def _construct_raw_data(images, fine_labels, coarse_labels):
-    """
-    Construct raw data from cifar100 data.
+    def _construct_raw_data(self, images, fine_labels, coarse_labels):
+        """
+        Construct raw data from cifar100 data.
 
-    Args:
-        images (list): image list from cifar100.
-        fine_labels (list): fine label list from cifar100.
-        coarse_labels (list): coarse label list from cifar100.
+        Args:
+            images (list): image list from cifar100.
+            fine_labels (list): fine label list from cifar100.
+            coarse_labels (list): coarse label list from cifar100.
 
-    Returns:
-        list[dict], data dictionary constructed from cifar100.
-    """
-    if not cv_import:
-        raise ModuleNotFoundError("opencv-python module not found, please use pip install it.")
-
-    raw_data = []
-    for i, img in enumerate(images):
-        fine_label = np.int(fine_labels[i][0])
-        coarse_label = np.int(coarse_labels[i][0])
-        _, img = cv_import.imencode(".jpeg", img[..., [2, 1, 0]])
-        row_data = {"id": int(i),
-                    "data": img.tobytes(),
-                    "fine_label": int(fine_label),
-                    "coarse_label": int(coarse_label)}
-        raw_data.append(row_data)
-    return raw_data
-
-
-def _generate_mindrecord(file_name, raw_data, fields, schema_desc):
-    """
-    Generate MindRecord file from raw data.
+        Returns:
+            list[dict], data dictionary constructed from cifar100.
+        """
+        raw_data = []
+        for i, img in enumerate(images):
+            fine_label = fine_labels[i][0]
+            coarse_label = coarse_labels[i][0]
+            _, img = self.cv_import.imencode(".jpeg", img[..., [2, 1, 0]])
+            row_data = {"id": int(i),
+                        "data": img.tobytes(),
+                        "fine_label": int(fine_label),
+                        "coarse_label": int(coarse_label)}
+            raw_data.append(row_data)
+        return raw_data
+
+
+    def _generate_mindrecord(self, file_name, raw_data, fields, schema_desc):
+        """
+        Generate MindRecord file from raw data.
 
-    Args:
-        file_name (str): File name of MindRecord File.
-        fields (list[str]): Fields would be set as index which
-          could not belong to blob fields and type could not be 'array' or 'bytes'.
-        raw_data (dict): Dict of raw data.
-        schema_desc (str): String of schema description.
-
-    Returns:
-        MSRStatus, SUCCESS or FAILED.
-    """
-    schema = {"id": {"type": "int64"}, "fine_label": {"type": "int64"},
-              "coarse_label": {"type": "int64"}, "data": {"type": "bytes"}}
+        Args:
+            file_name (str): File name of MindRecord File.
+            fields (list[str]): Fields would be set as index which
+            could not belong to blob fields and type could not be 'array' or 'bytes'.
+            raw_data (dict): Dict of raw data.
+            schema_desc (str): String of schema description.
+
+        Returns:
+            SUCCESS or FAILED.
+        """
+        schema = {"id": {"type": "int64"}, "fine_label": {"type": "int64"},
+                  "coarse_label": {"type": "int64"}, "data": {"type": "bytes"}}
 
-    logger.info("transformed MindRecord schema is: {}".format(schema))
+        logger.info("transformed MindRecord schema is: {}".format(schema))
 
-    writer = FileWriter(file_name, 1)
-    writer.add_schema(schema, schema_desc)
-    if fields and isinstance(fields, list):
-        writer.add_index(fields)
-    writer.write_raw_data(raw_data)
-    return writer.commit()
+        writer = FileWriter(file_name, 1)
+        writer.add_schema(schema, schema_desc)
+        if fields and isinstance(fields, list):
+            writer.add_index(fields)
+        writer.write_raw_data(raw_data)
+        writer.commit()