mindspore 1.10.0__cp37-none-any.whl → 2.0.0rc1__cp37-none-any.whl

mindspore/mindrecord/mindpage.py

@@ -34,7 +34,7 @@ class MindPage:
             It should not be smaller than 1 or larger than the number of processor cores.
 
     Raises:
-        ParamValueError: If `file_name`, `num_consumer` or columns is invalid.
+        ParamValueError: If `file_name` , `num_consumer` or columns is invalid.
         MRMInitSegmentError: If failed to initialize ShardSegment.
     """
     @check_parameter
@@ -54,6 +54,16 @@ class MindPage:
         """
         return self._candidate_fields
 
+    @property
+    def category_field(self):
+        """
+        Getter function for category fields.
+
+        Returns:
+            list[str], by which data could be grouped.
+        """
+        return self._category_field
+
     def get_category_fields(self):
         """
         Return candidate category fields.
@@ -86,16 +96,6 @@ class MindPage:
             raise MRMDefineCategoryError("Field '{}' is not a candidate category field.".format(category_field))
         return self._segment.set_category_field(category_field)
 
-    @property
-    def category_field(self):
-        """
-        Getter function for category fields.
-
-        Returns:
-            list[str], by which data could be grouped.
-        """
-        return self._category_field
-
     @category_field.setter
     def category_field(self, category_field):
         """
@@ -128,7 +128,7 @@ class MindPage:
         Query by category id in pagination.
 
         Args:
-             category_id (int): Category id, referred to the return of `read_category_info`.
+             category_id (int): Category id, referred to the return of `read_category_info` .
              page (int): Index of page.
              num_row (int): Number of rows in a page.
 
@@ -154,7 +154,7 @@ class MindPage:
 
         Args:
             category_name (str): String of category field's value,
-                referred to the return of `read_category_info`.
+                referred to the return of `read_category_info` .
             page (int): Index of page.
             num_row (int): Number of row in a page.
 

mindspore/mindrecord/shardheader.py

@@ -34,6 +34,21 @@ class ShardHeader:
         else:
             self._header = ms.ShardHeader()
 
+    @property
+    def header(self):
+        """Getter of header"""
+        return self._header
+
+    @property
+    def blob_fields(self):
+        """Getter of blob fields"""
+        return self._get_blob_fields()
+
+    @property
+    def schema(self):
+        """Getter of schema"""
+        return self._get_schema()
+
     def add_schema(self, schema):
         """
         Add object of ShardSchema.
@@ -93,11 +108,6 @@ class ShardHeader:
             raise MRMBuildSchemaError
         return schema
 
-    @property
-    def header(self):
-        """Getter of header"""
-        return self._header
-
     def _get_schema(self):
         """
         Get schema info.
@@ -129,13 +139,3 @@ class ShardHeader:
 
         logger.critical("Failed to get meta info.")
         raise MRMGetMetaError
-
-    @property
-    def blob_fields(self):
-        """Getter of blob fields"""
-        return self._get_blob_fields()
-
-    @property
-    def schema(self):
-        """Getter of schema"""
-        return self._get_schema()

mindspore/mindrecord/shardreader.py

@@ -106,3 +106,12 @@ class ShardReader:
     def close(self):
         """close MindRecord File."""
         self._reader.close()
+
+    def len(self):
+        """
+        Get the number of the samples in MindRecord.
+
+        Returns:
+            int, the number of the samples in MindRecord.
+        """
+        return self._reader.len()

mindspore/mindrecord/shardwriter.py

@@ -36,6 +36,18 @@ class ShardWriter:
         self._header = None
         self._is_open = False
 
+    @property
+    def is_open(self):
+        """getter function"""
+        return self._is_open
+
+    @staticmethod
+    def convert_np_types(val):
+        """convert numpy type to python primitive type"""
+        if isinstance(val, (np.int32, np.int64, np.float32, np.float64)):
+            return val.item()
+        return val
+
     def open(self, paths, override):
         """
         Open a new MindRecord File and prepare to write raw data.
@@ -137,13 +149,6 @@ class ShardWriter:
     def get_shard_header(self):
         return self._header
 
-    @staticmethod
-    def convert_np_types(val):
-        """convert numpy type to python primitive type"""
-        if isinstance(val, (np.int32, np.int64, np.float32, np.float64)):
-            return val.item()
-        return val
-
     def write_raw_data(self, data, validate=True, parallel_writer=False):
         """
         Write raw data of cv dataset.
@@ -168,7 +173,7 @@ class ShardWriter:
         for item in data:
             row_blob = self._merge_blob({field: item[field] for field in self._header.blob_fields})
             if row_blob:
-                blob_data.append(list(row_blob))
+                blob_data.append(row_blob)
             # filter raw data according to schema
             row_raw = {field: self.convert_np_types(item[field])
                        for field in self._header.schema.keys() - self._header.blob_fields if field in item}
@@ -181,6 +186,22 @@ class ShardWriter:
             raise MRMWriteDatasetError
         return ret
 
+    def commit(self):
+        """
+        Flush data to disk.
+
+        Returns:
+            MSRStatus, SUCCESS or FAILED.
+
+        Raises:
+            MRMCommitError: If failed to flush data to disk.
+        """
+        ret = self._writer.commit()
+        if ret != ms.MSRStatus.SUCCESS:
+            logger.critical("Failed to commit.")
+            raise MRMCommitError
+        return ret
+
     def _merge_blob(self, blob_data):
         """
         Merge multiple blob data whose type is bytes or ndarray
@@ -207,24 +228,3 @@ class ShardWriter:
             merged += int_to_bytes(len(v))
             merged += v
         return merged
-
-    def commit(self):
-        """
-        Flush data to disk.
-
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
-        Raises:
-            MRMCommitError: If failed to flush data to disk.
-        """
-        ret = self._writer.commit()
-        if ret != ms.MSRStatus.SUCCESS:
-            logger.critical("Failed to commit.")
-            raise MRMCommitError
-        return ret
-
-    @property
-    def is_open(self):
-        """getter function"""
-        return self._is_open

mindspore/mindrecord/tools/cifar100_to_mr.py

@@ -27,9 +27,9 @@ from ..filewriter import FileWriter
 from ..shardutils import check_filename, ExceptionThread, SUCCESS, FAILED
 
 try:
-    cv2 = import_module("cv2")
+    cv_import = import_module("cv2")
 except ModuleNotFoundError:
-    cv2 = None
+    cv_import = None
 
 __all__ = ['Cifar100ToMR']
 
@@ -40,12 +40,12 @@ class Cifar100ToMR:
 
     Note:
         For details about Examples, please refer to `Converting the CIFAR-10 Dataset <https://
-        www.mindspore.cn/tutorials/en/r1.10/advanced/dataset/record.html#converting-the-cifar-10-dataset>`_.
+        www.mindspore.cn/tutorials/en/r2.0/advanced/dataset/record.html#converting-the-cifar-10-dataset>`_ .
 
     Args:
         source (str): The cifar100 directory to be transformed.
-        destination (str): MindRecord file path to transform into, ensure that no file with the same name
-            exists in the directory.
+        destination (str): MindRecord file path to transform into, ensure that the directory is created in advance and
+            no file with the same name exists in the directory.
 
     Raises:
         ValueError: If source or destination is invalid.
@@ -80,7 +80,7 @@ class Cifar100ToMR:
         Args:
             fields (list[str], optional):
                 A list of index field, e.g.["fine_label", "coarse_label"]. Default: None. For index
-                field settings, please refer to :func:`mindspore.mindrecord.FileWriter.add_index`.
+                field settings, please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
@@ -121,7 +121,7 @@ class Cifar100ToMR:
         Args:
             fields (list[str], optional):
                 A list of index field, e.g.["fine_label", "coarse_label"]. Default: None. For index
-                field settings, please refer to :func:`mindspore.mindrecord.FileWriter.add_index`.
+                field settings, please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
@@ -148,14 +148,14 @@ def _construct_raw_data(images, fine_labels, coarse_labels):
     Returns:
         list[dict], data dictionary constructed from cifar100.
     """
-    if not cv2:
+    if not cv_import:
         raise ModuleNotFoundError("opencv-python module not found, please use pip install it.")
 
     raw_data = []
     for i, img in enumerate(images):
         fine_label = np.int(fine_labels[i][0])
         coarse_label = np.int(coarse_labels[i][0])
-        _, img = cv2.imencode(".jpeg", img[..., [2, 1, 0]])
+        _, img = cv_import.imencode(".jpeg", img[..., [2, 1, 0]])
         row_data = {"id": int(i),
                     "data": img.tobytes(),
                     "fine_label": int(fine_label),

mindspore/mindrecord/tools/cifar10_to_mr.py

@@ -27,9 +27,9 @@ from ..filewriter import FileWriter
 from ..shardutils import check_filename, ExceptionThread, SUCCESS, FAILED
 
 try:
-    cv2 = import_module("cv2")
+    cv_import = import_module("cv2")
 except ModuleNotFoundError:
-    cv2 = None
+    cv_import = None
 
 __all__ = ['Cifar10ToMR']
 
@@ -40,12 +40,12 @@ class Cifar10ToMR:
 
     Note:
         For details about Examples, please refer to `Converting the CIFAR-10 Dataset <https://
-        www.mindspore.cn/tutorials/en/r1.10/advanced/dataset/record.html#converting-the-cifar-10-dataset>`_.
+        www.mindspore.cn/tutorials/en/r2.0/advanced/dataset/record.html#converting-the-cifar-10-dataset>`_ .
 
     Args:
         source (str): The cifar10 directory to be transformed.
-        destination (str): MindRecord file path to transform into, ensure that no file with the same name
-            exists in the directory.
+        destination (str): MindRecord file path to transform into, ensure that the directory is created in advance and
+            no file with the same name exists in the directory.
 
     Raises:
         ValueError: If source or destination is invalid.
@@ -79,7 +79,7 @@ class Cifar10ToMR:
 
         Args:
             fields (list[str], optional): A list of index fields. Default: None. For index field settings,
-                please refer to :func:`mindspore.mindrecord.FileWriter.add_index`.
+                please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
@@ -116,7 +116,7 @@ class Cifar10ToMR:
 
         Args:
             fields (list[str], optional): A list of index fields. Default: None. For index field settings,
-                please refer to :func:`mindspore.mindrecord.FileWriter.add_index`.
+                please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
@@ -143,13 +143,13 @@ def _construct_raw_data(images, labels):
         list[dict], data dictionary constructed from cifar10.
     """
 
-    if not cv2:
+    if not cv_import:
         raise ModuleNotFoundError("opencv-python module not found, please use pip install it.")
 
     raw_data = []
     for i, img in enumerate(images):
         label = np.int(labels[i][0])
-        _, img = cv2.imencode(".jpeg", img[..., [2, 1, 0]])
+        _, img = cv_import.imencode(".jpeg", img[..., [2, 1, 0]])
         row_data = {"id": int(i),
                     "data": img.tobytes(),
                     "label": int(label)}

mindspore/mindrecord/tools/csv_to_mr.py

@@ -36,17 +36,17 @@ class CsvToMR:
 
     Note:
         For details about Examples, please refer to `Converting CSV Dataset <https://
-        www.mindspore.cn/tutorials/en/r1.10/advanced/dataset/record.html#converting-csv-dataset>`_.
+        www.mindspore.cn/tutorials/en/r2.0/advanced/dataset/record.html#converting-csv-dataset>`_ .
 
     Args:
         source (str): The file path of csv.
-        destination (str): The MindRecord file path to transform into, ensure that no file with the same name
-            exists in the directory.
+        destination (str): The MindRecord file path to transform into, ensure that the directory is created in advance
+            and no file with the same name exists in the directory.
         columns_list(list[str], optional): A list of columns to be read. Default: None.
         partition_number (int, optional): The partition size, Default: 1.
 
     Raises:
-        ValueError: If `source`, `destination`, `partition_number` is invalid.
+        ValueError: If `source` , `destination` , `partition_number` is invalid.
         RuntimeError: If `columns_list` is invalid.
     """
 

mindspore/mindrecord/tools/imagenet_to_mr.py

@@ -27,26 +27,31 @@ __all__ = ['ImageNetToMR']
 
 
 class ImageNetToMR:
-    """
+    r"""
     A class to transform from imagenet to MindRecord.
 
     Args:
-        map_file (str): The map file that indicates label. The map file content should be like this:
+        map_file (str): The map file that indicates label. This file can be generated by command
+            :code:`ls -l [image_dir] | grep -vE "total|\." | awk -F " " '{print $9, NR-1;}' > [file_path]` ,
+            where `image_dir` is image directory contains n01440764, n01443537, n01484850 and n15075141 directory
+            and `file_path` is the generated `map_file` . An example of `map_file` is as below:
 
             .. code-block::
 
-              n02119789 0
-              n02100735 1
-              n02110185 2
-              n02096294 3
+              n01440764 0
+              n01443537 1
+              n01484850 2
+              n01491361 3
+              ...
+              n15075141 999
 
-        image_dir (str): Image directory contains n02119789, n02100735, n02110185 and n02096294 directory.
-        destination (str): MindRecord file path to transform into, ensure that no file with the same name
-            exists in the directory.
+        image_dir (str): Image directory contains n01440764, n01443537, n01484850 and n15075141 directory.
+        destination (str): MindRecord file path to transform into, ensure that the directory is created in advance and
+            no file with the same name exists in the directory.
         partition_number (int, optional): The partition size. Default: 1.
 
     Raises:
-        ValueError: If `map_file`, `image_dir` or `destination` is invalid.
+        ValueError: If `map_file` , `image_dir` or `destination` is invalid.
     """
 
     def __init__(self, map_file, image_dir, destination, partition_number=1):
@@ -68,59 +73,6 @@ class ImageNetToMR:
 
         self.writer = FileWriter(self.destination, self.partition_number)
 
-    def _get_imagenet_as_dict(self):
-        """
-        Get data from imagenet as dict.
-
-        Yields:
-            data (dict of list): imagenet data list which contains dict.
-        """
-        real_file_path = os.path.realpath(self.map_file)
-        if not os.path.exists(real_file_path):
-            raise IOError("map file {} not exists".format(self.map_file))
-
-        label_dict = {}
-        with open(real_file_path) as fp:
-            line = fp.readline()
-            while line:
-                labels = line.split(" ")
-                label_dict[labels[1]] = labels[0]
-                line = fp.readline()
-
-        # get all the dir which are n02087046, n02094114, n02109525
-        dir_paths = {}
-        for item in label_dict:
-            real_path = os.path.join(self.image_dir, label_dict[item])
-            if not os.path.isdir(real_path):
-                logger.warning("{} dir is not exist".format(real_path))
-                continue
-            dir_paths[item] = real_path
-
-        if not dir_paths:
-            raise PathNotExistsError("not valid image dir in {}".format(self.image_dir))
-
-        # get the filename, label and image binary as a dict
-        for label in dir_paths:
-            for item in os.listdir(dir_paths[label]):
-                file_name = os.path.join(dir_paths[label], item)
-                if not item.endswith("JPEG") and not item.endswith("jpg"):
-                    logger.warning("{} file is not suffix with JPEG/jpg, skip it.".format(file_name))
-                    continue
-                data = {}
-                data["file_name"] = str(file_name)
-                data["label"] = int(label)
-
-                # get the image data
-                real_file_path = os.path.realpath(file_name)
-                image_file = open(real_file_path, "rb")
-                image_bytes = image_file.read()
-                image_file.close()
-                if not image_bytes:
-                    logger.warning("The image file: {} is invalid.".format(file_name))
-                    continue
-                data["image"] = image_bytes
-                yield data
-
     def run(self):
         """
         Execute transformation from imagenet to MindRecord.
@@ -158,11 +110,11 @@ class ImageNetToMR:
                 for _ in range(batch_size):
                     data_list.append(imagenet_iter.__next__())
                     transform_count += 1
-                self.writer.write_raw_data(data_list)
+                self.writer.write_raw_data(data_list, True)
                 logger.info("transformed {} record...".format(transform_count))
             except StopIteration:
                 if data_list:
-                    self.writer.write_raw_data(data_list)
+                    self.writer.write_raw_data(data_list, True)
                     logger.info("transformed {} record...".format(transform_count))
                 break
 
@@ -190,3 +142,56 @@ class ImageNetToMR:
         if t.exitcode != 0:
             raise t.exception
         return t.res
+
+    def _get_imagenet_as_dict(self):
+        """
+        Get data from imagenet as dict.
+
+        Yields:
+            data (dict of list): imagenet data list which contains dict.
+        """
+        real_file_path = os.path.realpath(self.map_file)
+        if not os.path.exists(real_file_path):
+            raise IOError("map file {} not exists".format(self.map_file))
+
+        label_dict = {}
+        with open(real_file_path) as fp:
+            line = fp.readline()
+            while line:
+                labels = line.split(" ")
+                label_dict[labels[1]] = labels[0]
+                line = fp.readline()
+
+        # get all the dir which are n02087046, n02094114, n02109525
+        dir_paths = {}
+        for item in label_dict:
+            real_path = os.path.join(self.image_dir, label_dict[item])
+            if not os.path.isdir(real_path):
+                logger.warning("{} dir is not exist".format(real_path))
+                continue
+            dir_paths[item] = real_path
+
+        if not dir_paths:
+            raise PathNotExistsError("not valid image dir in {}".format(self.image_dir))
+
+        # get the filename, label and image binary as a dict
+        for label in dir_paths:
+            for item in os.listdir(dir_paths[label]):
+                file_name = os.path.join(dir_paths[label], item)
+                if not item.endswith("JPEG") and not item.endswith("jpg"):
+                    logger.warning("{} file is not suffix with JPEG/jpg, skip it.".format(file_name))
+                    continue
+                data = {}
+                data["file_name"] = str(file_name)
+                data["label"] = int(label)
+
+                # get the image data
+                real_file_path = os.path.realpath(file_name)
+                image_file = open(real_file_path, "rb")
+                image_bytes = image_file.read()
+                image_file.close()
+                if not image_bytes:
+                    logger.warning("The image file: {} is invalid.".format(file_name))
+                    continue
+                data["image"] = image_bytes
+                yield data

mindspore/mindrecord/tools/mnist_to_mr.py

@@ -26,9 +26,9 @@ from ..filewriter import FileWriter
 from ..shardutils import check_filename, ExceptionThread, SUCCESS, FAILED
 
 try:
-    cv2 = import_module("cv2")
+    cv_import = import_module("cv2")
 except ModuleNotFoundError:
-    cv2 = None
+    cv_import = None
 
 __all__ = ['MnistToMR']
 
@@ -41,12 +41,12 @@ class MnistToMR:
         source (str): Directory that contains t10k-images-idx3-ubyte.gz,
                       train-images-idx3-ubyte.gz, t10k-labels-idx1-ubyte.gz
                       and train-labels-idx1-ubyte.gz.
-        destination (str): MindRecord file path to transform into, ensure that no file with the same name
-            exists in the directory.
+        destination (str): MindRecord file path to transform into, ensure that the directory is created in advance and
+            no file with the same name exists in the directory.
         partition_number (int, optional): The partition size. Default: 1.
 
     Raises:
-        ValueError: If `source`, `destination`, `partition_number` is invalid.
+        ValueError: If `source` , `destination` , `partition_number` is invalid.
     """
 
     def __init__(self, source, destination, partition_number=1):
@@ -79,6 +79,40 @@ class MnistToMR:
 
         self.mnist_schema_json = {"label": {"type": "int64"}, "data": {"type": "bytes"}}
 
+    def run(self):
+        """
+        Execute transformation from Mnist to MindRecord.
+
+        Returns:
+            MSRStatus, SUCCESS or FAILED.
+        """
+
+        if not cv_import:
+            raise ModuleNotFoundError("opencv-python module not found, please use pip install it.")
+
+        if self._transform_train() == FAILED:
+            return FAILED
+        if self._transform_test() == FAILED:
+            return FAILED
+
+        return SUCCESS
+
+    def transform(self):
+        """
+        Encapsulate the :func:`mindspore.mindrecord.MnistToMR.run` function to exit normally.
+
+        Returns:
+            MSRStatus, SUCCESS or FAILED.
+        """
+
+        t = ExceptionThread(target=self.run)
+        t.daemon = True
+        t.start()
+        t.join()
+        if t.exitcode != 0:
+            raise t.exception
+        return t.res
+
     def _extract_images(self, filename):
         """Extract the images into a 4D tensor [image index, y, x, channels]."""
         real_file_path = os.path.realpath(filename)
@@ -108,7 +142,7 @@ class MnistToMR:
         train_data = self._extract_images(self.train_data_filename_)
         train_labels = self._extract_labels(self.train_labels_filename_)
         for data, label in zip(train_data, train_labels):
-            _, img = cv2.imencode(".jpeg", data)
+            _, img = cv_import.imencode(".jpeg", data)
             yield {"label": int(label), "data": img.tobytes()}
 
     def _mnist_test_iterator(self):
@@ -121,7 +155,7 @@ class MnistToMR:
         test_data = self._extract_images(self.test_data_filename_)
         test_labels = self._extract_labels(self.test_labels_filename_)
         for data, label in zip(test_data, test_labels):
-            _, img = cv2.imencode(".jpeg", data)
+            _, img = cv_import.imencode(".jpeg", data)
             yield {"label": int(label), "data": img.tobytes()}
 
     def _transform_train(self):
@@ -220,37 +254,3 @@ class MnistToMR:
         logger.info("--------------------------------------------")
 
         return ret
-
-    def run(self):
-        """
-        Execute transformation from Mnist to MindRecord.
-
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-        """
-
-        if not cv2:
-            raise ModuleNotFoundError("opencv-python module not found, please use pip install it.")
-
-        if self._transform_train() == FAILED:
-            return FAILED
-        if self._transform_test() == FAILED:
-            return FAILED
-
-        return SUCCESS
-
-    def transform(self):
-        """
-        Encapsulate the :func:`mindspore.mindrecord.MnistToMR.run` function to exit normally.
-
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-        """
-
-        t = ExceptionThread(target=self.run)
-        t.daemon = True
-        t.start()
-        t.join()
-        if t.exitcode != 0:
-            raise t.exception
-        return t.res