mindspore 2.2.11__cp39-cp39-win_amd64.whl → 2.3.0__cp39-cp39-win_amd64.whl

mindspore/mindrecord/tools/cifar100_to_mr.py

@@ -18,18 +18,13 @@ Cifar100 convert tool for MindRecord.
 
 from importlib import import_module
 import os
-import numpy as np
 
 from mindspore import log as logger
 from .cifar100 import Cifar100
 from ..common.exceptions import PathNotExistsError
 from ..filewriter import FileWriter
-from ..shardutils import check_filename, ExceptionThread, SUCCESS, FAILED
+from ..shardutils import check_filename, ExceptionThread, SUCCESS
 
-try:
-    cv_import = import_module("cv2")
-except ModuleNotFoundError:
-    cv_import = None
 
 __all__ = ['Cifar100ToMR']
 
@@ -53,10 +48,12 @@ class Cifar100ToMR:
         >>> cifar100_dir = "/path/to/cifar100"
         >>> mindrecord_file = "/path/to/mindrecord/file"
         >>> cifar100_to_mr = Cifar100ToMR(cifar100_dir, mindrecord_file)
-        >>> status = cifar100_to_mr.transform()
+        >>> cifar100_to_mr.transform()
     """
 
     def __init__(self, source, destination):
+        self.cv_import = import_module("cv2")
+
         check_filename(source)
         self.source = source
 
@@ -100,13 +97,11 @@ class Cifar100ToMR:
         test_coarse_labels = cifar100_data.Test.coarse_labels
         logger.info("test images coarse label: {}".format(coarse_labels.shape))
 
-        data_list = _construct_raw_data(images, fine_labels, coarse_labels)
-        test_data_list = _construct_raw_data(test_images, test_fine_labels, test_coarse_labels)
+        data_list = self._construct_raw_data(images, fine_labels, coarse_labels)
+        test_data_list = self._construct_raw_data(test_images, test_fine_labels, test_coarse_labels)
 
-        if _generate_mindrecord(self.destination, data_list, fields, "img_train") != SUCCESS:
-            return FAILED
-        if _generate_mindrecord(self.destination + "_test", test_data_list, fields, "img_test") != SUCCESS:
-            return FAILED
+        self._generate_mindrecord(self.destination, data_list, fields, "img_train")
+        self._generate_mindrecord(self.destination + "_test", test_data_list, fields, "img_test")
         return SUCCESS
 
     def transform(self, fields=None):
@@ -121,16 +116,12 @@ class Cifar100ToMR:
                 A list of index field, e.g.["fine_label", "coarse_label"]. Default: ``None`` . For index
                 field settings, please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             ParamTypeError: If index field is invalid.
             MRMOpenError: If failed to open MindRecord file.
             MRMValidateDataError: If data does not match blob fields.
             MRMSetHeaderError: If failed to set header.
             MRMWriteDatasetError: If failed to write dataset.
-            TypeError: If `parallel_writer` is not bool.
             ValueError: If parameter `fields` is invalid.
         """
 
@@ -140,59 +131,55 @@ class Cifar100ToMR:
         t.join()
         if t.exitcode != 0:
             raise t.exception
-        return t.res
 
 
-def _construct_raw_data(images, fine_labels, coarse_labels):
-    """
-    Construct raw data from cifar100 data.
+    def _construct_raw_data(self, images, fine_labels, coarse_labels):
+        """
+        Construct raw data from cifar100 data.
 
-    Args:
-        images (list): image list from cifar100.
-        fine_labels (list): fine label list from cifar100.
-        coarse_labels (list): coarse label list from cifar100.
+        Args:
+            images (list): image list from cifar100.
+            fine_labels (list): fine label list from cifar100.
+            coarse_labels (list): coarse label list from cifar100.
 
-    Returns:
-        list[dict], data dictionary constructed from cifar100.
-    """
-    if not cv_import:
-        raise ModuleNotFoundError("opencv-python module not found, please use pip install it.")
-
-    raw_data = []
-    for i, img in enumerate(images):
-        fine_label = np.int(fine_labels[i][0])
-        coarse_label = np.int(coarse_labels[i][0])
-        _, img = cv_import.imencode(".jpeg", img[..., [2, 1, 0]])
-        row_data = {"id": int(i),
-                    "data": img.tobytes(),
-                    "fine_label": int(fine_label),
-                    "coarse_label": int(coarse_label)}
-        raw_data.append(row_data)
-    return raw_data
-
-
-def _generate_mindrecord(file_name, raw_data, fields, schema_desc):
-    """
-    Generate MindRecord file from raw data.
+        Returns:
+            list[dict], data dictionary constructed from cifar100.
+        """
+        raw_data = []
+        for i, img in enumerate(images):
+            fine_label = fine_labels[i][0]
+            coarse_label = coarse_labels[i][0]
+            _, img = self.cv_import.imencode(".jpeg", img[..., [2, 1, 0]])
+            row_data = {"id": int(i),
+                        "data": img.tobytes(),
+                        "fine_label": int(fine_label),
+                        "coarse_label": int(coarse_label)}
+            raw_data.append(row_data)
+        return raw_data
+
+
+    def _generate_mindrecord(self, file_name, raw_data, fields, schema_desc):
+        """
+        Generate MindRecord file from raw data.
 
-    Args:
-        file_name (str): File name of MindRecord File.
-        fields (list[str]): Fields would be set as index which
-          could not belong to blob fields and type could not be 'array' or 'bytes'.
-        raw_data (dict): Dict of raw data.
-        schema_desc (str): String of schema description.
-
-    Returns:
-        MSRStatus, SUCCESS or FAILED.
-    """
-    schema = {"id": {"type": "int64"}, "fine_label": {"type": "int64"},
-              "coarse_label": {"type": "int64"}, "data": {"type": "bytes"}}
+        Args:
+            file_name (str): File name of MindRecord File.
+            fields (list[str]): Fields would be set as index which
+            could not belong to blob fields and type could not be 'array' or 'bytes'.
+            raw_data (dict): Dict of raw data.
+            schema_desc (str): String of schema description.
+
+        Returns:
+            SUCCESS or FAILED.
+        """
+        schema = {"id": {"type": "int64"}, "fine_label": {"type": "int64"},
+                  "coarse_label": {"type": "int64"}, "data": {"type": "bytes"}}
 
-    logger.info("transformed MindRecord schema is: {}".format(schema))
+        logger.info("transformed MindRecord schema is: {}".format(schema))
 
-    writer = FileWriter(file_name, 1)
-    writer.add_schema(schema, schema_desc)
-    if fields and isinstance(fields, list):
-        writer.add_index(fields)
-    writer.write_raw_data(raw_data)
-    return writer.commit()
+        writer = FileWriter(file_name, 1)
+        writer.add_schema(schema, schema_desc)
+        if fields and isinstance(fields, list):
+            writer.add_index(fields)
+        writer.write_raw_data(raw_data)
+        writer.commit()

mindspore/mindrecord/tools/cifar10_to_mr.py

@@ -18,18 +18,13 @@ Cifar10 convert tool for MindRecord.
 
 from importlib import import_module
 import os
-import numpy as np
 
 from mindspore import log as logger
 from .cifar10 import Cifar10
 from ..common.exceptions import PathNotExistsError
 from ..filewriter import FileWriter
-from ..shardutils import check_filename, ExceptionThread, SUCCESS, FAILED
+from ..shardutils import check_filename, ExceptionThread, SUCCESS
 
-try:
-    cv_import = import_module("cv2")
-except ModuleNotFoundError:
-    cv_import = None
 
 __all__ = ['Cifar10ToMR']
 
@@ -53,10 +48,12 @@ class Cifar10ToMR:
         >>> cifar10_dir = "/path/to/cifar10"
         >>> mindrecord_file = "/path/to/mindrecord/file"
         >>> cifar10_to_mr = Cifar10ToMR(cifar10_dir, mindrecord_file)
-        >>> status = cifar10_to_mr.transform()
+        >>> cifar10_to_mr.transform()
     """
 
     def __init__(self, source, destination):
+        self.cv_import = import_module("cv2")
+
         check_filename(source)
         self.source = source
 
@@ -96,13 +93,11 @@ class Cifar10ToMR:
         test_labels = cifar10_data.Test.labels
         logger.info("test images label: {}".format(test_labels.shape))
 
-        data_list = _construct_raw_data(images, labels)
-        test_data_list = _construct_raw_data(test_images, test_labels)
+        data_list = self._construct_raw_data(images, labels)
+        test_data_list = self._construct_raw_data(test_images, test_labels)
 
-        if _generate_mindrecord(self.destination, data_list, fields, "img_train") != SUCCESS:
-            return FAILED
-        if _generate_mindrecord(self.destination + "_test", test_data_list, fields, "img_test") != SUCCESS:
-            return FAILED
+        self._generate_mindrecord(self.destination, data_list, fields, "img_train")
+        self._generate_mindrecord(self.destination + "_test", test_data_list, fields, "img_test")
         return SUCCESS
 
     def transform(self, fields=None):
@@ -116,16 +111,12 @@ class Cifar10ToMR:
             fields (list[str], optional): A list of index fields. Default: ``None`` . For index field settings,
                 please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             ParamTypeError: If index field is invalid.
             MRMOpenError: If failed to open MindRecord file.
             MRMValidateDataError: If data does not match blob fields.
             MRMSetHeaderError: If failed to set header.
             MRMWriteDatasetError: If failed to write dataset.
-            TypeError: If `parallel_writer` is not bool.
             ValueError: If parameter `fields` is invalid.
         """
 
@@ -135,58 +126,52 @@ class Cifar10ToMR:
         t.join()
         if t.exitcode != 0:
             raise t.exception
-        return t.res
-
-
-def _construct_raw_data(images, labels):
-    """
-    Construct raw data from cifar10 data.
 
-    Args:
-        images (list): image list from cifar10.
-        labels (list): label list from cifar10.
 
-    Returns:
-        list[dict], data dictionary constructed from cifar10.
-    """
-
-    if not cv_import:
-        raise ModuleNotFoundError("opencv-python module not found, please use pip install it.")
+    def _construct_raw_data(self, images, labels):
+        """
+        Construct raw data from cifar10 data.
 
-    raw_data = []
-    for i, img in enumerate(images):
-        label = np.int(labels[i][0])
-        _, img = cv_import.imencode(".jpeg", img[..., [2, 1, 0]])
-        row_data = {"id": int(i),
-                    "data": img.tobytes(),
-                    "label": int(label)}
-        raw_data.append(row_data)
-    return raw_data
+        Args:
+            images (list): image list from cifar10.
+            labels (list): label list from cifar10.
 
+        Returns:
+            list[dict], data dictionary constructed from cifar10.
+        """
+        raw_data = []
+        for i, img in enumerate(images):
+            label = labels[i][0]
+            _, img = self.cv_import.imencode(".jpeg", img[..., [2, 1, 0]])
+            row_data = {"id": int(i),
+                        "data": img.tobytes(),
+                        "label": int(label)}
+            raw_data.append(row_data)
+        return raw_data
+
+
+    def _generate_mindrecord(self, file_name, raw_data, fields, schema_desc):
+        """
+        Generate MindRecord file from raw data.
 
-def _generate_mindrecord(file_name, raw_data, fields, schema_desc):
-    """
-    Generate MindRecord file from raw data.
+        Args:
+            file_name (str): File name of MindRecord File.
+            fields (list[str]): Fields would be set as index which
+            could not belong to blob fields and type could not be 'array' or 'bytes'.
+            raw_data (dict): dict of raw data.
+            schema_desc (str): String of schema description.
 
-    Args:
-        file_name (str): File name of MindRecord File.
-        fields (list[str]): Fields would be set as index which
-          could not belong to blob fields and type could not be 'array' or 'bytes'.
-        raw_data (dict): dict of raw data.
-        schema_desc (str): String of schema description.
-
-    Returns:
-        MSRStatus, SUCCESS or FAILED.
-    """
+        Returns:
+            SUCCESS or FAILED.
+        """
 
-    schema = {"id": {"type": "int64"}, "label": {"type": "int64"},
-              "data": {"type": "bytes"}}
+        schema = {"id": {"type": "int64"}, "label": {"type": "int64"}, "data": {"type": "bytes"}}
 
-    logger.info("transformed MindRecord schema is: {}".format(schema))
+        logger.info("transformed MindRecord schema is: {}".format(schema))
 
-    writer = FileWriter(file_name, 1)
-    writer.add_schema(schema, schema_desc)
-    if fields and isinstance(fields, list):
-        writer.add_index(fields)
-    writer.write_raw_data(raw_data)
-    return writer.commit()
+        writer = FileWriter(file_name, 1)
+        writer.add_schema(schema, schema_desc)
+        if fields and isinstance(fields, list):
+            writer.add_index(fields)
+        writer.write_raw_data(raw_data)
+        writer.commit()

mindspore/mindrecord/tools/csv_to_mr.py

@@ -22,10 +22,6 @@ from mindspore import log as logger
 from ..filewriter import FileWriter
 from ..shardutils import check_filename, ExceptionThread
 
-try:
-    pd = import_module("pandas")
-except ModuleNotFoundError:
-    pd = None
 
 __all__ = ['CsvToMR']
 
@@ -51,12 +47,12 @@ class CsvToMR:
         >>> csv_file = "/path/to/csv/file"
         >>> mindrecord_file = "/path/to/mindrecord/file"
         >>> csv_to_mr = CsvToMR(csv_file, mindrecord_file)
-        >>> status = csv_to_mr.transform()
+        >>> csv_to_mr.transform()
     """
 
     def __init__(self, source, destination, columns_list=None, partition_number=1):
-        if not pd:
-            raise Exception("Module pandas is not found, please use pip install it.")
+        self.pd = import_module("pandas")
+
         if isinstance(source, str):
             check_filename(source)
             self.source = source
@@ -135,8 +131,8 @@ class CsvToMR:
         if not os.path.exists(self.source):
             raise IOError("Csv file {} do not exist.".format(self.source))
 
-        pd.set_option('display.max_columns', None)
-        df = pd.read_csv(self.source)
+        self.pd.set_option('display.max_columns', None)
+        df = self.pd.read_csv(self.source)
 
         csv_schema = self._get_schema(df)
 
@@ -176,9 +172,8 @@ class CsvToMR:
                         "transformed {} record...".format(transform_count))
                 break
 
-        ret = self.writer.commit()
-
-        return ret
+        self.writer.commit()
+        return 0
 
     def transform(self):
         """
@@ -187,16 +182,12 @@ class CsvToMR:
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.CsvToMR` .
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             ParamTypeError: If index field is invalid.
             MRMOpenError: If failed to open MindRecord file.
             MRMValidateDataError: If data does not match blob fields.
             MRMSetHeaderError: If failed to set header.
             MRMWriteDatasetError: If failed to write dataset.
-            TypeError: If parallel_writer is not bool.
             IOError: Csv file does not exist.
             ValueError: The first line of the CSV file is used as column name and each field cannot start with a number.
         """
@@ -207,4 +198,3 @@ class CsvToMR:
         t.join()
         if t.exitcode != 0:
             raise t.exception
-        return t.res

mindspore/mindrecord/tools/imagenet_to_mr.py

@@ -60,7 +60,7 @@ class ImageNetToMR:
         >>> imagenet_dir = "/path/to/imagenet/train"
         >>> mindrecord_file = "/path/to/mindrecord/file"
         >>> imagenet_to_mr = ImageNetToMR(map_file, imagenet_dir, mindrecord_file, 8)
-        >>> status = imagenet_to_mr.transform()
+        >>> imagenet_to_mr.transform()
     """
 
     def __init__(self, map_file, image_dir, destination, partition_number=1):
@@ -121,14 +121,14 @@ class ImageNetToMR:
                     logger.info("transformed {} record...".format(transform_count))
                 break
 
-        ret = self.writer.commit()
+        self.writer.commit()
 
         t1_total = time.time()
         logger.info("--------------------------------------------")
         logger.info("END. Total time: {}".format(t1_total - t0_total))
         logger.info("--------------------------------------------")
 
-        return ret
+        return 0
 
     def transform(self):
         """
@@ -137,16 +137,12 @@ class ImageNetToMR:
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.ImageNetToMR` .
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             ParamTypeError: If index field is invalid.
             MRMOpenError: If failed to open MindRecord file.
             MRMValidateDataError: If data does not match blob fields.
             MRMSetHeaderError: If failed to set header.
             MRMWriteDatasetError: If failed to write dataset.
-            TypeError: If `parallel_writer` is not bool.
         """
 
         t = ExceptionThread(target=self.run)
@@ -155,7 +151,6 @@ class ImageNetToMR:
         t.join()
         if t.exitcode != 0:
             raise t.exception
-        return t.res
 
     def _get_imagenet_as_dict(self):
         """

mindspore/mindrecord/tools/mnist_to_mr.py

@@ -25,10 +25,6 @@ from mindspore import log as logger
 from ..filewriter import FileWriter
 from ..shardutils import check_filename, ExceptionThread, SUCCESS, FAILED
 
-try:
-    cv_import = import_module("cv2")
-except ModuleNotFoundError:
-    cv_import = None
 
 __all__ = ['MnistToMR']
 
@@ -54,10 +50,12 @@ class MnistToMR:
         >>> mnist_dir = "/path/to/mnist"
         >>> mindrecord_file = "/path/to/mindrecord/file"
         >>> mnist_to_mr = MnistToMR(mnist_dir, mindrecord_file)
-        >>> status = mnist_to_mr.transform()
+        >>> mnist_to_mr.transform()
     """
 
     def __init__(self, source, destination, partition_number=1):
+        self.cv_import = import_module("cv2")
+
         self.image_size = 28
         self.num_channels = 1
 
@@ -89,9 +87,6 @@ class MnistToMR:
 
     # pylint: disable=missing-docstring
     def run(self):
-        if not cv_import:
-            raise ModuleNotFoundError("opencv-python module not found, please use pip install it.")
-
         if self._transform_train() == FAILED:
             return FAILED
         if self._transform_test() == FAILED:
@@ -106,16 +101,12 @@ class MnistToMR:
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.MnistToMR` .
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             ParamTypeError: If index field is invalid.
             MRMOpenError: If failed to open MindRecord file.
             MRMValidateDataError: If data does not match blob fields.
             MRMSetHeaderError: If failed to set header.
             MRMWriteDatasetError: If failed to write dataset.
-            TypeError: If `parallel_writer` is not bool.
         """
 
         t = ExceptionThread(target=self.run)
@@ -124,7 +115,6 @@ class MnistToMR:
         t.join()
         if t.exitcode != 0:
             raise t.exception
-        return t.res
 
     def _extract_images(self, filename):
         """Extract the images into a 4D tensor [image index, y, x, channels]."""
@@ -155,7 +145,7 @@ class MnistToMR:
         train_data = self._extract_images(self.train_data_filename_)
         train_labels = self._extract_labels(self.train_labels_filename_)
         for data, label in zip(train_data, train_labels):
-            _, img = cv_import.imencode(".jpeg", data)
+            _, img = self.cv_import.imencode(".jpeg", data)
             yield {"label": int(label), "data": img.tobytes()}
 
     def _mnist_test_iterator(self):
@@ -168,7 +158,7 @@ class MnistToMR:
         test_data = self._extract_images(self.test_data_filename_)
         test_labels = self._extract_labels(self.test_labels_filename_)
         for data, label in zip(test_data, test_labels):
-            _, img = cv_import.imencode(".jpeg", data)
+            _, img = self.cv_import.imencode(".jpeg", data)
             yield {"label": int(label), "data": img.tobytes()}
 
     def _transform_train(self):
@@ -176,7 +166,7 @@ class MnistToMR:
         Execute transformation from Mnist train part to MindRecord.
 
         Returns:
-            MSRStatus, whether successfully written into MindRecord.
+            SUCCESS or FAILED.
         """
         t0_total = time.time()
 
@@ -210,21 +200,21 @@ class MnistToMR:
                     logger.info("transformed {} record...".format(transform_count))
                 break
 
-        ret = self.writer_train.commit()
+        self.writer_train.commit()
 
         t1_total = time.time()
         logger.info("--------------------------------------------")
         logger.info("Total time [train]: {}".format(t1_total - t0_total))
         logger.info("--------------------------------------------")
 
-        return ret
+        return 0
 
     def _transform_test(self):
         """
         Execute transformation from Mnist test part to MindRecord.
 
         Returns:
-            MSRStatus, whether Mnist is successfully transformed to MindRecord.
+            SUCCESS or FAILED.
         """
         t0_total = time.time()
 
@@ -259,11 +249,11 @@ class MnistToMR:
                     logger.info("transformed {} record...".format(transform_count))
                 break
 
-        ret = self.writer_test.commit()
+        self.writer_test.commit()
 
         t1_total = time.time()
         logger.info("--------------------------------------------")
         logger.info("Total time [test]: {}".format(t1_total - t0_total))
         logger.info("--------------------------------------------")
 
-        return ret
+        return 0

mindspore/mindrecord/tools/tfrecord_to_mr.py

@@ -93,14 +93,11 @@ class TFRecordToMR:
         ...                 "int64_list": tf.io.FixedLenFeature([6], tf.int64),
         ...                 "float_list": tf.io.FixedLenFeature([7], tf.float32)}
         >>> tfrecord_to_mr = TFRecordToMR(tfrecord_file, mindrecord_file, feature_dict, ["image_bytes"])
-        >>> status = tfrecord_to_mr.transform()
+        >>> tfrecord_to_mr.transform()
     """
 
     def __init__(self, source, destination, feature_dict, bytes_fields=None):
-        try:
-            self.tf = import_module("tensorflow")  # just used to convert tfrecord to mindrecord
-        except ModuleNotFoundError:
-            raise Exception("Module tensorflow is not found, please use pip install it.")
+        self.tf = import_module("tensorflow")  # just used to convert tfrecord to mindrecord
 
         if self.tf.__version__ < SupportedTensorFlowVersion:
             raise Exception("Module tensorflow version must be greater or equal {}.".format(SupportedTensorFlowVersion))
@@ -317,16 +314,12 @@ class TFRecordToMR:
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.TFRecordToMR` .
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             ParamTypeError: If index field is invalid.
             MRMOpenError: If failed to open MindRecord file.
             MRMValidateDataError: If data does not match blob fields.
             MRMSetHeaderError: If failed to set header.
             MRMWriteDatasetError: If failed to write dataset.
-            TypeError: If `parallel_writer` is not bool.
         """
 
         t = ExceptionThread(target=self.run)
@@ -335,7 +328,6 @@ class TFRecordToMR:
         t.join()
         if t.exitcode != 0:
             raise t.exception
-        return t.res
 
     def _cast_type(self, value):
         """