mindspore 2.0.0rc1__cp38-cp38-manylinux1_x86_64.whl → 2.2.0__cp38-cp38-manylinux1_x86_64.whl

mindspore/mindrecord/tools/cifar10_to_mr.py

@@ -36,11 +36,8 @@ __all__ = ['Cifar10ToMR']
 
 class Cifar10ToMR:
     """
-    A class to transform from cifar10 to MindRecord.
-
-    Note:
-        For details about Examples, please refer to `Converting the CIFAR-10 Dataset <https://
-        www.mindspore.cn/tutorials/en/r2.0/advanced/dataset/record.html#converting-the-cifar-10-dataset>`_ .
+    A class to transform from cifar10 which needs to be a Python version with a name
+    similar to: ``cifar-10-python.tar.gz`` to MindRecord.
 
     Args:
         source (str): The cifar10 directory to be transformed.
@@ -49,6 +46,14 @@ class Cifar10ToMR:
 
     Raises:
         ValueError: If source or destination is invalid.
+
+    Examples:
+        >>> from mindspore.mindrecord import Cifar10ToMR
+        >>>
+        >>> cifar10_dir = "/path/to/cifar10"
+        >>> mindrecord_file = "/path/to/mindrecord/file"
+        >>> cifar10_to_mr = Cifar10ToMR(cifar10_dir, mindrecord_file)
+        >>> status = cifar10_to_mr.transform()
     """
 
     def __init__(self, source, destination):
@@ -73,18 +78,8 @@ class Cifar10ToMR:
         self.destination = destination
         self.writer = None
 
+    # pylint: disable=missing-docstring
     def run(self, fields=None):
-        """
-        Execute transformation from cifar10 to MindRecord.
-
-        Args:
-            fields (list[str], optional): A list of index fields. Default: None. For index field settings,
-                please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .
-
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-        """
-
         if fields and not isinstance(fields, list):
             raise ValueError("The parameter fields should be None or list")
 
@@ -112,14 +107,26 @@ class Cifar10ToMR:
 
     def transform(self, fields=None):
         """
-        Encapsulate the :func:`mindspore.mindrecord.Cifar10ToMR.run` function to exit normally.
+        Execute transformation from cifar10 to MindRecord.
+
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.Cifar10ToMR` .
 
         Args:
-            fields (list[str], optional): A list of index fields. Default: None. For index field settings,
+            fields (list[str], optional): A list of index fields. Default: ``None`` . For index field settings,
                 please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
+
+        Raises:
+            ParamTypeError: If index field is invalid.
+            MRMOpenError: If failed to open MindRecord file.
+            MRMValidateDataError: If data does not match blob fields.
+            MRMSetHeaderError: If failed to set header.
+            MRMWriteDatasetError: If failed to write dataset.
+            TypeError: If `parallel_writer` is not bool.
+            ValueError: If parameter `fields` is invalid.
         """
 
         t = ExceptionThread(target=self.run, kwargs={'fields': fields})

mindspore/mindrecord/tools/csv_to_mr.py

@@ -34,20 +34,24 @@ class CsvToMR:
     """
     A class to transform from csv to MindRecord.
 
-    Note:
-        For details about Examples, please refer to `Converting CSV Dataset <https://
-        www.mindspore.cn/tutorials/en/r2.0/advanced/dataset/record.html#converting-csv-dataset>`_ .
-
     Args:
         source (str): The file path of csv.
         destination (str): The MindRecord file path to transform into, ensure that the directory is created in advance
             and no file with the same name exists in the directory.
-        columns_list(list[str], optional): A list of columns to be read. Default: None.
-        partition_number (int, optional): The partition size, Default: 1.
+        columns_list(list[str], optional): A list of columns to be read. Default: ``None`` .
+        partition_number (int, optional): The partition size, Default: ``1`` .
 
     Raises:
         ValueError: If `source` , `destination` , `partition_number` is invalid.
         RuntimeError: If `columns_list` is invalid.
+
+    Examples:
+        >>> from mindspore.mindrecord import CsvToMR
+        >>>
+        >>> csv_file = "/path/to/csv/file"
+        >>> mindrecord_file = "/path/to/mindrecord/file"
+        >>> csv_to_mr = CsvToMR(csv_file, mindrecord_file)
+        >>> status = csv_to_mr.transform()
     """
 
     def __init__(self, source, destination, columns_list=None, partition_number=1):
@@ -126,13 +130,8 @@ class CsvToMR:
                     row[col] = r[col]
             yield row
 
+    # pylint: disable=missing-docstring
     def run(self):
-        """
-        Execute transformation from csv to MindRecord.
-
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-        """
         if not os.path.exists(self.source):
             raise IOError("Csv file {} do not exist.".format(self.source))
 
@@ -153,6 +152,10 @@ class CsvToMR:
         self.writer.add_schema(csv_schema, "csv_schema")
 
         # add the index
+        for column_name in list(self.columns_list):
+            if str(column_name).isdigit():
+                raise ValueError("The first line content: {} of the CSV file is used as a column name and "
+                                 "does not allow starting with a number.".format(str(column_name)))
         self.writer.add_index(list(self.columns_list))
 
         csv_iter = self._get_row_of_csv(df, self.columns_list)
@@ -179,10 +182,23 @@ class CsvToMR:
 
     def transform(self):
         """
-        Encapsulate the :func:`mindspore.mindrecord.CsvToMR.run` function to exit normally.
+        Execute transformation from csv to MindRecord.
+
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.CsvToMR` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
+
+        Raises:
+            ParamTypeError: If index field is invalid.
+            MRMOpenError: If failed to open MindRecord file.
+            MRMValidateDataError: If data does not match blob fields.
+            MRMSetHeaderError: If failed to set header.
+            MRMWriteDatasetError: If failed to write dataset.
+            TypeError: If parallel_writer is not bool.
+            IOError: Csv file does not exist.
+            ValueError: The first line of the CSV file is used as column name and each field cannot start with a number.
         """
 
         t = ExceptionThread(target=self.run)

mindspore/mindrecord/tools/imagenet_to_mr.py

@@ -48,10 +48,19 @@ class ImageNetToMR:
         image_dir (str): Image directory contains n01440764, n01443537, n01484850 and n15075141 directory.
         destination (str): MindRecord file path to transform into, ensure that the directory is created in advance and
             no file with the same name exists in the directory.
-        partition_number (int, optional): The partition size. Default: 1.
+        partition_number (int, optional): The partition size. Default: ``1`` .
 
     Raises:
         ValueError: If `map_file` , `image_dir` or `destination` is invalid.
+
+    Examples:
+        >>> from mindspore.mindrecord import ImageNetToMR
+        >>>
+        >>> map_file = "/path/to/imagenet/map_file"
+        >>> imagenet_dir = "/path/to/imagenet/train"
+        >>> mindrecord_file = "/path/to/mindrecord/file"
+        >>> imagenet_to_mr = ImageNetToMR(map_file, imagenet_dir, mindrecord_file, 8)
+        >>> status = imagenet_to_mr.transform()
     """
 
     def __init__(self, map_file, image_dir, destination, partition_number=1):
@@ -73,14 +82,8 @@ class ImageNetToMR:
 
         self.writer = FileWriter(self.destination, self.partition_number)
 
+    # pylint: disable=missing-docstring
     def run(self):
-        """
-        Execute transformation from imagenet to MindRecord.
-
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-        """
-
         t0_total = time.time()
 
         imagenet_schema_json = {"label": {"type": "int32"},
@@ -129,10 +132,21 @@ class ImageNetToMR:
 
     def transform(self):
         """
-        Encapsulate the :func:`mindspore.mindrecord.ImageNetToMR.run` function to exit normally.
+        Execute transformation from imagenet to MindRecord.
+
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.ImageNetToMR` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
+
+        Raises:
+            ParamTypeError: If index field is invalid.
+            MRMOpenError: If failed to open MindRecord file.
+            MRMValidateDataError: If data does not match blob fields.
+            MRMSetHeaderError: If failed to set header.
+            MRMWriteDatasetError: If failed to write dataset.
+            TypeError: If `parallel_writer` is not bool.
         """
 
         t = ExceptionThread(target=self.run)
@@ -155,7 +169,7 @@ class ImageNetToMR:
             raise IOError("map file {} not exists".format(self.map_file))
 
         label_dict = {}
-        with open(real_file_path) as fp:
+        with open(real_file_path, "r") as fp:
             line = fp.readline()
             while line:
                 labels = line.split(" ")

mindspore/mindrecord/tools/mnist_to_mr.py

@@ -43,10 +43,18 @@ class MnistToMR:
                       and train-labels-idx1-ubyte.gz.
         destination (str): MindRecord file path to transform into, ensure that the directory is created in advance and
             no file with the same name exists in the directory.
-        partition_number (int, optional): The partition size. Default: 1.
+        partition_number (int, optional): The partition size. Default: ``1`` .
 
     Raises:
         ValueError: If `source` , `destination` , `partition_number` is invalid.
+
+    Examples:
+        >>> from mindspore.mindrecord import MnistToMR
+        >>>
+        >>> mnist_dir = "/path/to/mnist"
+        >>> mindrecord_file = "/path/to/mindrecord/file"
+        >>> mnist_to_mr = MnistToMR(mnist_dir, mindrecord_file)
+        >>> status = mnist_to_mr.transform()
     """
 
     def __init__(self, source, destination, partition_number=1):
@@ -79,14 +87,8 @@ class MnistToMR:
 
         self.mnist_schema_json = {"label": {"type": "int64"}, "data": {"type": "bytes"}}
 
+    # pylint: disable=missing-docstring
     def run(self):
-        """
-        Execute transformation from Mnist to MindRecord.
-
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-        """
-
         if not cv_import:
             raise ModuleNotFoundError("opencv-python module not found, please use pip install it.")
 
@@ -99,10 +101,21 @@ class MnistToMR:
 
     def transform(self):
         """
-        Encapsulate the :func:`mindspore.mindrecord.MnistToMR.run` function to exit normally.
+        Execute transformation from Mnist to MindRecord.
+
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.MnistToMR` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
+
+        Raises:
+            ParamTypeError: If index field is invalid.
+            MRMOpenError: If failed to open MindRecord file.
+            MRMValidateDataError: If data does not match blob fields.
+            MRMSetHeaderError: If failed to set header.
+            MRMWriteDatasetError: If failed to write dataset.
+            TypeError: If `parallel_writer` is not bool.
         """
 
         t = ExceptionThread(target=self.run)
@@ -116,7 +129,7 @@ class MnistToMR:
     def _extract_images(self, filename):
         """Extract the images into a 4D tensor [image index, y, x, channels]."""
         real_file_path = os.path.realpath(filename)
-        with gzip.open(real_file_path) as bytestream:
+        with gzip.open(real_file_path, "rb") as bytestream:
             bytestream.read(16)
             buf = bytestream.read()
             data = np.frombuffer(buf, dtype=np.uint8)
@@ -126,7 +139,7 @@ class MnistToMR:
     def _extract_labels(self, filename):
         """Extract the labels into a vector of int64 label IDs."""
         real_file_path = os.path.realpath(filename)
-        with gzip.open(real_file_path) as bytestream:
+        with gzip.open(real_file_path, "rb") as bytestream:
             bytestream.read(8)
             buf = bytestream.read()
             labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int64)

mindspore/mindrecord/tools/tfrecord_to_mr.py

@@ -67,10 +67,6 @@ class TFRecordToMR:
     """
     A class to transform from TFRecord to MindRecord.
 
-    Note:
-        For details about Examples, please refer to `Converting TFRecord Dataset <https://
-        www.mindspore.cn/tutorials/en/r2.0/advanced/dataset/record.html#converting-tfrecord-dataset>`_ .
-
     Args:
         source (str): TFRecord file to be transformed.
         destination (str): MindRecord file path to transform into, ensure that the directory is created in advance and
@@ -78,11 +74,26 @@ class TFRecordToMR:
         feature_dict (dict[str, FixedLenFeature]): Dictionary that states the feature type, and
             `FixedLenFeature <https://www.tensorflow.org/api_docs/python/tf/io/FixedLenFeature>`_ is supported.
         bytes_fields (list[str], optional): The bytes fields which are in `feature_dict` and can be images bytes.
-            Default: None, means that there is no byte dtype field such as image.
+            Default: ``None`` , means that there is no byte dtype field such as image.
 
     Raises:
         ValueError: If parameter is invalid.
         Exception: when tensorflow module is not found or version is not correct.
+
+    Examples:
+        >>> from mindspore.mindrecord import TFRecordToMR
+        >>> import tensorflow as tf
+        >>>
+        >>> tfrecord_file = "/path/to/tfrecord/file"
+        >>> mindrecord_file = "/path/to/mindrecord/file"
+        >>> feature_dict = {"file_name": tf.io.FixedLenFeature([], tf.string),
+        ...                 "image_bytes": tf.io.FixedLenFeature([], tf.string),
+        ...                 "int64_scalar": tf.io.FixedLenFeature([], tf.int64),
+        ...                 "float_scalar": tf.io.FixedLenFeature([], tf.float32),
+        ...                 "int64_list": tf.io.FixedLenFeature([6], tf.int64),
+        ...                 "float_list": tf.io.FixedLenFeature([7], tf.float32)}
+        >>> tfrecord_to_mr = TFRecordToMR(tfrecord_file, mindrecord_file, feature_dict, ["image_bytes"])
+        >>> status = tfrecord_to_mr.transform()
     """
 
     def __init__(self, source, destination, feature_dict, bytes_fields=None):
@@ -207,14 +218,8 @@ class TFRecordToMR:
             else:
                 ms_dict[cast_key] = float(val)
 
+    # pylint: disable=missing-docstring
     def tfrecord_iterator_oldversion(self):
-        """
-        Yield a dict with key to be fields in schema, and value to be data.
-        This function is for old version tensorflow whose version number < 2.1.0.
-
-        Returns:
-            dict, data dictionary whose keys are the same as columns.
-        """
         logger.warning("This interface will be deleted or invisible in the future.")
 
         dataset = self.tf.data.TFRecordDataset(self.source)
@@ -261,14 +266,8 @@ class TFRecordToMR:
                     np.asarray(val, _cast_string_type_to_np_type(self.mindrecord_schema[cast_key]["type"]))
         return ms_dict
 
-
+    # pylint: disable=missing-docstring
     def tfrecord_iterator(self):
-        """
-        Yield a dictionary whose keys are fields in schema.
-
-        Returns:
-            dict, data dictionary whose keys are the same as columns.
-        """
         logger.warning("This interface will be deleted or invisible in the future.")
 
         dataset = self.tf.data.TFRecordDataset(self.source)
@@ -282,13 +281,8 @@ class TFRecordToMR:
             except self.tf.errors.InvalidArgumentError:
                 raise ValueError("TFRecord feature_dict parameter error.")
 
+    # pylint: disable=missing-docstring
     def run(self):
-        """
-        Execute transformation from TFRecord to MindRecord.
-
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-        """
         writer = FileWriter(self.destination)
         logger.info("Transformed MindRecord schema is: {}, TFRecord feature dict is: {}"
                     .format(self.mindrecord_schema, self.feature_dict))
@@ -318,10 +312,21 @@ class TFRecordToMR:
 
     def transform(self):
         """
-        Encapsulate the :func:`mindspore.mindrecord.TFRecordToMR.run` function to exit normally.
+        Execute transformation from TFRecord to MindRecord.
+
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.TFRecordToMR` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
+
+        Raises:
+            ParamTypeError: If index field is invalid.
+            MRMOpenError: If failed to open MindRecord file.
+            MRMValidateDataError: If data does not match blob fields.
+            MRMSetHeaderError: If failed to set header.
+            MRMWriteDatasetError: If failed to write dataset.
+            TypeError: If `parallel_writer` is not bool.
         """
 
         t = ExceptionThread(target=self.run)