mindspore 1.10.0__cp39-cp39-win_amd64.whl → 2.0.0rc1__cp39-cp39-win_amd64.whl

mindspore/mindrecord/tools/cifar100_to_mr.py

@@ -27,9 +27,9 @@ from ..filewriter import FileWriter
 from ..shardutils import check_filename, ExceptionThread, SUCCESS, FAILED
 
 try:
-    cv2 = import_module("cv2")
+    cv_import = import_module("cv2")
 except ModuleNotFoundError:
-    cv2 = None
+    cv_import = None
 
 __all__ = ['Cifar100ToMR']
 
@@ -40,12 +40,12 @@ class Cifar100ToMR:
 
     Note:
         For details about Examples, please refer to `Converting the CIFAR-10 Dataset <https://
-        www.mindspore.cn/tutorials/en/r1.10/advanced/dataset/record.html#converting-the-cifar-10-dataset>`_.
+        www.mindspore.cn/tutorials/en/r2.0/advanced/dataset/record.html#converting-the-cifar-10-dataset>`_ .
 
     Args:
         source (str): The cifar100 directory to be transformed.
-        destination (str): MindRecord file path to transform into, ensure that no file with the same name
-            exists in the directory.
+        destination (str): MindRecord file path to transform into, ensure that the directory is created in advance and
+            no file with the same name exists in the directory.
 
     Raises:
         ValueError: If source or destination is invalid.
@@ -80,7 +80,7 @@ class Cifar100ToMR:
         Args:
             fields (list[str], optional):
                 A list of index field, e.g.["fine_label", "coarse_label"]. Default: None. For index
-                field settings, please refer to :func:`mindspore.mindrecord.FileWriter.add_index`.
+                field settings, please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
@@ -121,7 +121,7 @@ class Cifar100ToMR:
         Args:
             fields (list[str], optional):
                 A list of index field, e.g.["fine_label", "coarse_label"]. Default: None. For index
-                field settings, please refer to :func:`mindspore.mindrecord.FileWriter.add_index`.
+                field settings, please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
@@ -148,14 +148,14 @@ def _construct_raw_data(images, fine_labels, coarse_labels):
     Returns:
         list[dict], data dictionary constructed from cifar100.
     """
-    if not cv2:
+    if not cv_import:
         raise ModuleNotFoundError("opencv-python module not found, please use pip install it.")
 
     raw_data = []
     for i, img in enumerate(images):
         fine_label = np.int(fine_labels[i][0])
         coarse_label = np.int(coarse_labels[i][0])
-        _, img = cv2.imencode(".jpeg", img[..., [2, 1, 0]])
+        _, img = cv_import.imencode(".jpeg", img[..., [2, 1, 0]])
         row_data = {"id": int(i),
                     "data": img.tobytes(),
                     "fine_label": int(fine_label),

mindspore/mindrecord/tools/cifar10_to_mr.py

@@ -27,9 +27,9 @@ from ..filewriter import FileWriter
 from ..shardutils import check_filename, ExceptionThread, SUCCESS, FAILED
 
 try:
-    cv2 = import_module("cv2")
+    cv_import = import_module("cv2")
 except ModuleNotFoundError:
-    cv2 = None
+    cv_import = None
 
 __all__ = ['Cifar10ToMR']
 
@@ -40,12 +40,12 @@ class Cifar10ToMR:
 
     Note:
         For details about Examples, please refer to `Converting the CIFAR-10 Dataset <https://
-        www.mindspore.cn/tutorials/en/r1.10/advanced/dataset/record.html#converting-the-cifar-10-dataset>`_.
+        www.mindspore.cn/tutorials/en/r2.0/advanced/dataset/record.html#converting-the-cifar-10-dataset>`_ .
 
     Args:
         source (str): The cifar10 directory to be transformed.
-        destination (str): MindRecord file path to transform into, ensure that no file with the same name
-            exists in the directory.
+        destination (str): MindRecord file path to transform into, ensure that the directory is created in advance and
+            no file with the same name exists in the directory.
 
     Raises:
         ValueError: If source or destination is invalid.
@@ -79,7 +79,7 @@ class Cifar10ToMR:
 
         Args:
             fields (list[str], optional): A list of index fields. Default: None. For index field settings,
-                please refer to :func:`mindspore.mindrecord.FileWriter.add_index`.
+                please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
@@ -116,7 +116,7 @@ class Cifar10ToMR:
 
         Args:
             fields (list[str], optional): A list of index fields. Default: None. For index field settings,
-                please refer to :func:`mindspore.mindrecord.FileWriter.add_index`.
+                please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
@@ -143,13 +143,13 @@ def _construct_raw_data(images, labels):
         list[dict], data dictionary constructed from cifar10.
     """
 
-    if not cv2:
+    if not cv_import:
         raise ModuleNotFoundError("opencv-python module not found, please use pip install it.")
 
     raw_data = []
     for i, img in enumerate(images):
         label = np.int(labels[i][0])
-        _, img = cv2.imencode(".jpeg", img[..., [2, 1, 0]])
+        _, img = cv_import.imencode(".jpeg", img[..., [2, 1, 0]])
         row_data = {"id": int(i),
                     "data": img.tobytes(),
                     "label": int(label)}

mindspore/mindrecord/tools/csv_to_mr.py

@@ -36,17 +36,17 @@ class CsvToMR:
 
     Note:
         For details about Examples, please refer to `Converting CSV Dataset <https://
-        www.mindspore.cn/tutorials/en/r1.10/advanced/dataset/record.html#converting-csv-dataset>`_.
+        www.mindspore.cn/tutorials/en/r2.0/advanced/dataset/record.html#converting-csv-dataset>`_ .
 
     Args:
         source (str): The file path of csv.
-        destination (str): The MindRecord file path to transform into, ensure that no file with the same name
-            exists in the directory.
+        destination (str): The MindRecord file path to transform into, ensure that the directory is created in advance
+            and no file with the same name exists in the directory.
         columns_list(list[str], optional): A list of columns to be read. Default: None.
         partition_number (int, optional): The partition size, Default: 1.
 
     Raises:
-        ValueError: If `source`, `destination`, `partition_number` is invalid.
+        ValueError: If `source` , `destination` , `partition_number` is invalid.
         RuntimeError: If `columns_list` is invalid.
     """
 

mindspore/mindrecord/tools/imagenet_to_mr.py

@@ -27,26 +27,31 @@ __all__ = ['ImageNetToMR']
 
 
 class ImageNetToMR:
-    """
+    r"""
     A class to transform from imagenet to MindRecord.
 
     Args:
-        map_file (str): The map file that indicates label. The map file content should be like this:
+        map_file (str): The map file that indicates label. This file can be generated by command
+            :code:`ls -l [image_dir] | grep -vE "total|\." | awk -F " " '{print $9, NR-1;}' > [file_path]` ,
+            where `image_dir` is image directory contains n01440764, n01443537, n01484850 and n15075141 directory
+            and `file_path` is the generated `map_file` . An example of `map_file` is as below:
 
             .. code-block::
 
-              n02119789 0
-              n02100735 1
-              n02110185 2
-              n02096294 3
+              n01440764 0
+              n01443537 1
+              n01484850 2
+              n01491361 3
+              ...
+              n15075141 999
 
-        image_dir (str): Image directory contains n02119789, n02100735, n02110185 and n02096294 directory.
-        destination (str): MindRecord file path to transform into, ensure that no file with the same name
-            exists in the directory.
+        image_dir (str): Image directory contains n01440764, n01443537, n01484850 and n15075141 directory.
+        destination (str): MindRecord file path to transform into, ensure that the directory is created in advance and
+            no file with the same name exists in the directory.
         partition_number (int, optional): The partition size. Default: 1.
 
     Raises:
-        ValueError: If `map_file`, `image_dir` or `destination` is invalid.
+        ValueError: If `map_file` , `image_dir` or `destination` is invalid.
     """
 
     def __init__(self, map_file, image_dir, destination, partition_number=1):
@@ -68,59 +73,6 @@ class ImageNetToMR:
 
         self.writer = FileWriter(self.destination, self.partition_number)
 
-    def _get_imagenet_as_dict(self):
-        """
-        Get data from imagenet as dict.
-
-        Yields:
-            data (dict of list): imagenet data list which contains dict.
-        """
-        real_file_path = os.path.realpath(self.map_file)
-        if not os.path.exists(real_file_path):
-            raise IOError("map file {} not exists".format(self.map_file))
-
-        label_dict = {}
-        with open(real_file_path) as fp:
-            line = fp.readline()
-            while line:
-                labels = line.split(" ")
-                label_dict[labels[1]] = labels[0]
-                line = fp.readline()
-
-        # get all the dir which are n02087046, n02094114, n02109525
-        dir_paths = {}
-        for item in label_dict:
-            real_path = os.path.join(self.image_dir, label_dict[item])
-            if not os.path.isdir(real_path):
-                logger.warning("{} dir is not exist".format(real_path))
-                continue
-            dir_paths[item] = real_path
-
-        if not dir_paths:
-            raise PathNotExistsError("not valid image dir in {}".format(self.image_dir))
-
-        # get the filename, label and image binary as a dict
-        for label in dir_paths:
-            for item in os.listdir(dir_paths[label]):
-                file_name = os.path.join(dir_paths[label], item)
-                if not item.endswith("JPEG") and not item.endswith("jpg"):
-                    logger.warning("{} file is not suffix with JPEG/jpg, skip it.".format(file_name))
-                    continue
-                data = {}
-                data["file_name"] = str(file_name)
-                data["label"] = int(label)
-
-                # get the image data
-                real_file_path = os.path.realpath(file_name)
-                image_file = open(real_file_path, "rb")
-                image_bytes = image_file.read()
-                image_file.close()
-                if not image_bytes:
-                    logger.warning("The image file: {} is invalid.".format(file_name))
-                    continue
-                data["image"] = image_bytes
-                yield data
-
     def run(self):
         """
         Execute transformation from imagenet to MindRecord.
@@ -158,11 +110,11 @@ class ImageNetToMR:
                 for _ in range(batch_size):
                     data_list.append(imagenet_iter.__next__())
                     transform_count += 1
-                self.writer.write_raw_data(data_list)
+                self.writer.write_raw_data(data_list, True)
                 logger.info("transformed {} record...".format(transform_count))
             except StopIteration:
                 if data_list:
-                    self.writer.write_raw_data(data_list)
+                    self.writer.write_raw_data(data_list, True)
                     logger.info("transformed {} record...".format(transform_count))
                 break
 
@@ -190,3 +142,56 @@ class ImageNetToMR:
         if t.exitcode != 0:
             raise t.exception
         return t.res
+
+    def _get_imagenet_as_dict(self):
+        """
+        Get data from imagenet as dict.
+
+        Yields:
+            data (dict of list): imagenet data list which contains dict.
+        """
+        real_file_path = os.path.realpath(self.map_file)
+        if not os.path.exists(real_file_path):
+            raise IOError("map file {} not exists".format(self.map_file))
+
+        label_dict = {}
+        with open(real_file_path) as fp:
+            line = fp.readline()
+            while line:
+                labels = line.split(" ")
+                label_dict[labels[1]] = labels[0]
+                line = fp.readline()
+
+        # get all the dir which are n02087046, n02094114, n02109525
+        dir_paths = {}
+        for item in label_dict:
+            real_path = os.path.join(self.image_dir, label_dict[item])
+            if not os.path.isdir(real_path):
+                logger.warning("{} dir is not exist".format(real_path))
+                continue
+            dir_paths[item] = real_path
+
+        if not dir_paths:
+            raise PathNotExistsError("not valid image dir in {}".format(self.image_dir))
+
+        # get the filename, label and image binary as a dict
+        for label in dir_paths:
+            for item in os.listdir(dir_paths[label]):
+                file_name = os.path.join(dir_paths[label], item)
+                if not item.endswith("JPEG") and not item.endswith("jpg"):
+                    logger.warning("{} file is not suffix with JPEG/jpg, skip it.".format(file_name))
+                    continue
+                data = {}
+                data["file_name"] = str(file_name)
+                data["label"] = int(label)
+
+                # get the image data
+                real_file_path = os.path.realpath(file_name)
+                image_file = open(real_file_path, "rb")
+                image_bytes = image_file.read()
+                image_file.close()
+                if not image_bytes:
+                    logger.warning("The image file: {} is invalid.".format(file_name))
+                    continue
+                data["image"] = image_bytes
+                yield data

mindspore/mindrecord/tools/mnist_to_mr.py

@@ -26,9 +26,9 @@ from ..filewriter import FileWriter
 from ..shardutils import check_filename, ExceptionThread, SUCCESS, FAILED
 
 try:
-    cv2 = import_module("cv2")
+    cv_import = import_module("cv2")
 except ModuleNotFoundError:
-    cv2 = None
+    cv_import = None
 
 __all__ = ['MnistToMR']
 
@@ -41,12 +41,12 @@ class MnistToMR:
         source (str): Directory that contains t10k-images-idx3-ubyte.gz,
                       train-images-idx3-ubyte.gz, t10k-labels-idx1-ubyte.gz
                       and train-labels-idx1-ubyte.gz.
-        destination (str): MindRecord file path to transform into, ensure that no file with the same name
-            exists in the directory.
+        destination (str): MindRecord file path to transform into, ensure that the directory is created in advance and
+            no file with the same name exists in the directory.
         partition_number (int, optional): The partition size. Default: 1.
 
     Raises:
-        ValueError: If `source`, `destination`, `partition_number` is invalid.
+        ValueError: If `source` , `destination` , `partition_number` is invalid.
     """
 
     def __init__(self, source, destination, partition_number=1):
@@ -79,6 +79,40 @@ class MnistToMR:
 
         self.mnist_schema_json = {"label": {"type": "int64"}, "data": {"type": "bytes"}}
 
+    def run(self):
+        """
+        Execute transformation from Mnist to MindRecord.
+
+        Returns:
+            MSRStatus, SUCCESS or FAILED.
+        """
+
+        if not cv_import:
+            raise ModuleNotFoundError("opencv-python module not found, please use pip install it.")
+
+        if self._transform_train() == FAILED:
+            return FAILED
+        if self._transform_test() == FAILED:
+            return FAILED
+
+        return SUCCESS
+
+    def transform(self):
+        """
+        Encapsulate the :func:`mindspore.mindrecord.MnistToMR.run` function to exit normally.
+
+        Returns:
+            MSRStatus, SUCCESS or FAILED.
+        """
+
+        t = ExceptionThread(target=self.run)
+        t.daemon = True
+        t.start()
+        t.join()
+        if t.exitcode != 0:
+            raise t.exception
+        return t.res
+
     def _extract_images(self, filename):
         """Extract the images into a 4D tensor [image index, y, x, channels]."""
         real_file_path = os.path.realpath(filename)
@@ -108,7 +142,7 @@ class MnistToMR:
         train_data = self._extract_images(self.train_data_filename_)
         train_labels = self._extract_labels(self.train_labels_filename_)
         for data, label in zip(train_data, train_labels):
-            _, img = cv2.imencode(".jpeg", data)
+            _, img = cv_import.imencode(".jpeg", data)
             yield {"label": int(label), "data": img.tobytes()}
 
     def _mnist_test_iterator(self):
@@ -121,7 +155,7 @@ class MnistToMR:
         test_data = self._extract_images(self.test_data_filename_)
         test_labels = self._extract_labels(self.test_labels_filename_)
         for data, label in zip(test_data, test_labels):
-            _, img = cv2.imencode(".jpeg", data)
+            _, img = cv_import.imencode(".jpeg", data)
             yield {"label": int(label), "data": img.tobytes()}
 
     def _transform_train(self):
@@ -220,37 +254,3 @@ class MnistToMR:
         logger.info("--------------------------------------------")
 
         return ret
-
-    def run(self):
-        """
-        Execute transformation from Mnist to MindRecord.
-
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-        """
-
-        if not cv2:
-            raise ModuleNotFoundError("opencv-python module not found, please use pip install it.")
-
-        if self._transform_train() == FAILED:
-            return FAILED
-        if self._transform_test() == FAILED:
-            return FAILED
-
-        return SUCCESS
-
-    def transform(self):
-        """
-        Encapsulate the :func:`mindspore.mindrecord.MnistToMR.run` function to exit normally.
-
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-        """
-
-        t = ExceptionThread(target=self.run)
-        t.daemon = True
-        t.start()
-        t.join()
-        if t.exitcode != 0:
-            raise t.exception
-        return t.res

mindspore/mindrecord/tools/tfrecord_to_mr.py

@@ -69,16 +69,16 @@ class TFRecordToMR:
 
     Note:
         For details about Examples, please refer to `Converting TFRecord Dataset <https://
-        www.mindspore.cn/tutorials/en/r1.10/advanced/dataset/record.html#converting-tfrecord-dataset>`_.
+        www.mindspore.cn/tutorials/en/r2.0/advanced/dataset/record.html#converting-tfrecord-dataset>`_ .
 
     Args:
         source (str): TFRecord file to be transformed.
-        destination (str): MindRecord file path to transform into, ensure that no file with the same name
-            exists in the directory.
+        destination (str): MindRecord file path to transform into, ensure that the directory is created in advance and
+            no file with the same name exists in the directory.
         feature_dict (dict[str, FixedLenFeature]): Dictionary that states the feature type, and
             `FixedLenFeature <https://www.tensorflow.org/api_docs/python/tf/io/FixedLenFeature>`_ is supported.
         bytes_fields (list[str], optional): The bytes fields which are in `feature_dict` and can be images bytes.
-            Default: None.
+            Default: None, means that there is no byte dtype field such as image.
 
     Raises:
         ValueError: If parameter is invalid.
@@ -307,11 +307,11 @@ class TFRecordToMR:
                     data_list.append(tf_iter.__next__())
                     transform_count += 1
 
-                writer.write_raw_data(data_list)
+                writer.write_raw_data(data_list, True)
                 logger.info("Transformed {} records...".format(transform_count))
             except StopIteration:
                 if data_list:
-                    writer.write_raw_data(data_list)
+                    writer.write_raw_data(data_list, True)
                     logger.info("Transformed {} records...".format(transform_count))
                 break
         return writer.commit()

mindspore/nn/__init__.py

@@ -19,7 +19,7 @@ Pre-defined building blocks or computing units to construct neural networks.
 """
 from __future__ import absolute_import
 
-from mindspore.nn import layer, loss, optim, metrics, wrap, grad, probability, sparse, dynamic_lr, reinforcement
+from mindspore.nn import layer, loss, optim, wrap, grad, metrics, probability, sparse, dynamic_lr, reinforcement
 from mindspore.nn.learning_rate_schedule import *
 from mindspore.nn.dynamic_lr import *
 from mindspore.nn.cell import Cell, GraphCell
@@ -31,9 +31,6 @@ from mindspore.nn.wrap import *
 from mindspore.nn.grad import Jvp, Vjp
 from mindspore.nn.sparse import *
 from mindspore.nn.reinforcement import *
-from mindspore.nn.transformer import AttentionMask, VocabEmbedding, MultiHeadAttention, FeedForward, \
-    TransformerEncoder, TransformerDecoder, TransformerEncoderLayer, TransformerDecoderLayer, Transformer,\
-    TransformerOpParallelConfig, EmbeddingOpParallelConfig, TransformerRecomputeConfig, MoEConfig, OpParallelConfig
 
 __all__ = ["Cell", "GraphCell"]
 __all__.extend(layer.__all__)
@@ -46,6 +43,5 @@ __all__.extend(sparse.__all__)
 __all__.extend(learning_rate_schedule.__all__)
 __all__.extend(dynamic_lr.__all__)
 __all__.extend(reinforcement.__all__)
-__all__.extend(transformer.__all__)
 
 __all__.sort()