mindspore 2.2.11__cp37-cp37m-manylinux1_x86_64.whl → 2.3.0rc1__cp37-cp37m-manylinux1_x86_64.whl

mindspore/mindrecord/tools/cifar10_to_mr.py

@@ -18,18 +18,13 @@ Cifar10 convert tool for MindRecord.
 
 from importlib import import_module
 import os
-import numpy as np
 
 from mindspore import log as logger
 from .cifar10 import Cifar10
 from ..common.exceptions import PathNotExistsError
 from ..filewriter import FileWriter
-from ..shardutils import check_filename, ExceptionThread, SUCCESS, FAILED
+from ..shardutils import check_filename, ExceptionThread, SUCCESS
 
-try:
-    cv_import = import_module("cv2")
-except ModuleNotFoundError:
-    cv_import = None
 
 __all__ = ['Cifar10ToMR']
 
@@ -53,10 +48,12 @@ class Cifar10ToMR:
         >>> cifar10_dir = "/path/to/cifar10"
         >>> mindrecord_file = "/path/to/mindrecord/file"
         >>> cifar10_to_mr = Cifar10ToMR(cifar10_dir, mindrecord_file)
-        >>> status = cifar10_to_mr.transform()
+        >>> cifar10_to_mr.transform()
     """
 
     def __init__(self, source, destination):
+        self.cv_import = import_module("cv2")
+
         check_filename(source)
         self.source = source
 
@@ -96,13 +93,11 @@ class Cifar10ToMR:
         test_labels = cifar10_data.Test.labels
         logger.info("test images label: {}".format(test_labels.shape))
 
-        data_list = _construct_raw_data(images, labels)
-        test_data_list = _construct_raw_data(test_images, test_labels)
+        data_list = self._construct_raw_data(images, labels)
+        test_data_list = self._construct_raw_data(test_images, test_labels)
 
-        if _generate_mindrecord(self.destination, data_list, fields, "img_train") != SUCCESS:
-            return FAILED
-        if _generate_mindrecord(self.destination + "_test", test_data_list, fields, "img_test") != SUCCESS:
-            return FAILED
+        self._generate_mindrecord(self.destination, data_list, fields, "img_train")
+        self._generate_mindrecord(self.destination + "_test", test_data_list, fields, "img_test")
         return SUCCESS
 
     def transform(self, fields=None):
@@ -116,16 +111,12 @@ class Cifar10ToMR:
             fields (list[str], optional): A list of index fields. Default: ``None`` . For index field settings,
                 please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             ParamTypeError: If index field is invalid.
             MRMOpenError: If failed to open MindRecord file.
             MRMValidateDataError: If data does not match blob fields.
             MRMSetHeaderError: If failed to set header.
             MRMWriteDatasetError: If failed to write dataset.
-            TypeError: If `parallel_writer` is not bool.
             ValueError: If parameter `fields` is invalid.
         """
 
@@ -135,58 +126,52 @@ class Cifar10ToMR:
         t.join()
         if t.exitcode != 0:
             raise t.exception
-        return t.res
-
-
-def _construct_raw_data(images, labels):
-    """
-    Construct raw data from cifar10 data.
 
-    Args:
-        images (list): image list from cifar10.
-        labels (list): label list from cifar10.
 
-    Returns:
-        list[dict], data dictionary constructed from cifar10.
-    """
-
-    if not cv_import:
-        raise ModuleNotFoundError("opencv-python module not found, please use pip install it.")
+    def _construct_raw_data(self, images, labels):
+        """
+        Construct raw data from cifar10 data.
 
-    raw_data = []
-    for i, img in enumerate(images):
-        label = np.int(labels[i][0])
-        _, img = cv_import.imencode(".jpeg", img[..., [2, 1, 0]])
-        row_data = {"id": int(i),
-                    "data": img.tobytes(),
-                    "label": int(label)}
-        raw_data.append(row_data)
-    return raw_data
+        Args:
+            images (list): image list from cifar10.
+            labels (list): label list from cifar10.
 
+        Returns:
+            list[dict], data dictionary constructed from cifar10.
+        """
+        raw_data = []
+        for i, img in enumerate(images):
+            label = labels[i][0]
+            _, img = self.cv_import.imencode(".jpeg", img[..., [2, 1, 0]])
+            row_data = {"id": int(i),
+                        "data": img.tobytes(),
+                        "label": int(label)}
+            raw_data.append(row_data)
+        return raw_data
+
+
+    def _generate_mindrecord(self, file_name, raw_data, fields, schema_desc):
+        """
+        Generate MindRecord file from raw data.
 
-def _generate_mindrecord(file_name, raw_data, fields, schema_desc):
-    """
-    Generate MindRecord file from raw data.
+        Args:
+            file_name (str): File name of MindRecord File.
+            fields (list[str]): Fields would be set as index which
+            could not belong to blob fields and type could not be 'array' or 'bytes'.
+            raw_data (dict): dict of raw data.
+            schema_desc (str): String of schema description.
 
-    Args:
-        file_name (str): File name of MindRecord File.
-        fields (list[str]): Fields would be set as index which
-          could not belong to blob fields and type could not be 'array' or 'bytes'.
-        raw_data (dict): dict of raw data.
-        schema_desc (str): String of schema description.
-
-    Returns:
-        MSRStatus, SUCCESS or FAILED.
-    """
+        Returns:
+            SUCCESS or FAILED.
+        """
 
-    schema = {"id": {"type": "int64"}, "label": {"type": "int64"},
-              "data": {"type": "bytes"}}
+        schema = {"id": {"type": "int64"}, "label": {"type": "int64"}, "data": {"type": "bytes"}}
 
-    logger.info("transformed MindRecord schema is: {}".format(schema))
+        logger.info("transformed MindRecord schema is: {}".format(schema))
 
-    writer = FileWriter(file_name, 1)
-    writer.add_schema(schema, schema_desc)
-    if fields and isinstance(fields, list):
-        writer.add_index(fields)
-    writer.write_raw_data(raw_data)
-    return writer.commit()
+        writer = FileWriter(file_name, 1)
+        writer.add_schema(schema, schema_desc)
+        if fields and isinstance(fields, list):
+            writer.add_index(fields)
+        writer.write_raw_data(raw_data)
+        writer.commit()

mindspore/mindrecord/tools/csv_to_mr.py

@@ -22,10 +22,6 @@ from mindspore import log as logger
 from ..filewriter import FileWriter
 from ..shardutils import check_filename, ExceptionThread
 
-try:
-    pd = import_module("pandas")
-except ModuleNotFoundError:
-    pd = None
 
 __all__ = ['CsvToMR']
 
@@ -51,12 +47,12 @@ class CsvToMR:
         >>> csv_file = "/path/to/csv/file"
         >>> mindrecord_file = "/path/to/mindrecord/file"
         >>> csv_to_mr = CsvToMR(csv_file, mindrecord_file)
-        >>> status = csv_to_mr.transform()
+        >>> csv_to_mr.transform()
     """
 
     def __init__(self, source, destination, columns_list=None, partition_number=1):
-        if not pd:
-            raise Exception("Module pandas is not found, please use pip install it.")
+        self.pd = import_module("pandas")
+
         if isinstance(source, str):
             check_filename(source)
             self.source = source
@@ -135,8 +131,8 @@ class CsvToMR:
         if not os.path.exists(self.source):
             raise IOError("Csv file {} do not exist.".format(self.source))
 
-        pd.set_option('display.max_columns', None)
-        df = pd.read_csv(self.source)
+        self.pd.set_option('display.max_columns', None)
+        df = self.pd.read_csv(self.source)
 
         csv_schema = self._get_schema(df)
 
@@ -176,9 +172,8 @@ class CsvToMR:
                         "transformed {} record...".format(transform_count))
                 break
 
-        ret = self.writer.commit()
-
-        return ret
+        self.writer.commit()
+        return 0
 
     def transform(self):
         """
@@ -187,16 +182,12 @@ class CsvToMR:
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.CsvToMR` .
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             ParamTypeError: If index field is invalid.
             MRMOpenError: If failed to open MindRecord file.
             MRMValidateDataError: If data does not match blob fields.
             MRMSetHeaderError: If failed to set header.
             MRMWriteDatasetError: If failed to write dataset.
-            TypeError: If parallel_writer is not bool.
             IOError: Csv file does not exist.
             ValueError: The first line of the CSV file is used as column name and each field cannot start with a number.
         """
@@ -207,4 +198,3 @@ class CsvToMR:
         t.join()
         if t.exitcode != 0:
             raise t.exception
-        return t.res

mindspore/mindrecord/tools/imagenet_to_mr.py

@@ -60,7 +60,7 @@ class ImageNetToMR:
         >>> imagenet_dir = "/path/to/imagenet/train"
         >>> mindrecord_file = "/path/to/mindrecord/file"
         >>> imagenet_to_mr = ImageNetToMR(map_file, imagenet_dir, mindrecord_file, 8)
-        >>> status = imagenet_to_mr.transform()
+        >>> imagenet_to_mr.transform()
     """
 
     def __init__(self, map_file, image_dir, destination, partition_number=1):
@@ -121,14 +121,14 @@ class ImageNetToMR:
                     logger.info("transformed {} record...".format(transform_count))
                 break
 
-        ret = self.writer.commit()
+        self.writer.commit()
 
         t1_total = time.time()
         logger.info("--------------------------------------------")
         logger.info("END. Total time: {}".format(t1_total - t0_total))
         logger.info("--------------------------------------------")
 
-        return ret
+        return 0
 
     def transform(self):
         """
@@ -137,16 +137,12 @@ class ImageNetToMR:
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.ImageNetToMR` .
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             ParamTypeError: If index field is invalid.
             MRMOpenError: If failed to open MindRecord file.
             MRMValidateDataError: If data does not match blob fields.
             MRMSetHeaderError: If failed to set header.
             MRMWriteDatasetError: If failed to write dataset.
-            TypeError: If `parallel_writer` is not bool.
         """
 
         t = ExceptionThread(target=self.run)
@@ -155,7 +151,6 @@ class ImageNetToMR:
         t.join()
         if t.exitcode != 0:
             raise t.exception
-        return t.res
 
     def _get_imagenet_as_dict(self):
         """

mindspore/mindrecord/tools/mnist_to_mr.py

@@ -25,10 +25,6 @@ from mindspore import log as logger
 from ..filewriter import FileWriter
 from ..shardutils import check_filename, ExceptionThread, SUCCESS, FAILED
 
-try:
-    cv_import = import_module("cv2")
-except ModuleNotFoundError:
-    cv_import = None
 
 __all__ = ['MnistToMR']
 
@@ -54,10 +50,12 @@ class MnistToMR:
         >>> mnist_dir = "/path/to/mnist"
         >>> mindrecord_file = "/path/to/mindrecord/file"
         >>> mnist_to_mr = MnistToMR(mnist_dir, mindrecord_file)
-        >>> status = mnist_to_mr.transform()
+        >>> mnist_to_mr.transform()
     """
 
     def __init__(self, source, destination, partition_number=1):
+        self.cv_import = import_module("cv2")
+
         self.image_size = 28
         self.num_channels = 1
 
@@ -89,9 +87,6 @@ class MnistToMR:
 
     # pylint: disable=missing-docstring
     def run(self):
-        if not cv_import:
-            raise ModuleNotFoundError("opencv-python module not found, please use pip install it.")
-
         if self._transform_train() == FAILED:
             return FAILED
         if self._transform_test() == FAILED:
@@ -106,16 +101,12 @@ class MnistToMR:
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.MnistToMR` .
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             ParamTypeError: If index field is invalid.
             MRMOpenError: If failed to open MindRecord file.
             MRMValidateDataError: If data does not match blob fields.
             MRMSetHeaderError: If failed to set header.
             MRMWriteDatasetError: If failed to write dataset.
-            TypeError: If `parallel_writer` is not bool.
         """
 
         t = ExceptionThread(target=self.run)
@@ -124,7 +115,6 @@ class MnistToMR:
         t.join()
         if t.exitcode != 0:
             raise t.exception
-        return t.res
 
     def _extract_images(self, filename):
         """Extract the images into a 4D tensor [image index, y, x, channels]."""
@@ -155,7 +145,7 @@ class MnistToMR:
         train_data = self._extract_images(self.train_data_filename_)
         train_labels = self._extract_labels(self.train_labels_filename_)
         for data, label in zip(train_data, train_labels):
-            _, img = cv_import.imencode(".jpeg", data)
+            _, img = self.cv_import.imencode(".jpeg", data)
             yield {"label": int(label), "data": img.tobytes()}
 
     def _mnist_test_iterator(self):
@@ -168,7 +158,7 @@ class MnistToMR:
         test_data = self._extract_images(self.test_data_filename_)
         test_labels = self._extract_labels(self.test_labels_filename_)
         for data, label in zip(test_data, test_labels):
-            _, img = cv_import.imencode(".jpeg", data)
+            _, img = self.cv_import.imencode(".jpeg", data)
             yield {"label": int(label), "data": img.tobytes()}
 
     def _transform_train(self):
@@ -176,7 +166,7 @@ class MnistToMR:
         Execute transformation from Mnist train part to MindRecord.
 
         Returns:
-            MSRStatus, whether successfully written into MindRecord.
+            SUCCESS or FAILED.
         """
         t0_total = time.time()
 
@@ -210,21 +200,21 @@ class MnistToMR:
                     logger.info("transformed {} record...".format(transform_count))
                 break
 
-        ret = self.writer_train.commit()
+        self.writer_train.commit()
 
         t1_total = time.time()
         logger.info("--------------------------------------------")
         logger.info("Total time [train]: {}".format(t1_total - t0_total))
         logger.info("--------------------------------------------")
 
-        return ret
+        return 0
 
     def _transform_test(self):
         """
         Execute transformation from Mnist test part to MindRecord.
 
         Returns:
-            MSRStatus, whether Mnist is successfully transformed to MindRecord.
+            SUCCESS or FAILED.
         """
         t0_total = time.time()
 
@@ -259,11 +249,11 @@ class MnistToMR:
                     logger.info("transformed {} record...".format(transform_count))
                 break
 
-        ret = self.writer_test.commit()
+        self.writer_test.commit()
 
         t1_total = time.time()
         logger.info("--------------------------------------------")
         logger.info("Total time [test]: {}".format(t1_total - t0_total))
         logger.info("--------------------------------------------")
 
-        return ret
+        return 0

mindspore/mindrecord/tools/tfrecord_to_mr.py

@@ -93,14 +93,11 @@ class TFRecordToMR:
         ...                 "int64_list": tf.io.FixedLenFeature([6], tf.int64),
         ...                 "float_list": tf.io.FixedLenFeature([7], tf.float32)}
         >>> tfrecord_to_mr = TFRecordToMR(tfrecord_file, mindrecord_file, feature_dict, ["image_bytes"])
-        >>> status = tfrecord_to_mr.transform()
+        >>> tfrecord_to_mr.transform()
     """
 
     def __init__(self, source, destination, feature_dict, bytes_fields=None):
-        try:
-            self.tf = import_module("tensorflow")  # just used to convert tfrecord to mindrecord
-        except ModuleNotFoundError:
-            raise Exception("Module tensorflow is not found, please use pip install it.")
+        self.tf = import_module("tensorflow")  # just used to convert tfrecord to mindrecord
 
         if self.tf.__version__ < SupportedTensorFlowVersion:
             raise Exception("Module tensorflow version must be greater or equal {}.".format(SupportedTensorFlowVersion))
@@ -317,16 +314,12 @@ class TFRecordToMR:
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.TFRecordToMR` .
 
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
         Raises:
             ParamTypeError: If index field is invalid.
             MRMOpenError: If failed to open MindRecord file.
             MRMValidateDataError: If data does not match blob fields.
             MRMSetHeaderError: If failed to set header.
             MRMWriteDatasetError: If failed to write dataset.
-            TypeError: If `parallel_writer` is not bool.
         """
 
         t = ExceptionThread(target=self.run)
@@ -335,7 +328,6 @@ class TFRecordToMR:
         t.join()
         if t.exitcode != 0:
             raise t.exception
-        return t.res
 
     def _cast_type(self, value):
         """

mindspore/multiprocessing/__init__.py

@@ -0,0 +1,68 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+mindspore.multiprocessing is a wrapper around the native `multiprocessing` module.
+Some methods are overrode to support fork-based multiprocess.
+"""
+import signal
+import multiprocessing as mp
+from multiprocessing import *
+from mindspore._c_expression import fork_utils
+
+__all__ = []
+__all__ += mp.__all__
+
+
+class Process(mp.Process): # pylint: disable=function-redefined
+    """
+    Trigger fork callbacks by overriding native multiprocessing methods.
+    """
+    _child_at_fork_func = None
+
+    def run(self):
+        """
+        Trigger child_at_fork callback function after fork by overriding
+        multiprocessing.run method.
+        """
+        if mp.get_start_method() == "fork":
+            fork_utils.prctl_set_pdeathsig(signal.SIGINT)
+            fork_utils.child_at_fork()
+            if Process._child_at_fork_func and callable(Process._child_at_fork_func):
+                Process._child_at_fork_func() # pylint: disable=not-callable
+        super().run()
+
+    def start(self):
+        """
+        Trigger prepare_before_fork and parent_at_fork callback functions
+        by overriding multiprocessing.start method.
+        """
+        if mp.get_start_method() == "fork":
+            fork_utils.prepare_before_fork()
+            super().start()
+            fork_utils.parent_at_fork()
+        else:
+            super().start()
+
+_MsProcess = Process
+
+
+class Pool(mp.pool.Pool): # pylint: disable=function-redefined, abstract-method
+    """
+    Trigger fork callbacks by overriding native multiprocessing methods.
+    """
+    def Process(self, *args, **kwds):
+        if self._ctx.get_start_method() == "fork":
+            return _MsProcess(*args, **kwds)
+        return super().Process(*args, **kwds)