mindspore 2.0.0rc1__cp38-none-any.whl → 2.2.0__cp38-none-any.whl

mindspore/log.py

@@ -100,7 +100,7 @@ class _DataFormatter(logging.Formatter):
 
         Args:
             sub_module (str): The submodule name.
-            fmt (str): Specified format pattern. Default: None.
+            fmt (str): Specified format pattern. Default: ``None``.
         """
         super(_DataFormatter, self).__init__(fmt=fmt, **kwargs)
         self.sub_module = sub_module.upper()
@@ -234,9 +234,9 @@ def get_level():
 
     Examples:
         >>> import os
+        >>> import mindspore as ms
         >>> os.environ['GLOG_v'] = '0'
-        >>> from mindspore import log as logger
-        >>> level = logger.get_level()
+        >>> level = ms.get_level()
         >>> print(level)
         '0'
     """
@@ -388,14 +388,14 @@ def get_log_config():
 
     Examples:
         >>> import os
+        >>> import mindspore as ms
         >>> os.environ['GLOG_v'] = '1'
         >>> os.environ['GLOG_logtostderr'] = '0'
         >>> os.environ['GLOG_log_dir'] = '/var/log'
         >>> os.environ['logger_maxBytes'] = '5242880'
         >>> os.environ['logger_backupCount'] = '10'
         >>> os.environ['GLOG_stderrthreshold'] = '2'
-        >>> from mindspore import log as logger
-        >>> config= logger.get_log_config()
+        >>> config = ms.get_log_config()
         >>> print(config)
         {'GLOG_v': '1', 'GLOG_logtostderr': '0', 'GLOG_log_dir': '/var/log',
          'logger_maxBytes': '5242880', 'logger_backupCount': '10', 'GLOG_stderrthreshold': '2'}
@@ -435,7 +435,7 @@ def _find_caller(stack_info=False, stacklevel=1):
     file name, function name and line number.
 
     Args:
-        stack_info (bool): If the value is true, print stack information to the log. Default: False.
+        stack_info (bool): If the value is true, print stack information to the log. Default: ``False``.
 
     Returns:
         tuple, the tuple of the frame data.
@@ -478,6 +478,7 @@ def _get_rank_id():
     """Get rank id."""
     rank_id = os.getenv('RANK_ID')
     gpu_rank_id = os.getenv('OMPI_COMM_WORLD_RANK')
+    ms_node_id = os.getenv('MS_NODE_ID')
     rank = '0'
     if rank_id and gpu_rank_id and rank_id != gpu_rank_id:
         warnings.warn(
@@ -487,6 +488,8 @@ def _get_rank_id():
         rank = rank_id
     elif gpu_rank_id:
         rank = gpu_rank_id
+    elif ms_node_id:
+        rank = ms_node_id
     return rank
 
 

mindspore/mindrecord/filereader.py

@@ -16,6 +16,7 @@
 This module is to read data from MindRecord.
 """
 import platform
+from mindspore import log as logger
 
 from .shardreader import ShardReader
 from .shardheader import ShardHeader
@@ -34,13 +35,14 @@ class FileReader:
         If `file_name` is a file path, it tries to load all MindRecord files generated \
         in a conversion, and throws an exception if a MindRecord file is missing.
         If `file_name` is file path list, only the MindRecord files in the list are loaded.
+        The parameter `operator` has no effect and will be deprecated in a future version.
 
     Args:
         file_name (str, list[str]): One of MindRecord file path or file path list.
-        num_consumer (int, optional): Number of reader workers which load data. Default: 4.
+        num_consumer (int, optional): Number of reader workers which load data. Default: ``4`` .
             It should not be smaller than 1 or larger than the number of processor cores.
-        columns (list[str], optional): A list of fields where corresponding data would be read. Default: None.
-        operator (int, optional): Reserved parameter for operators. Default: None.
+        columns (list[str], optional): A list of fields where corresponding data would be read. Default: ``None`` .
+        operator (int, optional): Reserved parameter for operators. Default: ``None`` .
 
     Raises:
         ParamValueError: If `file_name` , `num_consumer` or `columns` is invalid.
@@ -59,6 +61,9 @@ class FileReader:
 
     @check_parameter
     def __init__(self, file_name, num_consumer=4, columns=None, operator=None):
+        if operator is not None:
+            logger.warning("The parameter 'operator' will be deprecated in a future version.")
+
         if columns:
             if isinstance(columns, list):
                 self._columns = columns
@@ -85,6 +90,9 @@ class FileReader:
         """
         Yield a batch of data according to columns at a time.
 
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.FileReader` .
+
         Returns:
             dict, a batch whose keys are the same as columns.
 
@@ -98,7 +106,12 @@ class FileReader:
             iterator = self._reader.get_next()
 
     def close(self):
-        """Stop reader worker and close file."""
+        """
+        Stop reader worker and close file.
+
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.FileReader` .
+        """
         self._reader.close()
 
     def schema(self):
@@ -107,6 +120,14 @@ class FileReader:
 
         Returns:
             dict, the schema info.
+
+        Examples:
+            >>> from mindspore.mindrecord import FileReader
+            >>>
+            >>> mindrecord_file = "/path/to/mindrecord/file"
+            >>> reader = FileReader(file_name=mindrecord_file)
+            >>> schema = reader.schema()
+            >>> reader.close()
         """
         return self._header.schema
 
@@ -116,5 +137,13 @@ class FileReader:
 
         Returns:
             int, the number of the samples in MindRecord.
+
+        Examples:
+            >>> from mindspore.mindrecord import FileReader
+            >>>
+            >>> mindrecord_file = "/path/to/mindrecord/file"
+            >>> reader = FileReader(file_name=mindrecord_file)
+            >>> length = reader.len()
+            >>> reader.close()
         """
         return self._reader.len()

mindspore/mindrecord/filewriter.py

@@ -46,26 +46,24 @@ class FileWriter:
     Args:
         file_name (str): File name of MindRecord file.
         shard_num (int, optional): The Number of MindRecord files.
-            It should be between [1, 1000]. Default: 1.
-        overwrite (bool, optional): Whether to overwrite if the file already exists. Default: False.
+            It should be between [1, 1000]. Default: ``1`` .
+        overwrite (bool, optional): Whether to overwrite if the file already exists. Default: ``False`` .
 
     Raises:
         ParamValueError: If `file_name` or `shard_num` or `overwrite` is invalid.
 
     Examples:
         >>> from mindspore.mindrecord import FileWriter
-        >>> schema_json = {"file_name": {"type": "string"}, "label": {"type": "int32"}, "data": {"type": "bytes"}}
-        >>> indexes = ["file_name", "label"]
-        >>> data = [{"file_name": "1.jpg", "label": 0,
-        ...          "data": b"\x10c\xb3w\xa8\xee$o&<q\x8c\x8e(\xa2\x90\x90\x96\xbc\xb1\x1e\xd4QER\x13?\xff"},
-        ...         {"file_name": "2.jpg", "label": 56,
-        ...          "data": b"\xe6\xda\xd1\xae\x07\xb8>\xd4\x00\xf8\x129\x15\xd9\xf2q\xc0\xa2\x91YFUO\x1dsE1"},
-        ...         {"file_name": "3.jpg", "label": 99,
-        ...          "data": b"\xaf\xafU<\xb8|6\xbd}\xc1\x99[\xeaj+\x8f\x84\xd3\xcc\xa0,i\xbb\xb9-\xcdz\xecp{T\xb1"}]
+        >>>
         >>> writer = FileWriter(file_name="test.mindrecord", shard_num=1, overwrite=True)
+        >>> schema_json = {"file_name": {"type": "string"}, "label": {"type": "int32"}, "data": {"type": "bytes"}}
         >>> schema_id = writer.add_schema(schema_json, "test_schema")
+        >>> indexes = ["file_name", "label"]
         >>> status = writer.add_index(indexes)
-        >>> status = writer.write_raw_data(data)
+        >>> for i in range(10):
+        ...     data = [{"file_name": str(i) + ".jpg", "label": i,
+        ...              "data": b"\x10c\xb3w\xa8\xee$o&<q\x8c\x8e(\xa2\x90\x90\x96\xbc\xb1\x1e\xd4QER\x13?\xff"}]
+        ...     status = writer.write_raw_data(data)
         >>> status = writer.commit()
     """
 
@@ -132,15 +130,20 @@ class FileWriter:
 
         Examples:
             >>> from mindspore.mindrecord import FileWriter
-            >>> schema_json = {"file_name": {"type": "string"}, "label": {"type": "int32"}, "data": {"type": "bytes"}}
-            >>> data = [{"file_name": "1.jpg", "label": 0,
+            >>>
+            >>> data = [{"file_name": "0.jpg", "label": 0,
             ...          "data": b"\x10c\xb3w\xa8\xee$o&<q\x8c\x8e(\xa2\x90\x90\x96\xbc\xb1\x1e\xd4QER\x13?\xff"}]
             >>> writer = FileWriter(file_name="test.mindrecord", shard_num=1, overwrite=True)
+            >>> schema_json = {"file_name": {"type": "string"}, "label": {"type": "int32"}, "data": {"type": "bytes"}}
             >>> schema_id = writer.add_schema(schema_json, "test_schema")
             >>> status = writer.write_raw_data(data)
             >>> status = writer.commit()
+            >>>
             >>> write_append = FileWriter.open_for_append("test.mindrecord")
-            >>> status = write_append.write_raw_data(data)
+            >>> for i in range(9):
+            ...     data = [{"file_name": str(i+1) + ".jpg", "label": i,
+            ...              "data": b"\x10c\xb3w\xa8\xee$o&<q\x8c\x8e(\xa2\x90\x90\x96\xbc\xb1\x1e\xd4QER\x13?\xff"}]
+            ...     status = write_append.write_raw_data(data)
             >>> status = write_append.commit()
         """
         if platform.system().lower() == "windows":
@@ -173,11 +176,49 @@ class FileWriter:
         The schema is added to describe the raw data to be written.
 
         Note:
-            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter` .
+            Please refer to the Examples of :class:`mindspore.mindrecord.FileWriter` .
+
+        .. list-table:: The data types supported by MindRecord.
+           :widths: 25 25 50
+           :header-rows: 1
+
+           * - Data Type
+             - Data Shape
+             - Details
+           * - int32
+             - /
+             - integer number
+           * - int64
+             - /
+             - integer number
+           * - float32
+             - /
+             - real number
+           * - float64
+             - /
+             - real number
+           * - string
+             - /
+             - string data
+           * - bytes
+             - /
+             - binary data
+           * - int32
+             - [-1] / [-1, 32, 32] / [3, 224, 224]
+             - numpy ndarray
+           * - int64
+             - [-1] / [-1, 32, 32] / [3, 224, 224]
+             - numpy ndarray
+           * - float32
+             - [-1] / [-1, 32, 32] / [3, 224, 224]
+             - numpy ndarray
+           * - float64
+             - [-1] / [-1, 32, 32] / [3, 224, 224]
+             - numpy ndarray
 
         Args:
             content (dict): Dictionary of schema content.
-            desc (str, optional): String of schema description, Default: None.
+            desc (str, optional): String of schema description, Default: ``None`` .
 
         Returns:
             int, schema id.
@@ -186,6 +227,12 @@ class FileWriter:
             MRMInvalidSchemaError: If schema is invalid.
             MRMBuildSchemaError: If failed to build schema.
             MRMAddSchemaError: If failed to add schema.
+
+        Examples:
+            >>> # Examples of available schemas
+            >>> schema1 = {"file_name": {"type": "string"}, "label": {"type": "int32"}, "data": {"type": "bytes"}}
+            >>> schema2 = {"input_ids": {"type": "int32", "shape": [-1]},
+            ...            "input_masks": {"type": "int32", "shape": [-1]}}
         """
         ret, error_msg = self._validate_schema(content)
         if ret is False:
@@ -203,7 +250,7 @@ class FileWriter:
             If the function is not called, the fields of the primitive type
             in schema are set as indexes by default.
 
-            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter` .
+            Please refer to the Examples of :class:`mindspore.mindrecord.FileWriter` .
 
         Args:
             index_fields (list[str]): fields from schema.
@@ -228,17 +275,6 @@ class FileWriter:
         return self._header.add_index_fields(index_fields)
 
     def open_and_set_header(self):
-        """
-        Open writer and set header which stores meta information. The function is only used for parallel \
-        writing and is called before the `write_raw_data` .
-
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-
-        Raises:
-            MRMOpenError: If failed to open MindRecord file.
-            MRMSetHeaderError: If failed to set header.
-        """
         logger.warning("This interface will be deleted or invisible in the future.")
 
         if not self._writer.is_open:
@@ -253,11 +289,11 @@ class FileWriter:
         files after the raw data is verified against the schema.
 
         Note:
-            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter` .
+            Please refer to the Examples of :class:`mindspore.mindrecord.FileWriter` .
 
         Args:
            raw_data (list[dict]): List of raw data.
-           parallel_writer (bool, optional): Write raw data in parallel if it equals to True. Default: False.
+           parallel_writer (bool, optional): Write raw data in parallel if it equals to True. Default: ``False`` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
@@ -286,7 +322,7 @@ class FileWriter:
                 raise ParamTypeError('raw_data', 'list')
             if self._flush and not self._append:
                 raise RuntimeError("Not allowed to call `write_raw_data` on flushed MindRecord files." \
-                                   "When creating new Mindrecord files, please remove `commit` before " \
+                                   "When creating new MindRecord files, please remove `commit` before " \
                                    "`write_raw_data`. In other cases, when appending to existing MindRecord files, " \
                                    "please call `open_for_append` first and then `write_raw_data`.")
             for each_raw in raw_data:
@@ -330,7 +366,7 @@ class FileWriter:
                 # check the status of worker process
                 for i in range(len(self._paths)):
                     if not self._workers[i].is_alive():
-                        raise RuntimeError("Worker process(pid:{}) has stopped abnormal. Please check " \
+                        raise RuntimeError("Worker process(pid:{}) has stopped abnormally. Please check " \
                                            "the above log".format(self._workers[i].pid))
                 continue
             return SUCCESS
@@ -346,7 +382,6 @@ class FileWriter:
             header_size (int): Size of header, between 16*1024(16KB) and
                 128*1024*1024(128MB).
 
-
         Returns:
             MSRStatus, SUCCESS or FAILED.
 
@@ -390,7 +425,7 @@ class FileWriter:
         Flush data in memory to disk and generate the corresponding database files.
 
         Note:
-            Please refer to the Examples of class: `mindspore.mindrecord.FileWriter` .
+            Please refer to the Examples of :class:`mindspore.mindrecord.FileWriter` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
@@ -494,7 +529,7 @@ class FileWriter:
                     time.sleep(1)
                     continue
                 elif self._workers[index].exitcode != 0:
-                    raise RuntimeError("Worker process(pid:{}) has stopped abnormal. Please check " \
+                    raise RuntimeError("Worker process(pid:{}) has stopped abnormally. Please check " \
                                        "the above log".format(self._workers[index].pid))
                 break
 

mindspore/mindrecord/mindpage.py

@@ -16,7 +16,6 @@
 This module is to support reading page from MindRecord.
 """
 
-from mindspore import log as logger
 from .shardsegment import ShardSegment
 from .shardutils import check_parameter
 from .common.exceptions import ParamValueError, ParamTypeError, MRMDefineCategoryError
@@ -30,12 +29,33 @@ class MindPage:
 
     Args:
         file_name (Union[str, list[str]]): One of MindRecord files or a file list.
-        num_consumer (int, optional): The number of reader workers which load data. Default: 4.
+        num_consumer (int, optional): The number of reader workers which load data. Default: ``4`` .
             It should not be smaller than 1 or larger than the number of processor cores.
 
     Raises:
         ParamValueError: If `file_name` , `num_consumer` or columns is invalid.
         MRMInitSegmentError: If failed to initialize ShardSegment.
+
+    Examples:
+        >>> from mindspore.mindrecord import MindPage
+        >>>
+        >>> mindrecord_file = "/path/to/mindrecord/file"
+        >>> mind_page = MindPage(mindrecord_file)
+        >>>
+        >>> # get all the index fields
+        >>> fields = mind_page.candidate_fields
+        >>>
+        >>> # set the field to be retrieved
+        >>> mind_page.category_field = "file_name"
+        >>>
+        >>> # get all the group info
+        >>> info = mind_page.read_category_info()
+        >>>
+        >>> # get the row by id which is from category info
+        >>> row_by_id = mind_page.read_at_page_by_id(0, 0, 1)
+        >>>
+        >>> # get the row by name which is from category info
+        >>> row_by_name = mind_page.read_at_page_by_name("8.jpg", 0, 1)
     """
     @check_parameter
     def __init__(self, file_name, num_consumer=4):
@@ -49,6 +69,9 @@ class MindPage:
         """
         Return candidate category fields.
 
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
+
         Returns:
             list[str], by which data could be grouped.
         """
@@ -59,48 +82,22 @@ class MindPage:
         """
         Getter function for category fields.
 
-        Returns:
-            list[str], by which data could be grouped.
-        """
-        return self._category_field
-
-    def get_category_fields(self):
-        """
-        Return candidate category fields.
-
-        Returns:
-            list[str], by which data could be grouped.
-        """
-        logger.warning("WARN_DEPRECATED: The usage of get_category_fields is deprecated."
-                       " Please use candidate_fields")
-        return self.candidate_fields
-
-    def set_category_field(self, category_field):
-        """
-        Set category field for reading.
-
         Note:
-            Should be a candidate category field.
-
-        Args:
-            category_field (str): String of category field name.
+            Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
 
         Returns:
-            MSRStatus, SUCCESS or FAILED.
+            list[str], by which data could be grouped.
         """
-        logger.warning("WARN_DEPRECATED: The usage of set_category_field is deprecated."
-                       " Please use category_field")
-        if not category_field or not isinstance(category_field, str):
-            raise ParamTypeError('category_fields', 'str')
-        if category_field not in self._candidate_fields:
-            raise MRMDefineCategoryError("Field '{}' is not a candidate category field.".format(category_field))
-        return self._segment.set_category_field(category_field)
+        return self._category_field
 
     @category_field.setter
     def category_field(self, category_field):
         """
         Setter function for category field.
 
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
+
         Returns:
             MSRStatus, SUCCESS or FAILED.
         """
@@ -115,6 +112,9 @@ class MindPage:
         """
         Return category information when data is grouped by indicated category field.
 
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
+
         Returns:
             str, description of group information.
 
@@ -127,6 +127,9 @@ class MindPage:
         """
         Query by category id in pagination.
 
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
+
         Args:
              category_id (int): Category id, referred to the return of `read_category_info` .
              page (int): Index of page.
@@ -152,6 +155,9 @@ class MindPage:
         """
         Query by category name in pagination.
 
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
+
         Args:
             category_name (str): String of category field's value,
                 referred to the return of `read_category_info` .

mindspore/mindrecord/shardreader.py

@@ -38,7 +38,7 @@ class ShardReader:
            file_name (str, list[str]): File names of MindRecord File.
            num_consumer (int): Number of worker threads which load data in parallel. Default: 4.
            columns (list[str]): List of fields which correspond data would be read.
-           operator(int): Reserved parameter for operators. Default: None.
+           operator(int): Reserved parameter for operators. Default: ``None``.
 
         Returns:
             MSRStatus, SUCCESS or FAILED.

mindspore/mindrecord/shardsegment.py

@@ -43,7 +43,7 @@ class ShardSegment:
             file_name (str, list[str]): File names of MindRecord File.
             num_consumer (int): Number of worker threads which load data in parallel. Default: 4.
             columns (list[str]): List of fields which correspond data would be read.
-            operator(int): Reserved parameter for operators. Default: None.
+            operator(int): Reserved parameter for operators. Default: ``None``.
 
         Returns:
             MSRStatus, SUCCESS or FAILED.

mindspore/mindrecord/tools/cifar100_to_mr.py

@@ -36,11 +36,8 @@ __all__ = ['Cifar100ToMR']
 
 class Cifar100ToMR:
     """
-    A class to transform from cifar100 to MindRecord.
-
-    Note:
-        For details about Examples, please refer to `Converting the CIFAR-10 Dataset <https://
-        www.mindspore.cn/tutorials/en/r2.0/advanced/dataset/record.html#converting-the-cifar-10-dataset>`_ .
+    A class to transform from cifar100 which needs to be a Python version with a name
+    similar to: ``cifar-100-python.tar.gz`` to MindRecord.
 
     Args:
         source (str): The cifar100 directory to be transformed.
@@ -49,6 +46,14 @@ class Cifar100ToMR:
 
     Raises:
         ValueError: If source or destination is invalid.
+
+    Examples:
+        >>> from mindspore.mindrecord import Cifar100ToMR
+        >>>
+        >>> cifar100_dir = "/path/to/cifar100"
+        >>> mindrecord_file = "/path/to/mindrecord/file"
+        >>> cifar100_to_mr = Cifar100ToMR(cifar100_dir, mindrecord_file)
+        >>> status = cifar100_to_mr.transform()
     """
 
     def __init__(self, source, destination):
@@ -73,18 +78,8 @@ class Cifar100ToMR:
         self.destination = destination
         self.writer = None
 
+    # pylint: disable=missing-docstring
     def run(self, fields=None):
-        """
-        Execute transformation from cifar100 to MindRecord.
-
-        Args:
-            fields (list[str], optional):
-                A list of index field, e.g.["fine_label", "coarse_label"]. Default: None. For index
-                field settings, please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .
-
-        Returns:
-            MSRStatus, SUCCESS or FAILED.
-        """
         if fields and not isinstance(fields, list):
             raise ValueError("The parameter fields should be None or list")
 
@@ -116,15 +111,27 @@ class Cifar100ToMR:
 
     def transform(self, fields=None):
         """
-        Encapsulate the :func:`mindspore.mindrecord.Cifar100ToMR.run` function to exit normally.
+        Execute transformation from cifar100 to MindRecord.
+
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.Cifar100ToMR` .
 
         Args:
             fields (list[str], optional):
-                A list of index field, e.g.["fine_label", "coarse_label"]. Default: None. For index
+                A list of index field, e.g.["fine_label", "coarse_label"]. Default: ``None`` . For index
                 field settings, please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .
 
         Returns:
             MSRStatus, SUCCESS or FAILED.
+
+        Raises:
+            ParamTypeError: If index field is invalid.
+            MRMOpenError: If failed to open MindRecord file.
+            MRMValidateDataError: If data does not match blob fields.
+            MRMSetHeaderError: If failed to set header.
+            MRMWriteDatasetError: If failed to write dataset.
+            TypeError: If `parallel_writer` is not bool.
+            ValueError: If parameter `fields` is invalid.
         """
 
         t = ExceptionThread(target=self.run, kwargs={'fields': fields})