mindspore 2.4.0__cp310-cp310-macosx_10_15_x86_64.whl

mindspore/mindrecord/mindpage.py

@@ -0,0 +1,210 @@
+# Copyright 2019-2021 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""
+This module is to support reading page from MindRecord.
+"""
+
+from .shardsegment import ShardSegment
+from .shardutils import check_parameter
+from .common.exceptions import ParamValueError, ParamTypeError, MRMDefineCategoryError
+from .config import _get_enc_key, _get_dec_mode, decrypt, verify_file_hash
+
+__all__ = ['MindPage']
+
+
+class MindPage:
+    """
+    Class to read MindRecord files in pagination.
+
+    Args:
+        file_name (Union[str, list[str]]): One of MindRecord files or a file list.
+        num_consumer (int, optional): The number of reader workers which load data. Default: ``4`` .
+            It should not be smaller than 1 or larger than the number of processor cores.
+
+    Raises:
+        ParamValueError: If `file_name` is not type str or list[str].
+        ParamValueError: If `num_consumer` is not type int.
+
+    Examples:
+        >>> from mindspore.mindrecord import MindPage
+        >>>
+        >>> mindrecord_file = "/path/to/mindrecord/file"
+        >>> mind_page = MindPage(mindrecord_file)
+        >>>
+        >>> # get all the index fields
+        >>> fields = mind_page.candidate_fields
+        >>>
+        >>> # set the field to be retrieved
+        >>> mind_page.category_field = "file_name"
+        >>>
+        >>> # get all the group info
+        >>> info = mind_page.read_category_info()
+        >>>
+        >>> # get the row by id which is from category info
+        >>> row_by_id = mind_page.read_at_page_by_id(0, 0, 1)
+        >>>
+        >>> # get the row by name which is from category info
+        >>> row_by_name = mind_page.read_at_page_by_name("8.jpg", 0, 1)
+    """
+    @check_parameter
+    def __init__(self, file_name, num_consumer=4):
+        self._segment = ShardSegment()
+
+        if isinstance(file_name, str):
+            # decrypt the data file and index file
+            index_file_name = file_name + ".db"
+            decrypt_filename = decrypt(file_name, _get_enc_key(), _get_dec_mode())
+            file_name = decrypt_filename
+            decrypt(index_file_name, _get_enc_key(), _get_dec_mode())
+
+            # verify integrity check
+            verify_file_hash(file_name)
+            verify_file_hash(file_name + ".db")
+        else:
+            file_names_decrypted = []
+            for item in file_name:
+                # decrypt the data file and index file
+                index_file_name = item + ".db"
+                decrypt_filename = decrypt(item, _get_enc_key(), _get_dec_mode())
+                file_names_decrypted.append(decrypt_filename)
+                decrypt(index_file_name, _get_enc_key(), _get_dec_mode())
+
+                # verify integrity check
+                verify_file_hash(decrypt_filename)
+                verify_file_hash(decrypt_filename + ".db")
+            file_name = file_names_decrypted
+
+        self._segment.open(file_name, num_consumer)
+        self._category_field = None
+        self._candidate_fields = [field[:field.rfind('_')] for field in self._segment.get_category_fields()]
+
+    @property
+    def candidate_fields(self):
+        """
+        Return candidate category fields.
+
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
+
+        Returns:
+            list[str], by which data could be grouped.
+        """
+        return self._candidate_fields
+
+    @property
+    def category_field(self):
+        """
+        Setter / Getter function for category fields.
+
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
+
+        Returns:
+            list[str], by which data could be grouped.
+        """
+        return self._category_field
+
+    @category_field.setter
+    def category_field(self, category_field):
+        """
+        Setter function for category field.
+
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
+        """
+        if not category_field or not isinstance(category_field, str):
+            raise ParamTypeError('category_fields', 'str')
+        if category_field not in self._candidate_fields:
+            raise MRMDefineCategoryError("Field '{}' is not a candidate category field.".format(category_field))
+        self._category_field = category_field
+        return self._segment.set_category_field(self._category_field)
+
+    def read_category_info(self):
+        """
+        Return category information when data is grouped by indicated category field.
+
+        The result is similar to the following, where `key` represents the index field and `categories` represent
+        statistical information for the index.
+
+        .. code-block::
+
+            {"categories":[{"count":1,"id":0,"name":"0.jpg"},
+                           {"count":1,"id":1,"name":"1.jpg"},
+                           {"count":1,"id":2,"name":"2.jpg"},
+                           {"count":1,"id":3,"name":"3.jpg"}],
+             "key":"file_name_0"}
+
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
+
+        Returns:
+            str, description of group information.
+
+        Raises:
+            MRMReadCategoryInfoError: If failed to read category information.
+        """
+        return self._segment.read_category_info()
+
+    def read_at_page_by_id(self, category_id, page, num_row):
+        """
+        Query by category id in pagination.
+
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
+
+        Args:
+             category_id (int): Category id, referred to the return of `read_category_info` .
+             page (int): Index of page.
+             num_row (int): Number of rows in a page.
+
+        Returns:
+            list[dict], data queried by category id.
+
+        Raises:
+            ParamValueError: If any parameter is invalid.
+            MRMFetchDataError: If failed to fetch data by category.
+            MRMUnsupportedSchemaError: If schema is invalid.
+        """
+        if not isinstance(category_id, int) or category_id < 0:
+            raise ParamValueError("Category id should be int and greater than or equal to 0.")
+        if not isinstance(page, int) or page < 0:
+            raise ParamValueError("Page should be int and greater than or equal to 0.")
+        if not isinstance(num_row, int) or num_row <= 0:
+            raise ParamValueError("num_row should be int and greater than 0.")
+        return self._segment.read_at_page_by_id(category_id, page, num_row)
+
+    def read_at_page_by_name(self, category_name, page, num_row):
+        """
+        Query by category name in pagination.
+
+        Note:
+            Please refer to the Examples of :class:`mindspore.mindrecord.MindPage` .
+
+        Args:
+            category_name (str): String of category field's value,
+                referred to the return of `read_category_info` .
+            page (int): Index of page.
+            num_row (int): Number of row in a page.
+
+        Returns:
+            list[dict], data queried by category name.
+        """
+        if not isinstance(category_name, str):
+            raise ParamValueError("Category name should be str.")
+        if not isinstance(page, int) or page < 0:
+            raise ParamValueError("Page should be int and greater than or equal to 0.")
+        if not isinstance(num_row, int) or num_row <= 0:
+            raise ParamValueError("num_row should be int and greater than 0.")
+        return self._segment.read_at_page_by_name(category_name, page, num_row)

mindspore/mindrecord/shardheader.py

@@ -0,0 +1,141 @@
+# Copyright 2019 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""
+This module is to write data into mindrecord.
+"""
+import mindspore._c_mindrecord as ms
+from mindspore import log as logger
+from .common.exceptions import MRMAddSchemaError, MRMAddIndexError, MRMBuildSchemaError, MRMGetMetaError
+
+__all__ = ['ShardHeader']
+
+
+class ShardHeader:
+    """
+    Wrapper class which is represent ShardHeader class in c++ module.
+
+    The class would store meta data of MindRecord File.
+    """
+    def __init__(self, header=None):
+        if header:
+            self._header = header
+        else:
+            self._header = ms.ShardHeader()
+
+    @property
+    def header(self):
+        """Getter of header"""
+        return self._header
+
+    @property
+    def blob_fields(self):
+        """Getter of blob fields"""
+        return self._get_blob_fields()
+
+    @property
+    def schema(self):
+        """Getter of schema"""
+        return self._get_schema()
+
+    def add_schema(self, schema):
+        """
+        Add object of ShardSchema.
+
+        Args:
+          schema (ShardSchema): Object of ShardSchema.
+
+        Returns:
+           int, schema id.
+
+        Raises:
+            MRMAddSchemaError: If failed to add schema.
+        """
+        schema_id = self._header.add_schema(schema)
+        if schema_id == -1:
+            logger.critical("Failed to add schema.")
+            raise MRMAddSchemaError
+        return schema_id
+
+    def add_index_fields(self, index_fields):
+        """
+        Add object of ShardSchema.
+
+        Args:
+          index_fields (list[str]):
+
+        Returns:
+            MSRStatus, SUCCESS or FAILED.
+
+        Raises:
+            MRMAddSchemaError: If failed to add index field.
+        """
+        ret = self._header.add_index_fields(index_fields)
+        if ret != ms.MSRStatus.SUCCESS:
+            logger.critical("Failed to add index field.")
+            raise MRMAddIndexError
+        return ret
+
+    def build_schema(self, content, desc=None):
+        """
+        Build raw schema to generate schema object.
+
+        Args:
+            content (dict): Dict of user defined schema.
+            desc (str,optional): String of schema description.
+
+        Returns:
+           Class ShardSchema.
+
+        Raises:
+            MRMBuildSchemaError: If failed to build schema.
+        """
+        desc = desc if desc else ""
+        schema = ms.Schema.build(desc, content)
+        if not schema:
+            logger.critical("Failed to add build schema.")
+            raise MRMBuildSchemaError
+        return schema
+
+    def _get_schema(self):
+        """
+        Get schema info.
+
+        Returns:
+             List of dict.
+        """
+        return self._get_meta()['schema']
+
+    def _get_blob_fields(self):
+        """
+        Get blob fields info.
+
+        Returns:
+             List of dict.
+        """
+        return self._get_meta()['blob_fields']
+
+    def _get_meta(self):
+        """
+       Get metadata including schema, blob fields .etc.
+
+       Returns:
+            List of dict.
+       """
+        ret = self._header.get_meta()
+        if ret and len(ret) == 1:
+            return ret[0].get_schema_content()
+
+        logger.critical("Failed to get meta info.")
+        raise MRMGetMetaError

mindspore/mindrecord/shardindexgenerator.py

@@ -0,0 +1,74 @@
+# Copyright 2019 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""
+This module is to  write data into mindrecord.
+"""
+import mindspore._c_mindrecord as ms
+from mindspore import log as logger
+from .common.exceptions import MRMIndexGeneratorError, MRMGenerateIndexError
+
+__all__ = ['ShardIndexGenerator']
+
+
+class ShardIndexGenerator:
+    """
+    Wrapper class which is represent ShardIndexGenerator class in c++ module.
+
+    The class would generate db files for accelerating reading.
+
+    Args:
+        path (str): Absolute path of MindRecord File.
+        append (bool): If True, open existed MindRecord Files for appending, or create new MindRecord Files.
+
+    Raises:
+        MRMIndexGeneratorError: If failed to create index generator.
+    """
+    def __init__(self, path, append=False):
+        self._generator = ms.ShardIndexGenerator(path, append)
+        if not self._generator:
+            logger.critical("Failed to create index generator.")
+            raise MRMIndexGeneratorError
+
+    def build(self):
+        """
+        Build index generator.
+
+        Returns:
+            MSRStatus, SUCCESS or FAILED.
+
+        Raises:
+            MRMGenerateIndexError: If failed to build index generator.
+        """
+        ret = self._generator.build()
+        if ret != ms.MSRStatus.SUCCESS:
+            logger.critical("Failed to build index generator.")
+            raise MRMGenerateIndexError
+        return ret
+
+    def write_to_db(self):
+        """
+        Create index field in table for reading data.
+
+        Returns:
+            MSRStatus, SUCCESS or FAILED.
+
+        Raises:
+            MRMGenerateIndexError: If failed to write to database.
+        """
+        ret = self._generator.write_to_db()
+        if ret != ms.MSRStatus.SUCCESS:
+            logger.critical("Failed to write to database.")
+            raise MRMGenerateIndexError
+        return ret

mindspore/mindrecord/shardreader.py

@@ -0,0 +1,117 @@
+# Copyright 2019 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""
+This module is to read data from mindrecord.
+"""
+import mindspore._c_mindrecord as ms
+from mindspore import log as logger
+from .common.exceptions import MRMOpenError, MRMLaunchError
+__all__ = ['ShardReader']
+
+
+class ShardReader:
+    """
+    Wrapper class which is represent ShardReader class in c++ module.
+
+    The class would read a batch of data from MindRecord File series.
+    """
+    def __init__(self):
+        self._reader = ms.ShardReader()
+
+    def open(self, file_name, num_consumer=4, columns=None, operator=None):
+        """
+        Open file and prepare to read MindRecord File.
+
+        Args:
+           file_name (str, list[str]): File names of MindRecord File.
+           num_consumer (int): Number of worker threads which load data in parallel. Default: 4.
+           columns (list[str]): List of fields which correspond data would be read.
+           operator(int): Reserved parameter for operators. Default: ``None``.
+
+        Returns:
+            MSRStatus, SUCCESS or FAILED.
+
+        Raises:
+            MRMOpenError: If failed to open MindRecord File.
+        """
+        columns = columns if columns else []
+        operator = operator if operator else []
+        if isinstance(file_name, list):
+            load_dataset = False
+        else:
+            load_dataset = True
+            file_name = [file_name]
+        ret = self._reader.open(file_name, load_dataset, num_consumer, columns, operator)
+        if ret != ms.MSRStatus.SUCCESS:
+            logger.critical("Failed to open {}.".format(file_name))
+            self.close()
+            raise MRMOpenError
+        return ret
+
+    def launch(self):
+        """
+        Launch the worker threads to load data.
+
+        Returns:
+            MSRStatus, SUCCESS or FAILED.
+
+        Raises:
+            MRMLaunchError: If failed to launch worker threads.
+        """
+        ret = self._reader.launch()
+        if ret != ms.MSRStatus.SUCCESS:
+            logger.critical("Failed to launch worker threads.")
+            raise MRMLaunchError
+        return ret
+
+    def get_next(self):
+        """
+        Return a batch of data including blob data and raw data.
+
+        Returns:
+           list of dict.
+        """
+        return self._reader.get_next()
+
+    def get_blob_fields(self):
+        """
+        Return blob fields of MindRecord.
+
+        Returns:
+            list of str.
+        """
+        return self._reader.get_blob_fields()
+
+    def get_header(self):
+        """
+        Return header of MindRecord.
+
+        Returns:
+            pointer object refer to header.
+        """
+        return self._reader.get_header()
+
+    def close(self):
+        """close MindRecord File."""
+        self._reader.close()
+
+    def len(self):
+        """
+        Get the number of the samples in MindRecord.
+
+        Returns:
+            int, the number of the samples in MindRecord.
+        """
+        return self._reader.len()

mindspore/mindrecord/shardsegment.py

@@ -0,0 +1,128 @@
+# Copyright 2019 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""
+This module is to read page from mindrecord.
+"""
+import mindspore._c_mindrecord as ms
+from mindspore import log as logger
+from .shardutils import populate_data, SUCCESS
+from .shardheader import ShardHeader
+
+__all__ = ['ShardSegment']
+
+
+class ShardSegment:
+    """
+    Wrapper class which is represent ShardSegment class in c++ module.
+
+    The class would query data from MindRecord File in pagination.
+
+    """
+    def __init__(self):
+        self._segment = ms.ShardSegment()
+        self._header = None
+        self._columns = None
+
+    def open(self, file_name, num_consumer=4, columns=None, operator=None):
+        """
+        Initialize the ShardSegment.
+
+        Args:
+            file_name (str, list[str]): File names of MindRecord File.
+            num_consumer (int): Number of worker threads which load data in parallel. Default: 4.
+            columns (list[str]): List of fields which correspond data would be read.
+            operator(int): Reserved parameter for operators. Default: ``None``.
+
+        Returns:
+            MSRStatus, SUCCESS or FAILED.
+
+        Raises:
+            MRMOpenError: If failed to open MindRecord File.
+        """
+        self._columns = columns if columns else []
+        operator = operator if operator else []
+        if isinstance(file_name, list):
+            load_dataset = False
+        else:
+            load_dataset = True
+            file_name = [file_name]
+        ret = self._segment.open(file_name, load_dataset, num_consumer, self._columns, operator)
+        if ret != SUCCESS:
+            logger.critical("Failed to open {}.".format(file_name))
+            raise MRMOpenError
+        self._header = ShardHeader(self._segment.get_header())
+        return ret
+
+    def get_category_fields(self):
+        """
+        Get candidate category fields.
+
+        Returns:
+            list[str], by which data could be grouped.
+
+        """
+        return self._segment.get_category_fields()
+
+    def set_category_field(self, category_field):
+        """Select one category field to use."""
+        return self._segment.set_category_field(category_field)
+
+    def read_category_info(self):
+        """
+        Get the group info by the current category field.
+
+        Returns:
+            str, description fo group information.
+
+        """
+        return self._segment.read_category_info()
+
+    def read_at_page_by_id(self, category_id, page, num_row):
+        """
+        Get the data of some page by category id.
+
+        Args:
+            category_id (int): Category id, referred to the return of read_category_info.
+            page (int): Index of page.
+            num_row (int): Number of rows in a page.
+
+        Returns:
+            list[dict]
+
+        Raises:
+            MRMUnsupportedSchemaError: If schema is invalid.
+        """
+        data = self._segment.read_at_page_by_id(category_id, page, num_row)
+        return [populate_data(raw, blob, self._columns, self._header.blob_fields,
+                              self._header.schema) for blob, raw in data]
+
+    def read_at_page_by_name(self, category_name, page, num_row):
+        """
+        Get the data of some page by category name.
+
+        Args:
+            category_name (str): Category name, referred to the return of read_category_info.
+            page (int): Index of page.
+            num_row (int): Number of rows in a page.
+
+        Returns:
+            list[dict]
+
+        Raises:
+            MRMUnsupportedSchemaError: If schema is invalid.
+        """
+        data = self._segment.read_at_page_by_name(category_name, page, num_row)
+        return [populate_data(raw, blob, self._columns, self._header.blob_fields,
+                              self._header.schema) for blob, raw in data]