mindspore 1.10.0__cp37-none-any.whl → 2.0.0rc1__cp37-none-any.whl

mindspore/dataset/engine/datasets_standard_format.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2022 Huawei Technologies Co., Ltd
+# Copyright 2019-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -26,6 +26,7 @@ import platform
 import numpy as np
 
 import mindspore._c_dataengine as cde
+from mindspore import log as logger
 
 from .datasets import UnionBaseDataset, SourceDataset, MappableDataset, Shuffle, Schema, \
     shuffle_to_shuffle_mode, shuffle_to_bool
@@ -48,18 +49,19 @@ class CSVDataset(SourceDataset, UnionBaseDataset):
     Args:
         dataset_files (Union[str, list[str]]): String or list of files to be read or glob strings to search
             for a pattern of files. The list will be sorted in a lexicographical order.
-        field_delim (str, optional): A string that indicates the char delimiter to separate fields (default=',').
-        column_defaults (list, optional): List of default values for the CSV field (default=None). Each item
+        field_delim (str, optional): A string that indicates the char delimiter to separate fields. Default: ','.
+        column_defaults (list, optional): List of default values for the CSV field. Default: None. Each item
             in the list is either a valid type (float, int, or string). If this is not provided, treats all
             columns as string type.
-        column_names (list[str], optional): List of column names of the dataset (default=None). If this
+        column_names (list[str], optional): List of column names of the dataset. Default: None. If this
             is not provided, infers the column_names from the first row of CSV file.
-        num_samples (int, optional): The number of samples to be included in the dataset
-            (default=None, will include all images).
-        num_parallel_workers (int, optional): Number of workers to read the data
-            (default=None, number set in the config).
-        shuffle (Union[bool, Shuffle], optional): Perform reshuffling of the data every epoch
-            (default=Shuffle.GLOBAL). Bool type and Shuffle enum are both supported to pass in.
+        num_samples (int, optional): The number of samples to be included in the dataset.
+            Default: None, will include all images.
+        num_parallel_workers (int, optional): Number of worker threads to read the data.
+            Default: None, will use global default workers(8), it can be set
+            by `mindspore.dataset.config.set_num_parallel_workers` .
+        shuffle (Union[bool, Shuffle], optional): Perform reshuffling of the data every epoch.
+            Default: Shuffle.GLOBAL. Bool type and Shuffle enum are both supported to pass in.
             If shuffle is False, no shuffling will be performed.
             If shuffle is True, performs global shuffle.
             There are three levels of shuffling, desired shuffle enum defined by mindspore.dataset.Shuffle.
@@ -68,13 +70,13 @@ class CSVDataset(SourceDataset, UnionBaseDataset):
 
             - Shuffle.FILES: Shuffle files only.
 
-        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).
+        num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
             When this argument is specified, `num_samples` reflects the maximum sample number of per shard.
-        shard_id (int, optional): The shard ID within `num_shards` (default=None). This
+        shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
             argument can only be specified when `num_shards` is also specified.
         cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
-            `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r1.10/dataset/cache.html>`_
-            (default=None, which means no cache is used).
+            `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.0/dataset/cache.html>`_ .
+            Default: None, which means no cache is used.
 
     Raises:
         RuntimeError: If dataset_files are not valid or do not exist.
@@ -82,7 +84,7 @@ class CSVDataset(SourceDataset, UnionBaseDataset):
         ValueError: If `num_parallel_workers` exceeds the max thread numbers.
         RuntimeError: If `num_shards` is specified but `shard_id` is None.
         RuntimeError: If `shard_id` is specified but `num_shards` is None.
-        ValueError: If `shard_id` is invalid (< 0 or >= `num_shards`).
+        ValueError: If `shard_id` is not in range of [0, `num_shards` ).
 
     Examples:
         >>> csv_dataset_dir = ["/path/to/csv_dataset_file"] # contains 1 or multiple csv files
@@ -116,10 +118,12 @@ class MindDataset(MappableDataset, UnionBaseDataset):
             a file name of one component of a mindrecord source, other files with identical source
             in the same path will be found and loaded automatically. If dataset_file is a list,
             it represents for a list of dataset files to be read directly.
-        columns_list (list[str], optional): List of columns to be read (default=None).
-        num_parallel_workers (int, optional): The number of readers (default=None).
-        shuffle (Union[bool, Shuffle], optional): Perform reshuffling of the data every epoch
-            (default=None, performs global shuffle). Bool type and Shuffle enum are both supported to pass in.
+        columns_list (list[str], optional): List of columns to be read. Default: None, read all columns.
+        num_parallel_workers (int, optional): Number of worker threads to read the data.
+            Default: None, will use global default workers(8), it can be set
+            by `mindspore.dataset.config.set_num_parallel_workers` .
+        shuffle (Union[bool, Shuffle], optional): Perform reshuffling of the data every epoch.
+            Default: None, performs global shuffle. Bool type and Shuffle enum are both supported to pass in.
             If shuffle is False, no shuffling will be performed.
             If shuffle is True, performs global shuffle.
             There are three levels of shuffling, desired shuffle enum defined by mindspore.dataset.Shuffle.
@@ -130,33 +134,33 @@ class MindDataset(MappableDataset, UnionBaseDataset):
 
             - Shuffle.INFILE: Keep the file sequence the same but shuffle the data within each file.
 
-        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).
-            When this argument is specified, 'num_samples' reflects the maximum sample number of per shard.
-        shard_id (int, optional): The shard ID within `num_shards` (default=None). This
+        num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
+            When this argument is specified, `num_samples` reflects the maximum sample number of per shard.
+        shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
             argument can only be specified when `num_shards` is also specified.
         sampler (Sampler, optional): Object used to choose samples from the
-            dataset (default=None, sampler is exclusive
-            with shuffle and block_reader). Support list: SubsetRandomSampler,
+            dataset. Default: None, sampler is exclusive
+            with shuffle and block_reader. Support list: SubsetRandomSampler,
             PkSampler, RandomSampler, SequentialSampler, DistributedSampler.
         padded_sample (dict, optional): Samples will be appended to dataset, where
             keys are the same as column_list.
         num_padded (int, optional): Number of padding samples. Dataset size
             plus num_padded should be divisible by num_shards.
-        num_samples (int, optional): The number of samples to be included in the dataset
-            (default=None, all samples).
+        num_samples (int, optional): The number of samples to be included in the dataset.
+            Default: None, all samples.
         cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
-            `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r1.10/dataset/cache.html>`_
-            (default=None, which means no cache is used).
+            `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.0/dataset/cache.html>`_ .
+            Default: None, which means no cache is used.
 
     Raises:
         ValueError: If dataset_files are not valid or do not exist.
         ValueError: If `num_parallel_workers` exceeds the max thread numbers.
         RuntimeError: If `num_shards` is specified but `shard_id` is None.
         RuntimeError: If `shard_id` is specified but `num_shards` is None.
-        ValueError: If `shard_id` is invalid (< 0 or >= `num_shards`).
+        ValueError: If `shard_id` is not in range of [0, `num_shards` ).
 
     Note:
-        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.
+        - This dataset can take in a `sampler` . `sampler` and `shuffle` are mutually exclusive.
           The table below shows what input arguments are allowed and their expected behavior.
 
     .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`
@@ -247,45 +251,60 @@ class TFRecordDataset(SourceDataset, UnionBaseDataset):
 
     Args:
         dataset_files (Union[str, list[str]]): String or list of files to be read or glob strings to search for a
-            pattern of files. The list will be sorted in a lexicographical order.
-        schema (Union[str, Schema], optional): Path to the JSON schema file or schema object (default=None).
-            If the schema is not provided, the meta data from the TFData file is considered the schema.
-        columns_list (list[str], optional): List of columns to be read (default=None, read all columns).
-        num_samples (int, optional): The number of samples (rows) to be included in the dataset (default=None).
-            If num_samples is None and numRows(parsed from schema) does not exist, read the full dataset;
-            If num_samples is None and numRows(parsed from schema) is greater than 0, read numRows rows;
-            If both num_samples and numRows(parsed from schema) are greater than 0, read num_samples rows.
-        num_parallel_workers (int, optional): Number of workers to read the data
-            (default=None, number set in the config).
-        shuffle (Union[bool, Shuffle], optional): Perform reshuffling of the data every epoch
-            (default=Shuffle.GLOBAL). Bool type and Shuffle enum are both supported to pass in.
-            If shuffle is False, no shuffling will be performed.
-            If shuffle is True, performs global shuffle.
+            pattern of files. The list will be sorted in lexicographical order.
+        schema (Union[str, Schema], optional): Data format policy, which specifies the data types and shapes of the data
+            column to be read. Both JSON file path and objects constructed by mindspore.dataset.Schema are acceptable.
+            Default: None.
+        columns_list (list[str], optional): List of columns to be read. Default: None, read all columns.
+        num_samples (int, optional): The number of samples (rows) to be included in the dataset.
+            Default: None.
+            Processing priority for `num_samples` is as the following:
+            1. If `num_samples` is greater than 0, read `num_samples` rows.
+            2. Otherwise, if numRows (parsed from `schema` ) is greater than 0, read numRows rows.
+            3. Otherwise, read the full dataset.
+            `num_samples` or numRows (parsed from `schema` ) will be interpreted as number of rows per shard.
+            It is highly recommended to provide `num_samples` or numRows (parsed from `schema` )
+            when `compression_type` is "GZIP" or "ZLIB" to avoid performance degradation due to multiple
+            decompressions of the same file to obtain the file size.
+        num_parallel_workers (int, optional): Number of worker threads to read the data.
+            Default: None, will use global default workers(8), it can be set
+            by `mindspore.dataset.config.set_num_parallel_workers` .
+        shuffle (Union[bool, Shuffle], optional): Perform reshuffling of the data every epoch.
+            Default: Shuffle.GLOBAL. Bool type and Shuffle enum are both supported to pass in.
+            If `shuffle` is False, no shuffling will be performed.
+            If `shuffle` is True, perform global shuffle.
             There are three levels of shuffling, desired shuffle enum defined by mindspore.dataset.Shuffle.
 
-            - Shuffle.GLOBAL: Shuffle both the files and samples, same as setting shuffle to True.
+            - Shuffle.GLOBAL: Shuffle both the files and samples, same as setting `shuffle` to True.
 
             - Shuffle.FILES: Shuffle files only.
 
         num_shards (int, optional): Number of shards that the dataset will be divided
-            into (default=None). When this argument is specified, `num_samples` reflects
-            the maximum sample number of per shard.
-        shard_id (int, optional): The shard ID within `num_shards` (default=None). This
+            into. Default: None. When this argument is specified, `num_samples` reflects
+            the maximum sample number per shard.
+        shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
             argument can only be specified when `num_shards` is also specified.
-        shard_equal_rows (bool, optional): Get equal rows for all shards(default=False). If shard_equal_rows
-            is false, number of rows of each shard may be not equal, and may lead to a failure in distributed training.
-            When the number of samples of per TFRecord file are not equal, it is suggested to set to true.
+        shard_equal_rows (bool, optional): Get equal rows for all shards. Default: False. If `shard_equal_rows`
+            is False, the number of rows of each shard may not be equal, and may lead to a failure in distributed
+            training. When the number of samples per TFRecord file are not equal, it is suggested to set it to True.
             This argument should only be specified when `num_shards` is also specified.
+            When `compression_type` is not None, and `num_samples` or numRows (parsed from `schema` ) is provided,
+            `shard_equal_rows` will be implied as true.
         cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
-            `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r1.10/dataset/cache.html>`_
-            (default=None, which means no cache is used).
+            `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.0/dataset/cache.html>`_ .
+            Default: None, which means no cache is used.
+        compression_type (str, optional): The type of compression used for all files, must be either '', 'GZIP', or
+            'ZLIB'. Default: None, as in empty string.
 
     Raises:
         ValueError: If dataset_files are not valid or do not exist.
         ValueError: If `num_parallel_workers` exceeds the max thread numbers.
         RuntimeError: If `num_shards` is specified but `shard_id` is None.
         RuntimeError: If `shard_id` is specified but `num_shards` is None.
-        ValueError: If `shard_id` is invalid (< 0 or >= `num_shards`).
+        ValueError: If `shard_id` is not in range of [0, `num_shards` ).
+        ValueError: If `compression_type` is invalid (other than '', 'GZIP', or 'ZLIB').
+        ValueError: If `compression_type` is provided, but the number of dataset files < `num_shards` .
+        ValueError: If `num_samples` < 0.
 
     Examples:
         >>> from mindspore import dtype as mstype
@@ -302,13 +321,14 @@ class TFRecordDataset(SourceDataset, UnionBaseDataset):
         >>> schema.add_column(name='col_1d', de_type=mstype.int64, shape=[2])
         >>> dataset = ds.TFRecordDataset(dataset_files=tfrecord_dataset_dir, schema=schema)
         >>>
-        >>> # 3) Get all rows from tfrecord_dataset_dir with schema file.
+        >>> # 3) Get all rows from tfrecord_dataset_dir with the schema file.
         >>> dataset = ds.TFRecordDataset(dataset_files=tfrecord_dataset_dir, schema=tfrecord_schema_file)
     """
 
     @check_tfrecorddataset
     def __init__(self, dataset_files, schema=None, columns_list=None, num_samples=None, num_parallel_workers=None,
-                 shuffle=Shuffle.GLOBAL, num_shards=None, shard_id=None, shard_equal_rows=False, cache=None):
+                 shuffle=Shuffle.GLOBAL, num_shards=None, shard_id=None, shard_equal_rows=False,
+                 cache=None, compression_type=None):
         super().__init__(num_parallel_workers=num_parallel_workers, num_samples=num_samples, shuffle=shuffle,
                          num_shards=num_shards, shard_id=shard_id, cache=cache)
         self.dataset_files = self._find_files(dataset_files)
@@ -317,14 +337,20 @@ class TFRecordDataset(SourceDataset, UnionBaseDataset):
         self.schema = schema
         self.columns_list = replace_none(columns_list, [])
         self.shard_equal_rows = replace_none(shard_equal_rows, False)
+        self.compression_type = replace_none(compression_type, "")
 
+        # Only take numRows from schema when num_samples is not provided
         if self.schema is not None and (self.num_samples is None or self.num_samples == 0):
             self.num_samples = Schema.get_num_rows(self.schema)
 
+        if self.compression_type in ['ZLIB', 'GZIP'] and (self.num_samples is None or self.num_samples == 0):
+            logger.warning("Since compression_type is set, but neither num_samples nor numRows (from schema file) " +
+                           "is provided, performance might be degraded.")
+
     def parse(self, children=None):
         schema = self.schema.cpp_schema if isinstance(self.schema, Schema) else self.schema
         return cde.TFRecordNode(self.dataset_files, schema, self.columns_list, self.num_samples, self.shuffle_flag,
-                                self.num_shards, self.shard_id, self.shard_equal_rows)
+                                self.num_shards, self.shard_id, self.shard_equal_rows, self.compression_type)
 
 
 class OBSMindDataset(GeneratorDataset):
@@ -346,9 +372,9 @@ class OBSMindDataset(GeneratorDataset):
         sk (str): Secret key ID of cloud storage.
         sync_obs_path (str): Remote dir path used for synchronization, users need to
             create it on cloud storage in advance. Path is in the format of s3://bucketName/objectKey.
-        columns_list (list[str], optional): List of columns to be read (default=None, read all columns).
-        shuffle (Union[bool, Shuffle], optional): Perform reshuffling of the data every epoch
-            (default=None, performs global shuffle). Bool type and Shuffle enum are both supported to pass in.
+        columns_list (list[str], optional): List of columns to be read. Default: None, read all columns.
+        shuffle (Union[bool, Shuffle], optional): Perform reshuffling of the data every epoch.
+            Default: None, performs global shuffle. Bool type and Shuffle enum are both supported to pass in.
             If shuffle is False, no shuffling will be performed.
             If shuffle is True, performs global shuffle.
             There are three levels of shuffling, desired shuffle enum defined by mindspore.dataset.Shuffle.
@@ -360,26 +386,26 @@ class OBSMindDataset(GeneratorDataset):
             - Shuffle.INFILE: Keep the file sequence the same but shuffle the data within each file.
 
         num_shards (int, optional): Number of shards that the dataset will be divided
-            into (default=None).
-        shard_id (int, optional): The shard ID within num_shards (default=None). This
-            argument can only be specified when num_shards is also specified.
-        shard_equal_rows (bool, optional): Get equal rows for all shards(default=True). If shard_equal_rows
+            into. Default: None.
+        shard_id (int, optional): The shard ID within num_shards. Default: None. This
+            argument can only be specified when `num_shards` is also specified.
+        shard_equal_rows (bool, optional): Get equal rows for all shards. Default: True. If shard_equal_rows
             is false, number of rows of each shard may be not equal, and may lead to a failure in distributed training.
             When the number of samples of per MindRecord file are not equal, it is suggested to set to true.
-            This argument should only be specified when num_shards is also specified.
+            This argument should only be specified when `num_shards` is also specified.
 
     Raises:
         RuntimeError: If `sync_obs_path` do not exist.
         ValueError: If `columns_list` is invalid.
         RuntimeError: If `num_shards` is specified but `shard_id` is None.
         RuntimeError: If `shard_id` is specified but `num_shards` is None.
-        ValueError: If `shard_id` is invalid (< 0 or >= `num_shards`).
+        ValueError: If `shard_id` is not in range of [0, `num_shards` ).
 
     Note:
         - It's necessary to create a synchronization directory on cloud storage in
           advance which be defined by parameter: `sync_obs_path` .
         - If training is offline(no cloud), it's recommended to set the
-          environment variable `BATCH_JOB_ID`.
+          environment variable `BATCH_JOB_ID` .
         - In distributed training, if there are multiple nodes(servers), all 8
           devices must be used in each node(server). If there is only one
           node(server), there is no such restriction.