mindspore 2.0.0rc1__cp38-cp38-manylinux1_x86_64.whl → 2.2.0__cp38-cp38-manylinux1_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mindspore might be problematic. Click here for more details.
- mindspore/.commit_id +1 -1
- mindspore/Third_Party_Open_Source_Software_Notice +2 -2
- mindspore/__init__.py +5 -2
- mindspore/_akg/akg/build_module.py +5 -6
- mindspore/_akg/akg/composite/build_module.py +49 -16
- mindspore/_akg/akg/composite/split_stitch.py +10 -11
- mindspore/_akg/akg/config/repository.json +195 -0
- mindspore/_akg/akg/global_configs.py +5 -1
- mindspore/_akg/akg/ms/info_version_adapt.py +67 -1
- mindspore/_akg/akg/tvm/api.py +4 -3
- mindspore/_akg/akg/tvm/autotvm/__init__.py +1 -2
- mindspore/_akg/akg/tvm/autotvm/graph_tuner/base_graph_tuner.py +1 -5
- mindspore/_akg/akg/tvm/autotvm/measure/__init__.py +1 -1
- mindspore/_akg/akg/tvm/autotvm/measure/measure.py +1 -10
- mindspore/_akg/akg/tvm/autotvm/measure/measure_methods.py +1 -372
- mindspore/_akg/akg/tvm/build_module.py +16 -1
- mindspore/_akg/akg/tvm/contrib/graph_runtime.py +0 -53
- mindspore/_akg/akg/tvm/hybrid/parser.py +7 -6
- mindspore/_akg/akg/tvm/ir_builder.py +1 -1
- mindspore/_akg/akg/tvm/module.py +1 -2
- mindspore/_akg/akg/tvm/stmt.py +2 -2
- mindspore/_akg/akg/utils/composite_op_helper.py +9 -10
- mindspore/_akg/akg/utils/kernel_exec.py +58 -260
- mindspore/_akg/akg/utils/op_dsl.py +17 -1
- mindspore/_akg/akg/utils/result_analysis.py +4 -24
- mindspore/_akg/akg/utils/tbe_codegen_utils.py +198 -0
- mindspore/_c_dataengine.cpython-38-x86_64-linux-gnu.so +0 -0
- mindspore/_c_expression.cpython-38-x86_64-linux-gnu.so +0 -0
- mindspore/_c_mindrecord.cpython-38-x86_64-linux-gnu.so +0 -0
- mindspore/_check_jit_forbidden_api.py +5 -1
- mindspore/_checkparam.py +79 -62
- mindspore/_extends/graph_kernel/__init__.py +0 -1
- mindspore/_extends/graph_kernel/model/graph_split.py +2 -0
- mindspore/_extends/graph_kernel/model/model_builder.py +9 -50
- mindspore/_extends/graph_kernel/splitter.py +1 -9
- mindspore/_extends/parallel_compile/akg_compiler/akg_process.py +128 -21
- mindspore/_extends/parallel_compile/akg_compiler/build_tbe_kernel.py +2 -2
- mindspore/_extends/parallel_compile/akg_compiler/tbe_topi.py +4 -2
- mindspore/_extends/parallel_compile/tbe_compiler/tbe_adapter.py +18 -13
- mindspore/_extends/parallel_compile/tbe_compiler/tbe_helper.py +13 -9
- mindspore/_extends/parallel_compile/tbe_compiler/tbe_job.py +1 -1
- mindspore/_extends/parallel_compile/tbe_compiler/tbe_job_manager.py +1 -1
- mindspore/_extends/parse/__init__.py +19 -17
- mindspore/_extends/parse/namespace.py +7 -36
- mindspore/_extends/parse/parser.py +375 -189
- mindspore/_extends/parse/resources.py +36 -41
- mindspore/_extends/parse/standard_method.py +350 -245
- mindspore/_extends/parse/trope.py +2 -12
- mindspore/_extends/remote/kernel_build_server.py +24 -7
- mindspore/_extends/remote/kernel_build_server_akg_v2.py +55 -0
- mindspore/_install_custom.py +43 -0
- mindspore/_mindspore_offline_debug.cpython-38-x86_64-linux-gnu.so +0 -0
- mindspore/amp.py +85 -19
- mindspore/bin/cache_admin +0 -0
- mindspore/bin/cache_server +0 -0
- mindspore/boost/base.py +2 -2
- mindspore/boost/boost.py +27 -32
- mindspore/boost/boost_cell_wrapper.py +37 -13
- mindspore/boost/grad_accumulation.py +1 -1
- mindspore/boost/grad_freeze.py +34 -6
- mindspore/boost/group_loss_scale_manager.py +15 -14
- mindspore/boost/less_batch_normalization.py +28 -3
- mindspore/common/__init__.py +15 -11
- mindspore/common/_auto_dynamic.py +68 -0
- mindspore/common/_jit_fallback_utils.py +111 -0
- mindspore/common/_register_for_adapter.py +17 -5
- mindspore/common/_register_for_tensor.py +2 -2
- mindspore/common/_stub_tensor.py +18 -15
- mindspore/common/_utils.py +31 -7
- mindspore/common/api.py +269 -101
- mindspore/common/auto_dynamic_shape.py +498 -0
- mindspore/common/dtype.py +61 -21
- mindspore/common/dump.py +9 -7
- mindspore/common/initializer.py +106 -76
- mindspore/common/jit_config.py +35 -14
- mindspore/common/lazy_inline.py +187 -0
- mindspore/common/mindir_util.py +101 -0
- mindspore/common/mutable.py +10 -13
- mindspore/common/parameter.py +246 -55
- mindspore/common/seed.py +13 -7
- mindspore/common/sparse_tensor.py +29 -33
- mindspore/common/tensor.py +907 -251
- mindspore/communication/__init__.py +7 -4
- mindspore/communication/_comm_helper.py +84 -4
- mindspore/communication/management.py +160 -88
- mindspore/config/op_info.config +99 -75
- mindspore/config/super_bar_config.json +36 -4
- mindspore/context.py +526 -219
- mindspore/dataset/__init__.py +9 -46
- mindspore/dataset/audio/__init__.py +4 -19
- mindspore/dataset/audio/transforms.py +545 -233
- mindspore/dataset/audio/utils.py +21 -18
- mindspore/dataset/callback/ds_callback.py +42 -13
- mindspore/dataset/core/config.py +158 -100
- mindspore/dataset/core/validator_helpers.py +1 -63
- mindspore/dataset/debug/debug_hook.py +45 -13
- mindspore/dataset/debug/pre_defined_hook.py +5 -5
- mindspore/dataset/engine/__init__.py +0 -5
- mindspore/dataset/engine/cache_client.py +38 -15
- mindspore/dataset/engine/datasets.py +615 -278
- mindspore/dataset/engine/datasets_audio.py +154 -283
- mindspore/dataset/engine/datasets_standard_format.py +104 -116
- mindspore/dataset/engine/datasets_text.py +443 -326
- mindspore/dataset/engine/datasets_user_defined.py +251 -164
- mindspore/dataset/engine/datasets_vision.py +839 -1443
- mindspore/dataset/engine/iterators.py +11 -4
- mindspore/dataset/engine/obs/obs_mindrecord_dataset.py +7 -3
- mindspore/dataset/engine/obs/util.py +3 -0
- mindspore/dataset/engine/offload.py +6 -6
- mindspore/dataset/engine/queue.py +15 -14
- mindspore/dataset/engine/samplers.py +39 -23
- mindspore/dataset/engine/serializer_deserializer.py +22 -6
- mindspore/dataset/engine/validators.py +21 -331
- mindspore/dataset/text/__init__.py +5 -33
- mindspore/dataset/text/transforms.py +334 -165
- mindspore/dataset/text/utils.py +215 -145
- mindspore/dataset/transforms/__init__.py +1 -1
- mindspore/dataset/transforms/c_transforms.py +3 -2
- mindspore/dataset/transforms/py_transforms_util.py +40 -12
- mindspore/dataset/transforms/transforms.py +174 -71
- mindspore/dataset/utils/browse_dataset.py +25 -17
- mindspore/dataset/utils/line_reader.py +24 -21
- mindspore/dataset/vision/__init__.py +5 -26
- mindspore/dataset/vision/c_transforms.py +177 -165
- mindspore/dataset/vision/py_transforms.py +114 -119
- mindspore/dataset/vision/py_transforms_util.py +54 -51
- mindspore/dataset/vision/transforms.py +1127 -381
- mindspore/dataset/vision/utils.py +54 -38
- mindspore/dataset/vision/validators.py +12 -2
- mindspore/experimental/map_parameter.py +38 -4
- mindspore/{dataset/datapreprocess → experimental/optim}/__init__.py +14 -4
- mindspore/experimental/optim/adam.py +192 -0
- mindspore/experimental/optim/adamw.py +181 -0
- mindspore/experimental/optim/lr_scheduler.py +1427 -0
- mindspore/experimental/optim/optimizer.py +252 -0
- mindspore/experimental/optim/sgd.py +147 -0
- mindspore/gen_ops.py +273 -0
- mindspore/include/OWNERS +1 -2
- mindspore/include/api/context.h +21 -1
- mindspore/include/api/data_type.h +2 -1
- mindspore/include/api/graph.h +0 -15
- mindspore/include/api/kernel.h +2 -0
- mindspore/include/api/kernel_api.h +37 -12
- mindspore/include/api/model.h +29 -42
- mindspore/include/api/model_group.h +14 -3
- mindspore/include/api/model_parallel_runner.h +18 -2
- mindspore/include/api/serialization.h +26 -0
- mindspore/include/api/status.h +1 -0
- mindspore/include/api/types.h +38 -4
- mindspore/include/c_api/ms/abstract.h +67 -0
- mindspore/include/c_api/ms/attribute.h +197 -0
- mindspore/include/c_api/ms/base/handle_types.h +43 -0
- mindspore/include/c_api/ms/base/macros.h +32 -0
- mindspore/include/c_api/ms/base/status.h +33 -0
- mindspore/include/c_api/ms/base/types.h +282 -0
- mindspore/include/c_api/ms/context.h +102 -0
- mindspore/include/c_api/ms/graph.h +160 -0
- mindspore/include/c_api/ms/node.h +606 -0
- mindspore/include/c_api/ms/tensor.h +161 -0
- mindspore/include/c_api/ms/value.h +84 -0
- mindspore/include/c_api/status_c.h +3 -0
- mindspore/include/dataset/constants.h +6 -12
- mindspore/include/dataset/execute.h +23 -13
- mindspore/include/dataset/text.h +26 -26
- mindspore/include/dataset/transforms.h +25 -31
- mindspore/include/dataset/vision.h +60 -60
- mindspore/include/dataset/vision_ascend.h +5 -6
- mindspore/include/dataset/vision_lite.h +17 -17
- mindspore/include/mindapi/base/format.h +0 -1
- mindspore/include/mindapi/base/type_id.h +2 -1
- mindspore/include/mindapi/base/types.h +5 -1
- mindspore/lib/libdnnl.so.2 +0 -0
- mindspore/lib/libjemalloc.so.2 +0 -0
- mindspore/lib/libmindspore.so +0 -0
- mindspore/lib/libmindspore_backend.so +0 -0
- mindspore/lib/libmindspore_common.so +0 -0
- mindspore/lib/libmindspore_core.so +0 -0
- mindspore/lib/libmindspore_glog.so.0 +0 -0
- mindspore/lib/libmindspore_gpr.so.15 +0 -0
- mindspore/lib/libmindspore_grpc++.so.1 +0 -0
- mindspore/lib/libmindspore_grpc.so.15 +0 -0
- mindspore/lib/libmindspore_shared_lib.so +0 -0
- mindspore/lib/libmpi_adapter.so +0 -0
- mindspore/lib/libnnacl.so +0 -0
- mindspore/lib/libopencv_core.so.4.5 +0 -0
- mindspore/lib/libopencv_imgcodecs.so.4.5 +0 -0
- mindspore/lib/libopencv_imgproc.so.4.5 +0 -0
- mindspore/lib/libps_cache.so +0 -0
- mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/aicpu_kernel/impl/libcust_aicpu_kernels.so +0 -0
- mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/aicpu_kernel/impl/libcust_cpu_kernels.so +0 -0
- mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/config/cust_aicpu_kernel.json +9000 -0
- mindspore/lib/plugin/ascend/custom_aicpu_ops/op_proto/libcust_op_proto.so +0 -0
- mindspore/lib/plugin/ascend/libakg.so +0 -0
- mindspore/lib/plugin/ascend/libascend_collective.so +0 -0
- mindspore/lib/plugin/ascend/libdvpp_utils.so +0 -0
- mindspore/lib/plugin/ascend/libhccl_plugin.so +0 -0
- mindspore/lib/plugin/ascend/libmindspore_aicpu_kernels.so +0 -0
- mindspore/lib/plugin/ascend/libmindspore_cpu_kernels.so +0 -0
- mindspore/lib/plugin/cpu/libakg.so +0 -0
- mindspore/lib/plugin/gpu/libcuda_ops.so.10 +0 -0
- mindspore/lib/plugin/gpu/libcuda_ops.so.11 +0 -0
- mindspore/lib/plugin/gpu10.1/libakg.so +0 -0
- mindspore/lib/plugin/gpu10.1/libnccl.so.2 +0 -0
- mindspore/lib/plugin/gpu10.1/libnvidia_collective.so +0 -0
- mindspore/lib/plugin/gpu11.1/libakg.so +0 -0
- mindspore/lib/plugin/gpu11.1/libnccl.so.2 +0 -0
- mindspore/lib/plugin/gpu11.1/libnvidia_collective.so +0 -0
- mindspore/lib/plugin/gpu11.6/libakg.so +0 -0
- mindspore/lib/plugin/gpu11.6/libnccl.so.2 +0 -0
- mindspore/lib/plugin/gpu11.6/libnvidia_collective.so +0 -0
- mindspore/lib/plugin/libmindspore_ascend.so.1 +0 -0
- mindspore/lib/plugin/libmindspore_ascend.so.2 +0 -0
- mindspore/lib/plugin/libmindspore_gpu.so.10.1 +0 -0
- mindspore/lib/plugin/libmindspore_gpu.so.11.1 +0 -0
- mindspore/lib/plugin/libmindspore_gpu.so.11.6 +0 -0
- mindspore/log.py +9 -6
- mindspore/mindrecord/filereader.py +33 -4
- mindspore/mindrecord/filewriter.py +70 -35
- mindspore/mindrecord/mindpage.py +40 -34
- mindspore/mindrecord/shardreader.py +1 -1
- mindspore/mindrecord/shardsegment.py +1 -1
- mindspore/mindrecord/tools/cifar100_to_mr.py +25 -18
- mindspore/mindrecord/tools/cifar10_to_mr.py +25 -18
- mindspore/mindrecord/tools/csv_to_mr.py +29 -13
- mindspore/mindrecord/tools/imagenet_to_mr.py +24 -10
- mindspore/mindrecord/tools/mnist_to_mr.py +24 -11
- mindspore/mindrecord/tools/tfrecord_to_mr.py +31 -26
- mindspore/nn/cell.py +463 -169
- mindspore/nn/dynamic_lr.py +47 -43
- mindspore/nn/layer/activation.py +225 -82
- mindspore/nn/layer/basic.py +121 -79
- mindspore/nn/layer/channel_shuffle.py +21 -21
- mindspore/nn/layer/combined.py +33 -26
- mindspore/nn/layer/container.py +277 -22
- mindspore/nn/layer/conv.py +441 -304
- mindspore/nn/layer/dense.py +19 -13
- mindspore/nn/layer/embedding.py +62 -49
- mindspore/nn/layer/flash_attention.py +264 -0
- mindspore/nn/layer/image.py +50 -39
- mindspore/nn/layer/math.py +62 -51
- mindspore/nn/layer/normalization.py +219 -167
- mindspore/nn/layer/padding.py +58 -70
- mindspore/nn/layer/pooling.py +334 -287
- mindspore/nn/layer/rnn_cells.py +53 -38
- mindspore/nn/layer/rnns.py +59 -56
- mindspore/nn/layer/thor_layer.py +52 -44
- mindspore/nn/layer/timedistributed.py +6 -4
- mindspore/nn/layer/transformer.py +284 -164
- mindspore/nn/learning_rate_schedule.py +34 -25
- mindspore/nn/loss/__init__.py +3 -2
- mindspore/nn/loss/loss.py +554 -311
- mindspore/nn/optim/ada_grad.py +12 -9
- mindspore/nn/optim/adadelta.py +14 -11
- mindspore/nn/optim/adafactor.py +19 -16
- mindspore/nn/optim/adam.py +62 -47
- mindspore/nn/optim/adamax.py +13 -10
- mindspore/nn/optim/adasum.py +12 -8
- mindspore/nn/optim/asgd.py +10 -9
- mindspore/nn/optim/ftrl.py +20 -17
- mindspore/nn/optim/lamb.py +16 -12
- mindspore/nn/optim/lars.py +8 -6
- mindspore/nn/optim/lazyadam.py +25 -20
- mindspore/nn/optim/momentum.py +10 -7
- mindspore/nn/optim/optimizer.py +61 -9
- mindspore/nn/optim/proximal_ada_grad.py +14 -13
- mindspore/nn/optim/rmsprop.py +17 -13
- mindspore/nn/optim/rprop.py +30 -17
- mindspore/nn/optim/sgd.py +40 -23
- mindspore/nn/optim/thor.py +24 -26
- mindspore/nn/probability/bijector/bijector.py +11 -11
- mindspore/nn/probability/bijector/exp.py +1 -1
- mindspore/nn/probability/bijector/gumbel_cdf.py +3 -3
- mindspore/nn/probability/bijector/invert.py +1 -1
- mindspore/nn/probability/bijector/power_transform.py +29 -29
- mindspore/nn/probability/bijector/scalar_affine.py +3 -3
- mindspore/nn/probability/bijector/softplus.py +5 -5
- mindspore/nn/probability/bnn_layers/bnn_cell_wrapper.py +4 -2
- mindspore/nn/probability/bnn_layers/conv_variational.py +13 -13
- mindspore/nn/probability/bnn_layers/dense_variational.py +12 -12
- mindspore/nn/probability/bnn_layers/layer_distribution.py +9 -8
- mindspore/nn/probability/distribution/_utils/custom_ops.py +19 -3
- mindspore/nn/probability/distribution/_utils/utils.py +1 -1
- mindspore/nn/probability/distribution/bernoulli.py +9 -9
- mindspore/nn/probability/distribution/beta.py +8 -8
- mindspore/nn/probability/distribution/categorical.py +23 -15
- mindspore/nn/probability/distribution/cauchy.py +5 -6
- mindspore/nn/probability/distribution/distribution.py +3 -3
- mindspore/nn/probability/distribution/exponential.py +4 -4
- mindspore/nn/probability/distribution/gamma.py +10 -10
- mindspore/nn/probability/distribution/geometric.py +8 -8
- mindspore/nn/probability/distribution/gumbel.py +8 -9
- mindspore/nn/probability/distribution/half_normal.py +5 -5
- mindspore/nn/probability/distribution/laplace.py +5 -5
- mindspore/nn/probability/distribution/log_normal.py +12 -11
- mindspore/nn/probability/distribution/logistic.py +8 -8
- mindspore/nn/probability/distribution/normal.py +6 -5
- mindspore/nn/probability/distribution/poisson.py +10 -11
- mindspore/nn/probability/distribution/student_t.py +8 -9
- mindspore/nn/probability/distribution/transformed_distribution.py +5 -5
- mindspore/nn/probability/distribution/uniform.py +11 -11
- mindspore/nn/reinforcement/tensor_array.py +2 -2
- mindspore/nn/sparse/sparse.py +9 -9
- mindspore/nn/wrap/cell_wrapper.py +188 -63
- mindspore/nn/wrap/grad_reducer.py +21 -12
- mindspore/nn/wrap/loss_scale.py +136 -49
- mindspore/numpy/__init__.py +4 -4
- mindspore/numpy/array_creations.py +55 -56
- mindspore/numpy/array_ops.py +134 -35
- mindspore/numpy/logic_ops.py +66 -20
- mindspore/numpy/math_ops.py +142 -139
- mindspore/numpy/utils_const.py +2 -2
- mindspore/offline_debug/convert_async.py +2 -2
- mindspore/ops/_grad_experimental/__init__.py +7 -5
- mindspore/ops/_grad_experimental/grad_array_ops.py +231 -348
- mindspore/ops/{_grad → _grad_experimental}/grad_base.py +1 -33
- mindspore/ops/{_grad → _grad_experimental}/grad_comm_ops.py +25 -13
- mindspore/ops/{_grad/__init__.py → _grad_experimental/grad_debug_ops.py} +15 -7
- mindspore/ops/{_grad → _grad_experimental}/grad_implementations.py +17 -11
- mindspore/ops/_grad_experimental/grad_inner_ops.py +33 -52
- mindspore/ops/_grad_experimental/grad_math_ops.py +151 -1224
- mindspore/ops/_grad_experimental/grad_nn_ops.py +141 -414
- mindspore/ops/{_grad → _grad_experimental}/grad_quant_ops.py +10 -6
- mindspore/ops/_grad_experimental/grad_sparse.py +317 -2
- mindspore/ops/_grad_experimental/grad_sparse_ops.py +3 -13
- mindspore/ops/{_grad → _grad_experimental}/taylor_rule.py +1 -1
- mindspore/ops/_op_impl/_custom_op/dsd_back_impl.py +1 -1
- mindspore/ops/_op_impl/_custom_op/flash_attention/__init__.py +0 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/attention.py +406 -0
- mindspore/{_extends/graph_kernel/expanders/complex/__init__.py → ops/_op_impl/_custom_op/flash_attention/constants.py} +27 -8
- mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_bwd.py +467 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_fwd.py +563 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_impl.py +193 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/tik_ops_utils.py +435 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/__init__.py +0 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/sparse_tiling.py +45 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/strategy.py +67 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/wukong_tiling.py +62 -0
- mindspore/ops/_op_impl/_custom_op/matmul_cube_dense_left_impl.py +2 -2
- mindspore/ops/_op_impl/aicpu/__init__.py +41 -1
- mindspore/ops/_op_impl/aicpu/adaptive_max_pool_2d.py +37 -0
- mindspore/ops/_op_impl/aicpu/bias_add_grad.py +0 -1
- mindspore/ops/_op_impl/aicpu/cast.py +52 -0
- mindspore/ops/_op_impl/aicpu/coalesce.py +2 -0
- mindspore/ops/_op_impl/aicpu/col2im.py +3 -1
- mindspore/ops/_op_impl/aicpu/count_nonzero.py +43 -0
- mindspore/ops/_op_impl/aicpu/dropout_genmask.py +6 -0
- mindspore/ops/_op_impl/aicpu/eps.py +32 -0
- mindspore/ops/_op_impl/aicpu/eye.py +4 -4
- mindspore/ops/_op_impl/aicpu/fft_with_size.py +6 -0
- mindspore/ops/_op_impl/aicpu/fill_diagonal.py +5 -0
- mindspore/ops/_op_impl/aicpu/gamma.py +2 -2
- mindspore/ops/_op_impl/aicpu/im2col.py +3 -5
- mindspore/ops/_op_impl/aicpu/lgamma.py +1 -0
- mindspore/ops/_op_impl/aicpu/log_uniform_candidate_sampler.py +6 -3
- mindspore/ops/_op_impl/aicpu/lu.py +39 -0
- mindspore/ops/_op_impl/aicpu/lu_unpack_grad.py +0 -1
- mindspore/ops/_op_impl/aicpu/masked_scatter.py +1 -0
- mindspore/ops/_op_impl/aicpu/masked_select_grad.py +3 -0
- mindspore/ops/_op_impl/aicpu/matrix_band_part.py +59 -0
- mindspore/ops/_op_impl/aicpu/matrix_power.py +6 -1
- mindspore/ops/_op_impl/aicpu/median.py +1 -0
- mindspore/ops/_op_impl/aicpu/multinomial.py +9 -9
- mindspore/ops/_op_impl/aicpu/not_equal.py +0 -5
- mindspore/ops/_op_impl/aicpu/pad_v3.py +3 -1
- mindspore/ops/_op_impl/aicpu/pad_v3_grad.py +2 -0
- mindspore/ops/_op_impl/aicpu/parameterized_truncated_normal.py +15 -7
- mindspore/ops/_op_impl/aicpu/random_categorical.py +39 -19
- mindspore/ops/_op_impl/aicpu/random_choice_with_mask.py +5 -2
- mindspore/ops/_op_impl/aicpu/random_poisson.py +103 -52
- mindspore/ops/_op_impl/aicpu/random_shuffle.py +17 -15
- mindspore/ops/_op_impl/aicpu/resize_bilinear_grad.py +0 -1
- mindspore/ops/_op_impl/aicpu/resize_nearest_neighbor_v2.py +0 -6
- mindspore/ops/_op_impl/aicpu/resize_nearest_neighbor_v2_grad.py +0 -7
- mindspore/ops/_op_impl/aicpu/scatter_nd.py +2 -0
- mindspore/ops/_op_impl/aicpu/sequence_concat.py +40 -0
- mindspore/ops/_op_impl/aicpu/sequence_stack.py +40 -0
- mindspore/ops/_op_impl/aicpu/{sparseaddmm.py → sparse_addmm.py} +2 -2
- mindspore/ops/_op_impl/aicpu/{sparsesparsemaximum.py → sparse_sparse_maximum.py} +4 -4
- mindspore/ops/_op_impl/aicpu/standard_laplace.py +5 -4
- mindspore/ops/_op_impl/aicpu/standard_normal.py +5 -4
- mindspore/ops/_op_impl/aicpu/truncated_normal.py +9 -7
- mindspore/ops/_op_impl/aicpu/uniform.py +5 -3
- mindspore/ops/_op_impl/aicpu/uniform_candidate_sampler.py +8 -4
- mindspore/ops/_op_impl/aicpu/uniform_int.py +5 -5
- mindspore/ops/_op_impl/aicpu/uniform_real.py +4 -4
- mindspore/ops/_op_impl/aicpu/upsample_nearest_3d.py +14 -6
- mindspore/ops/_op_impl/aicpu/upsample_nearest_3d_grad.py +22 -8
- mindspore/ops/_op_impl/aicpu/upsample_trilinear_3d.py +11 -6
- mindspore/ops/_op_impl/aicpu/upsample_trilinear_3d_grad.py +21 -10
- mindspore/ops/_op_impl/tbe/__init__.py +6 -4
- mindspore/ops/_op_impl/tbe/atomic_addr_clean.py +1 -1
- mindspore/ops/_op_impl/tbe/avg_pool.py +2 -2
- mindspore/ops/_op_impl/tbe/avg_pool_3d.py +3 -3
- mindspore/ops/_op_impl/tbe/avg_pool_3d_grad.py +4 -4
- mindspore/ops/_op_impl/tbe/avg_pool_ds.py +2 -2
- mindspore/ops/_op_impl/tbe/avg_pool_grad.py +3 -3
- mindspore/ops/_op_impl/tbe/avg_pool_grad_vm.py +3 -3
- mindspore/ops/_op_impl/tbe/batch_to_space.py +1 -1
- mindspore/ops/_op_impl/tbe/batch_to_space_nd.py +2 -2
- mindspore/ops/_op_impl/tbe/bn_infer.py +2 -2
- mindspore/ops/_op_impl/tbe/bn_infer_ds.py +3 -2
- mindspore/ops/_op_impl/tbe/broadcast_to.py +1 -1
- mindspore/ops/_op_impl/tbe/depthwise_conv2d.py +3 -3
- mindspore/ops/_op_impl/tbe/expand_dims.py +1 -1
- mindspore/ops/_op_impl/tbe/gather_v2.py +56 -0
- mindspore/ops/_op_impl/tbe/im2col.py +4 -4
- mindspore/ops/_op_impl/tbe/inplace_index_add.py +7 -3
- mindspore/ops/_op_impl/tbe/mem_set.py +38 -0
- mindspore/ops/_op_impl/tbe/scatter_nd_add.py +3 -0
- mindspore/ops/_op_impl/tbe/scatter_nd_d.py +1 -1
- mindspore/ops/_op_impl/tbe/space_to_batch.py +1 -1
- mindspore/ops/_op_impl/tbe/space_to_batch_nd.py +2 -2
- mindspore/ops/_op_impl/tbe/trans_data_ds.py +2 -0
- mindspore/ops/_primitive_cache.py +1 -1
- mindspore/ops/_tracefunc.py +241 -0
- mindspore/ops/_utils/utils.py +10 -2
- mindspore/ops/_vmap/vmap_array_ops.py +5 -3
- mindspore/ops/_vmap/vmap_base.py +5 -4
- mindspore/ops/_vmap/vmap_convolution_ops.py +1 -1
- mindspore/ops/_vmap/vmap_grad_math_ops.py +6 -4
- mindspore/ops/_vmap/vmap_grad_nn_ops.py +11 -6
- mindspore/ops/_vmap/vmap_math_ops.py +5 -2
- mindspore/ops/_vmap/vmap_nn_ops.py +135 -11
- mindspore/ops/arg_dtype_cast.py +54 -0
- mindspore/ops/composite/__init__.py +7 -5
- mindspore/ops/composite/base.py +78 -34
- mindspore/ops/composite/math_ops.py +5 -695
- mindspore/ops/composite/multitype_ops/_compile_utils.py +403 -97
- mindspore/ops/composite/multitype_ops/_constexpr_utils.py +28 -22
- mindspore/ops/composite/multitype_ops/add_impl.py +69 -7
- mindspore/ops/composite/multitype_ops/bitwise_and_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/bitwise_or_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/bitwise_xor_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/div_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/floordiv_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/getitem_impl.py +48 -10
- mindspore/ops/composite/multitype_ops/greater_equal_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/greater_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/left_shift_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/less_equal_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/less_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/logic_not_impl.py +2 -2
- mindspore/ops/composite/multitype_ops/mod_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/mul_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/negative_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/not_in_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/ones_like_impl.py +6 -0
- mindspore/ops/composite/multitype_ops/pow_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/right_shift_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/setitem_impl.py +10 -7
- mindspore/ops/composite/multitype_ops/sub_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/uadd_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/zeros_like_impl.py +9 -0
- mindspore/ops/deprecated.py +304 -0
- mindspore/ops/function/__init__.py +41 -4
- mindspore/ops/function/array_func.py +1108 -467
- mindspore/ops/function/clip_func.py +94 -27
- mindspore/ops/function/debug_func.py +3 -1
- mindspore/ops/function/grad/grad_func.py +82 -73
- mindspore/ops/function/image_func.py +28 -12
- mindspore/ops/function/linalg_func.py +135 -39
- mindspore/ops/function/math_func.py +3779 -894
- mindspore/ops/function/nn_func.py +1584 -657
- mindspore/ops/function/parameter_func.py +13 -3
- mindspore/ops/function/random_func.py +247 -153
- mindspore/ops/function/sparse_func.py +14 -11
- mindspore/ops/function/sparse_unary_func.py +173 -47
- mindspore/ops/function/spectral_func.py +8 -4
- mindspore/ops/function/vmap_func.py +8 -7
- mindspore/ops/functional.py +47 -16
- mindspore/ops/op_info_register.py +346 -86
- mindspore/ops/operations/__init__.py +38 -22
- mindspore/ops/operations/_grad_ops.py +145 -149
- mindspore/ops/operations/_inner_ops.py +298 -56
- mindspore/ops/operations/_ms_kernel.py +3 -3
- mindspore/ops/operations/_quant_ops.py +24 -28
- mindspore/ops/operations/_rl_inner_ops.py +9 -7
- mindspore/ops/operations/_scalar_ops.py +115 -0
- mindspore/ops/operations/_sequence_ops.py +148 -10
- mindspore/ops/operations/_tensor_array.py +1 -1
- mindspore/ops/operations/_thor_ops.py +2 -2
- mindspore/ops/operations/array_ops.py +1239 -561
- mindspore/ops/operations/comm_ops.py +166 -90
- mindspore/ops/operations/control_ops.py +3 -3
- mindspore/ops/operations/custom_ops.py +124 -102
- mindspore/ops/operations/debug_ops.py +24 -11
- mindspore/ops/operations/image_ops.py +86 -71
- mindspore/ops/operations/inner_ops.py +18 -13
- mindspore/ops/operations/linalg_ops.py +30 -11
- mindspore/ops/operations/math_ops.py +1730 -435
- mindspore/ops/operations/nn_ops.py +1953 -943
- mindspore/ops/operations/other_ops.py +65 -43
- mindspore/ops/operations/random_ops.py +258 -98
- mindspore/ops/operations/rl_ops.py +4 -36
- mindspore/ops/operations/sparse_ops.py +38 -33
- mindspore/ops/operations/spectral_ops.py +8 -4
- mindspore/ops/primitive.py +66 -44
- mindspore/ops/signature.py +5 -5
- mindspore/parallel/_auto_parallel_context.py +80 -19
- mindspore/parallel/_cost_model_context.py +42 -0
- mindspore/parallel/_offload_context.py +162 -72
- mindspore/parallel/_parallel_serialization.py +2 -2
- mindspore/parallel/_ps_context.py +16 -4
- mindspore/parallel/_recovery_context.py +2 -1
- mindspore/parallel/_tensor.py +15 -13
- mindspore/parallel/_transformer/layers.py +8 -6
- mindspore/parallel/_transformer/loss.py +1 -0
- mindspore/parallel/_transformer/moe.py +7 -7
- mindspore/parallel/_transformer/op_parallel_config.py +12 -1
- mindspore/parallel/_transformer/transformer.py +34 -14
- mindspore/parallel/_utils.py +36 -14
- mindspore/parallel/algo_parameter_config.py +114 -20
- mindspore/parallel/checkpoint_transform.py +16 -18
- mindspore/parallel/shard.py +16 -13
- mindspore/profiler/__init__.py +1 -1
- mindspore/profiler/common/struct_type.py +3 -3
- mindspore/profiler/common/util.py +3 -2
- mindspore/profiler/envprofiling.py +11 -4
- mindspore/profiler/parser/aicpu_data_parser.py +5 -3
- mindspore/profiler/parser/ascend_flops_generator.py +94 -0
- mindspore/profiler/parser/ascend_fpbp_generator.py +76 -0
- mindspore/profiler/parser/ascend_hccl_generator.py +288 -0
- mindspore/profiler/parser/ascend_msprof_exporter.py +213 -0
- mindspore/profiler/parser/ascend_msprof_generator.py +199 -0
- mindspore/profiler/parser/ascend_op_generator.py +276 -0
- mindspore/profiler/parser/ascend_steptrace_generator.py +94 -0
- mindspore/profiler/parser/ascend_timeline_generator.py +110 -54
- mindspore/profiler/parser/base_timeline_generator.py +11 -7
- mindspore/profiler/parser/cpu_gpu_timeline_generator.py +45 -46
- mindspore/profiler/parser/flops_parser.py +15 -11
- mindspore/profiler/parser/framework_parser.py +92 -73
- mindspore/profiler/parser/hccl_parser.py +16 -12
- mindspore/profiler/parser/integrator.py +22 -11
- mindspore/profiler/parser/memory_usage_parser.py +36 -11
- mindspore/profiler/parser/minddata_analyzer.py +12 -14
- mindspore/profiler/parser/minddata_pipeline_parser.py +1 -1
- mindspore/profiler/parser/msadvisor_parser.py +8 -4
- mindspore/profiler/parser/op_intermediate_parser.py +5 -2
- mindspore/profiler/parser/optime_parser.py +1 -1
- mindspore/profiler/parser/profiler_info.py +4 -5
- mindspore/profiler/parser/step_trace_parser.py +11 -14
- mindspore/profiler/profiling.py +678 -377
- mindspore/rewrite/api/node.py +211 -54
- mindspore/rewrite/api/node_type.py +5 -0
- mindspore/rewrite/api/pattern_engine.py +22 -23
- mindspore/rewrite/api/scoped_value.py +20 -17
- mindspore/rewrite/api/symbol_tree.py +252 -106
- mindspore/rewrite/api/tree_node_helper.py +3 -0
- mindspore/rewrite/ast_helpers/__init__.py +2 -1
- mindspore/rewrite/ast_helpers/ast_finder.py +129 -0
- mindspore/rewrite/ast_helpers/ast_modifier.py +116 -104
- mindspore/rewrite/ast_transformers/flatten_recursive_stmt.py +97 -46
- mindspore/rewrite/common/rewrite_elog.py +5 -1
- mindspore/rewrite/namer.py +51 -51
- mindspore/rewrite/namespace.py +14 -5
- mindspore/{ops/bprop_mindir → rewrite/node}/__init__.py +9 -4
- mindspore/rewrite/node/call_function.py +79 -0
- mindspore/rewrite/node/cell_container.py +135 -0
- mindspore/rewrite/node/control_flow.py +88 -0
- mindspore/rewrite/{node.py → node/node.py} +313 -247
- mindspore/rewrite/node/node_manager.py +254 -0
- mindspore/rewrite/node/node_topological_manager.py +243 -0
- mindspore/rewrite/parsers/arguments_parser.py +22 -21
- mindspore/rewrite/parsers/assign_parser.py +225 -239
- mindspore/rewrite/parsers/attribute_parser.py +9 -7
- mindspore/rewrite/parsers/class_def_parser.py +179 -218
- mindspore/rewrite/parsers/constant_parser.py +9 -6
- mindspore/rewrite/parsers/container_parser.py +9 -7
- mindspore/rewrite/parsers/for_parser.py +36 -15
- mindspore/rewrite/parsers/function_def_parser.py +23 -20
- mindspore/rewrite/parsers/if_parser.py +28 -24
- mindspore/rewrite/parsers/module_parser.py +202 -25
- mindspore/rewrite/{parser.py → parsers/parser.py} +4 -2
- mindspore/rewrite/{parser_register.py → parsers/parser_register.py} +1 -1
- mindspore/rewrite/parsers/return_parser.py +6 -6
- mindspore/rewrite/sparsify/sparse_transformer.py +12 -3
- mindspore/rewrite/sparsify/sparsify.py +4 -1
- mindspore/rewrite/sparsify/utils.py +11 -5
- mindspore/rewrite/symbol_tree.py +577 -732
- mindspore/rewrite/symbol_tree_builder.py +9 -175
- mindspore/rewrite/symbol_tree_dumper.py +2 -2
- mindspore/run_check/_check_version.py +46 -39
- mindspore/run_check/run_check.py +3 -2
- mindspore/{scipy/sparse → safeguard}/__init__.py +4 -5
- mindspore/safeguard/rewrite_obfuscation.py +517 -0
- mindspore/scipy/__init__.py +1 -1
- mindspore/scipy/linalg.py +67 -61
- mindspore/scipy/ops.py +5 -41
- mindspore/scipy/ops_grad.py +3 -2
- mindspore/scipy/ops_wrapper.py +5 -5
- mindspore/scipy/optimize/line_search.py +8 -8
- mindspore/scipy/optimize/linear_sum_assignment.py +4 -4
- mindspore/scipy/optimize/minimize.py +16 -12
- mindspore/scipy/utils.py +1 -52
- mindspore/scipy/utils_const.py +4 -4
- mindspore/train/__init__.py +4 -4
- mindspore/train/_utils.py +13 -5
- mindspore/train/amp.py +410 -148
- mindspore/train/anf_ir_pb2.py +16 -4
- mindspore/train/callback/_backup_and_restore.py +8 -11
- mindspore/train/callback/_callback.py +80 -3
- mindspore/train/callback/_checkpoint.py +82 -51
- mindspore/train/callback/_early_stop.py +12 -15
- mindspore/train/callback/_history.py +1 -1
- mindspore/train/callback/_lambda_callback.py +13 -13
- mindspore/train/callback/_landscape.py +21 -17
- mindspore/train/callback/_loss_monitor.py +9 -10
- mindspore/train/callback/_on_request_exit.py +16 -33
- mindspore/train/callback/_reduce_lr_on_plateau.py +21 -24
- mindspore/train/callback/_summary_collector.py +44 -30
- mindspore/train/callback/_time_monitor.py +62 -12
- mindspore/train/data_sink.py +10 -16
- mindspore/train/dataset_helper.py +154 -86
- mindspore/train/loss_scale_manager.py +14 -9
- mindspore/train/metrics/__init__.py +10 -2
- mindspore/train/metrics/accuracy.py +1 -1
- mindspore/train/metrics/auc.py +1 -1
- mindspore/train/metrics/bleu_score.py +2 -2
- mindspore/train/metrics/confusion_matrix.py +14 -14
- mindspore/train/metrics/cosine_similarity.py +3 -3
- mindspore/train/metrics/dice.py +1 -1
- mindspore/train/metrics/fbeta.py +1 -1
- mindspore/train/metrics/hausdorff_distance.py +8 -6
- mindspore/train/metrics/mean_surface_distance.py +5 -4
- mindspore/train/metrics/metric.py +49 -17
- mindspore/train/metrics/occlusion_sensitivity.py +4 -4
- mindspore/train/metrics/perplexity.py +1 -1
- mindspore/train/metrics/precision.py +2 -2
- mindspore/train/metrics/recall.py +2 -3
- mindspore/train/metrics/roc.py +7 -7
- mindspore/train/metrics/root_mean_square_surface_distance.py +5 -4
- mindspore/train/metrics/topk.py +7 -4
- mindspore/train/mind_ir_pb2.py +193 -48
- mindspore/train/model.py +377 -133
- mindspore/train/serialization.py +697 -245
- mindspore/train/summary/_summary_adapter.py +5 -2
- mindspore/train/summary/_writer_pool.py +4 -3
- mindspore/train/summary/summary_record.py +25 -23
- mindspore/train/train_thor/convert_utils.py +39 -23
- mindspore/train/train_thor/dataset_helper.py +4 -3
- mindspore/train/train_thor/model_thor.py +8 -8
- mindspore/version.py +1 -1
- {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/METADATA +7 -8
- {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/RECORD +647 -818
- {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/entry_points.txt +0 -1
- mindspore/_akg/akg/tvm/contrib/debugger/__init__.py +0 -16
- mindspore/_akg/akg/tvm/contrib/debugger/debug_result.py +0 -274
- mindspore/_akg/akg/tvm/contrib/debugger/debug_runtime.py +0 -259
- mindspore/_akg/akg/tvm/contrib/peak.py +0 -341
- mindspore/_akg/akg/tvm/contrib/rpc.py +0 -25
- mindspore/_akg/akg/tvm/contrib/xcode.py +0 -257
- mindspore/_akg/akg/tvm/exec/__init__.py +0 -17
- mindspore/_akg/akg/tvm/exec/autotvm_log_editor.py +0 -60
- mindspore/_akg/akg/tvm/exec/measure_peak.py +0 -48
- mindspore/_akg/akg/tvm/exec/query_rpc_tracker.py +0 -48
- mindspore/_akg/akg/tvm/exec/rpc_proxy.py +0 -98
- mindspore/_akg/akg/tvm/exec/rpc_server.py +0 -88
- mindspore/_akg/akg/tvm/exec/rpc_tracker.py +0 -62
- mindspore/_akg/akg/tvm/rpc/__init__.py +0 -29
- mindspore/_akg/akg/tvm/rpc/base.py +0 -182
- mindspore/_akg/akg/tvm/rpc/client.py +0 -436
- mindspore/_akg/akg/tvm/rpc/proxy.py +0 -595
- mindspore/_akg/akg/tvm/rpc/server.py +0 -413
- mindspore/_akg/akg/tvm/rpc/tornado_util.py +0 -121
- mindspore/_akg/akg/tvm/rpc/tracker.py +0 -431
- mindspore/_extends/graph_kernel/expander.py +0 -80
- mindspore/_extends/graph_kernel/expanders/__init__.py +0 -57
- mindspore/_extends/graph_kernel/expanders/_utils.py +0 -269
- mindspore/_extends/graph_kernel/expanders/addn.py +0 -33
- mindspore/_extends/graph_kernel/expanders/batchnorm.py +0 -152
- mindspore/_extends/graph_kernel/expanders/batchnorm_grad.py +0 -105
- mindspore/_extends/graph_kernel/expanders/bias_add_grad.py +0 -49
- mindspore/_extends/graph_kernel/expanders/clip_by_norm_no_div_sum.py +0 -33
- mindspore/_extends/graph_kernel/expanders/complex/abs.py +0 -30
- mindspore/_extends/graph_kernel/expanders/complex/add.py +0 -44
- mindspore/_extends/graph_kernel/expanders/complex/div.py +0 -62
- mindspore/_extends/graph_kernel/expanders/complex/mul.py +0 -52
- mindspore/_extends/graph_kernel/expanders/complex/real_div.py +0 -62
- mindspore/_extends/graph_kernel/expanders/complex/sub.py +0 -45
- mindspore/_extends/graph_kernel/expanders/conv2d.py +0 -200
- mindspore/_extends/graph_kernel/expanders/dropout_grad.py +0 -30
- mindspore/_extends/graph_kernel/expanders/equal_count.py +0 -50
- mindspore/_extends/graph_kernel/expanders/erfc.py +0 -35
- mindspore/_extends/graph_kernel/expanders/expand_dims.py +0 -50
- mindspore/_extends/graph_kernel/expanders/fused_adam.py +0 -44
- mindspore/_extends/graph_kernel/expanders/fused_adam_weight_decay.py +0 -47
- mindspore/_extends/graph_kernel/expanders/fused_mul_add.py +0 -28
- mindspore/_extends/graph_kernel/expanders/gather.py +0 -43
- mindspore/_extends/graph_kernel/expanders/gelu_grad.py +0 -70
- mindspore/_extends/graph_kernel/expanders/gkdropout.py +0 -40
- mindspore/_extends/graph_kernel/expanders/identity.py +0 -25
- mindspore/_extends/graph_kernel/expanders/layernorm.py +0 -93
- mindspore/_extends/graph_kernel/expanders/layernorm_grad.py +0 -113
- mindspore/_extends/graph_kernel/expanders/logsoftmax.py +0 -46
- mindspore/_extends/graph_kernel/expanders/logsoftmax_grad.py +0 -36
- mindspore/_extends/graph_kernel/expanders/matmul.py +0 -80
- mindspore/_extends/graph_kernel/expanders/maximum_grad.py +0 -59
- mindspore/_extends/graph_kernel/expanders/minimum_grad.py +0 -80
- mindspore/_extends/graph_kernel/expanders/oneslike.py +0 -26
- mindspore/_extends/graph_kernel/expanders/reduce_mean.py +0 -43
- mindspore/_extends/graph_kernel/expanders/relu_grad.py +0 -32
- mindspore/_extends/graph_kernel/expanders/sigmoid_cross_entropy_with_logits.py +0 -41
- mindspore/_extends/graph_kernel/expanders/sigmoid_cross_entropy_with_logits_grad.py +0 -35
- mindspore/_extends/graph_kernel/expanders/sigmoid_grad.py +0 -31
- mindspore/_extends/graph_kernel/expanders/slice.py +0 -35
- mindspore/_extends/graph_kernel/expanders/softmax_cross_entropy_with_logits.py +0 -42
- mindspore/_extends/graph_kernel/expanders/softmax_grad_ext.py +0 -41
- mindspore/_extends/graph_kernel/expanders/softsign.py +0 -28
- mindspore/_extends/graph_kernel/expanders/sqrt_grad.py +0 -29
- mindspore/_extends/graph_kernel/expanders/square_sum_all.py +0 -44
- mindspore/_extends/graph_kernel/expanders/square_sum_v1.py +0 -37
- mindspore/_extends/graph_kernel/expanders/squared_difference.py +0 -43
- mindspore/_extends/graph_kernel/expanders/tanh_grad.py +0 -31
- mindspore/_extends/graph_kernel/expanders/tile.py +0 -54
- mindspore/_extends/graph_kernel/model/op_infer.py +0 -506
- mindspore/_extends/parse/jit_fallback_modules.py +0 -51
- mindspore/dataset/datapreprocess/preprocess_imagenet_validate_dataset.py +0 -54
- mindspore/dataset/engine/graphdata.py +0 -1586
- mindspore/include/api/net.h +0 -142
- mindspore/ops/_grad/grad_array_ops.py +0 -1347
- mindspore/ops/_grad/grad_clip_ops.py +0 -84
- mindspore/ops/_grad/grad_debug_ops.py +0 -68
- mindspore/ops/_grad/grad_inner_ops.py +0 -235
- mindspore/ops/_grad/grad_math_ops.py +0 -1684
- mindspore/ops/_grad/grad_nn_ops.py +0 -1529
- mindspore/ops/_grad/grad_other_ops.py +0 -89
- mindspore/ops/_grad/grad_sequence_ops.py +0 -296
- mindspore/ops/_grad/grad_sparse.py +0 -323
- mindspore/ops/_grad_experimental/grad_image_ops.py +0 -249
- mindspore/ops/_grad_experimental/grad_linalg_ops.py +0 -195
- mindspore/ops/_grad_experimental/grad_scalar_ops.py +0 -112
- mindspore/ops/bprop_mindir/AdaptiveAvgPool2D_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/AdaptiveMaxPool2D_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/ApproximateEqual_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/Argmax_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/Argmin_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/AssignSub_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/Assign_bprop.mindir +0 -17
- mindspore/ops/bprop_mindir/AvgPool3D_bprop.mindir +0 -150
- mindspore/ops/bprop_mindir/AvgPool_bprop.mindir +0 -66
- mindspore/ops/bprop_mindir/BCEWithLogitsLoss_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/BNTrainingReduce_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/BatchNormGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/BatchToSpaceND_bprop.mindir +0 -28
- mindspore/ops/bprop_mindir/BiasAddGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/BinaryCrossEntropy_bprop.mindir +0 -33
- mindspore/ops/bprop_mindir/BroadcastTo_bprop.mindir +0 -306
- mindspore/ops/bprop_mindir/Broadcast_bprop.mindir +0 -13
- mindspore/ops/bprop_mindir/CTCLoss_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Concat_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Conv2DBackpropFilter_bprop.mindir +0 -240
- mindspore/ops/bprop_mindir/Conv2DBackpropInput_bprop.mindir +0 -247
- mindspore/ops/bprop_mindir/Conv2DTranspose_bprop.mindir +0 -247
- mindspore/ops/bprop_mindir/Conv3DTranspose_bprop.mindir +0 -315
- mindspore/ops/bprop_mindir/Conv3D_bprop.mindir +0 -278
- mindspore/ops/bprop_mindir/DType_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/DeformableOffsets_bprop.mindir +0 -58
- mindspore/ops/bprop_mindir/Depend_bprop.mindir +0 -13
- mindspore/ops/bprop_mindir/DepthToSpace_bprop.mindir +0 -23
- mindspore/ops/bprop_mindir/DepthwiseConv2dNative_bprop.mindir +0 -138
- mindspore/ops/bprop_mindir/DiagPart_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/Dropout2D_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Dropout3D_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/DropoutDoMask_bprop.mindir +0 -25
- mindspore/ops/bprop_mindir/DropoutGenMask_bprop.mindir +0 -18
- mindspore/ops/bprop_mindir/DropoutGrad_bprop.mindir +0 -27
- mindspore/ops/bprop_mindir/Dropout_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/DynamicGRUV2_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/DynamicRNN_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/DynamicShape_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/Elu_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/EmbeddingLookup_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Equal_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/ExpandDims_bprop.mindir +0 -58
- mindspore/ops/bprop_mindir/FastGeLU_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/Flatten_bprop.mindir +0 -54
- mindspore/ops/bprop_mindir/FloorDiv_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/GatherD_bprop.mindir +0 -26
- mindspore/ops/bprop_mindir/GatherNd_bprop.mindir +0 -57
- mindspore/ops/bprop_mindir/Gather_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/GreaterEqual_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/Greater_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/HSigmoid_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/HSwish_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/IOU_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/InstanceNorm_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/IsFinite_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/IsInf_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/IsNan_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/KLDivLoss_bprop.mindir +0 -126
- mindspore/ops/bprop_mindir/L2Loss_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/L2Normalize_bprop.mindir +0 -30
- mindspore/ops/bprop_mindir/LRN_bprop.mindir +0 -43
- mindspore/ops/bprop_mindir/LayerNormGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/LessEqual_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/Less_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/LinSpace_bprop.mindir +0 -23
- mindspore/ops/bprop_mindir/Load_bprop.mindir +0 -13
- mindspore/ops/bprop_mindir/LogSoftmax_bprop.mindir +0 -23
- mindspore/ops/bprop_mindir/LogicalAnd_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/LogicalNot_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/MaskedSelect_bprop.mindir +0 -21
- mindspore/ops/bprop_mindir/MaxPool3DGradGrad_bprop.mindir +0 -74
- mindspore/ops/bprop_mindir/MaxPool3DGrad_bprop.mindir +0 -74
- mindspore/ops/bprop_mindir/MaxPool3D_bprop.mindir +0 -75
- mindspore/ops/bprop_mindir/MaxPoolGradGrad_bprop.mindir +0 -65
- mindspore/ops/bprop_mindir/MaxPoolWithArgmax_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Maximum_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Minimum_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/MirrorPad_bprop.mindir +0 -27
- mindspore/ops/bprop_mindir/Mish_bprop.mindir +0 -35
- mindspore/ops/bprop_mindir/MulNoNan_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/NLLLoss_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/NonZero_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/NotEqual_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/OneHot_bprop.mindir +0 -26
- mindspore/ops/bprop_mindir/OnesLike_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/PReLU_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Pad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Padding_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/RNNTLoss_bprop.mindir +0 -29
- mindspore/ops/bprop_mindir/ROIAlign_bprop.mindir +0 -82
- mindspore/ops/bprop_mindir/Range_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/Rank_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/ReLU6_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/ReLUV2_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/ReduceAll_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/ReduceAny_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/ReluGrad_bprop.mindir +0 -20
- mindspore/ops/bprop_mindir/Reshape_bprop.mindir +0 -60
- mindspore/ops/bprop_mindir/ResizeBilinear_bprop.mindir +0 -29
- mindspore/ops/bprop_mindir/ResizeNearestNeighbor_bprop.mindir +0 -89
- mindspore/ops/bprop_mindir/ReverseSequence_bprop.mindir +0 -52
- mindspore/ops/bprop_mindir/ReverseV2_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/Round_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/ScatterMax_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/ScatterMin_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/ScatterNdUpdate_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/ScatterNd_bprop.mindir +0 -24
- mindspore/ops/bprop_mindir/ScatterNonAliasingAdd_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/ScatterUpdate_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/SeLU_bprop.mindir +0 -21
- mindspore/ops/bprop_mindir/Select_bprop.mindir +0 -31
- mindspore/ops/bprop_mindir/Shape_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/SigmoidCrossEntropyWithLogits_bprop.mindir +0 -21
- mindspore/ops/bprop_mindir/SigmoidGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Sigmoid_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/Sign_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/Slice_bprop.mindir +0 -26
- mindspore/ops/bprop_mindir/SmoothL1Loss_bprop.mindir +0 -36
- mindspore/ops/bprop_mindir/SoftmaxCrossEntropyWithLogits_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Softplus_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/Softsign_bprop.mindir +0 -33
- mindspore/ops/bprop_mindir/Sort_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/SpaceToBatchND_bprop.mindir +0 -28
- mindspore/ops/bprop_mindir/SpaceToDepth_bprop.mindir +0 -23
- mindspore/ops/bprop_mindir/SparseGatherV2_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/SparseSoftmaxCrossEntropyWithLogits_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Split_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/Squeeze_bprop.mindir +0 -54
- mindspore/ops/bprop_mindir/StridedSliceGrad_bprop.mindir +0 -95
- mindspore/ops/bprop_mindir/StridedSlice_bprop.mindir +0 -98
- mindspore/ops/bprop_mindir/Switch_bprop.mindir +0 -29
- mindspore/ops/bprop_mindir/TanhGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Tanh_bprop.mindir +0 -66
- mindspore/ops/bprop_mindir/TensorScatterAdd_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/TensorScatterUpdate_bprop.mindir +0 -29
- mindspore/ops/bprop_mindir/TensorShape_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/Tile_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/TopK_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/TransShape_bprop.mindir +0 -23
- mindspore/ops/bprop_mindir/TruncateDiv_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/TupleGetItem_bprop.mindir +0 -20
- mindspore/ops/bprop_mindir/Unique_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/Unstack_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/UpsampleNearest3D_bprop.mindir +0 -32
- mindspore/ops/bprop_mindir/UpsampleTrilinear3D_bprop.mindir +0 -38
- mindspore/ops/bprop_mindir/ZerosLike_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/generate_mindir.py +0 -114
- mindspore/rewrite/node_visitor.py +0 -44
- mindspore/rewrite/topological_manager.py +0 -203
- mindspore/scipy/sparse/linalg.py +0 -192
- {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/WHEEL +0 -0
- {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2019-
|
|
1
|
+
# Copyright 2019-2023 Huawei Technologies Co., Ltd
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -112,9 +112,9 @@ class Caltech101Dataset(GeneratorDataset):
|
|
|
112
112
|
|
|
113
113
|
The columns of the generated dataset depend on the value of `target_type` .
|
|
114
114
|
|
|
115
|
-
- When `target_type` is 'category'
|
|
116
|
-
- When `target_type` is 'annotation'
|
|
117
|
-
- When `target_type` is 'all'
|
|
115
|
+
- When `target_type` is ``'category'``, the columns are :py:obj:`[image, category]` .
|
|
116
|
+
- When `target_type` is ``'annotation'``, the columns are :py:obj:`[image, annotation]` .
|
|
117
|
+
- When `target_type` is ``'all'``, the columns are :py:obj:`[image, category, annotation]` .
|
|
118
118
|
|
|
119
119
|
The tensor of column :py:obj:`image` is of the uint8 type.
|
|
120
120
|
The tensor of column :py:obj:`category` is of the uint32 type.
|
|
@@ -125,21 +125,23 @@ class Caltech101Dataset(GeneratorDataset):
|
|
|
125
125
|
dataset_dir (str): Path to the root directory that contains the dataset. This root directory contains two
|
|
126
126
|
subdirectories, one is called 101_ObjectCategories, which stores images,
|
|
127
127
|
and the other is called Annotations, which stores annotations.
|
|
128
|
-
target_type (str, optional): Target of the image. If `target_type` is 'category'
|
|
129
|
-
the target class. If `target_type` is 'annotation'
|
|
130
|
-
If `target_type` is 'all'
|
|
128
|
+
target_type (str, optional): Target of the image. If `target_type` is ``'category'``,
|
|
129
|
+
return category represents the target class. If `target_type` is ``'annotation'``,
|
|
130
|
+
return annotation. If `target_type` is ``'all'``, return category and annotation.
|
|
131
|
+
Default: ``None`` , means ``'category'``.
|
|
131
132
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
132
|
-
Default: None, all images.
|
|
133
|
-
num_parallel_workers (int, optional): Number of worker subprocesses to read the data.
|
|
133
|
+
Default: ``None`` , all images.
|
|
134
|
+
num_parallel_workers (int, optional): Number of worker subprocesses to read the data.
|
|
135
|
+
Default: ``1``.
|
|
134
136
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
135
|
-
Default: None, expected order behavior shown in the table below.
|
|
136
|
-
decode (bool, optional): Whether or not to decode the images after reading. Default: False
|
|
137
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
138
|
+
decode (bool, optional): Whether or not to decode the images after reading. Default: ``False``.
|
|
137
139
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
138
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
140
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
139
141
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
140
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
142
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
141
143
|
the maximum sample number of per shard.
|
|
142
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
144
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
143
145
|
argument can only be specified when `num_shards` is also specified.
|
|
144
146
|
|
|
145
147
|
Raises:
|
|
@@ -149,40 +151,21 @@ class Caltech101Dataset(GeneratorDataset):
|
|
|
149
151
|
RuntimeError: If `num_shards` is specified but `shard_id` is None.
|
|
150
152
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
151
153
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
152
|
-
ValueError: If `target_type` is not
|
|
154
|
+
ValueError: If `target_type` is not ``'category'``, ``'annotation'`` or ``'all'`` .
|
|
153
155
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
154
156
|
|
|
157
|
+
Tutorial Examples:
|
|
158
|
+
- `Load & Process Data With Dataset Pipeline
|
|
159
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
160
|
+
|
|
155
161
|
Note:
|
|
156
|
-
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
..
|
|
160
|
-
:widths: 25 25 50
|
|
161
|
-
:header-rows: 1
|
|
162
|
-
|
|
163
|
-
* - Parameter `sampler`
|
|
164
|
-
- Parameter `shuffle`
|
|
165
|
-
- Expected Order Behavior
|
|
166
|
-
* - None
|
|
167
|
-
- None
|
|
168
|
-
- random order
|
|
169
|
-
* - None
|
|
170
|
-
- True
|
|
171
|
-
- random order
|
|
172
|
-
* - None
|
|
173
|
-
- False
|
|
174
|
-
- sequential order
|
|
175
|
-
* - Sampler object
|
|
176
|
-
- None
|
|
177
|
-
- order defined by sampler
|
|
178
|
-
* - Sampler object
|
|
179
|
-
- True
|
|
180
|
-
- not allowed
|
|
181
|
-
* - Sampler object
|
|
182
|
-
- False
|
|
183
|
-
- not allowed
|
|
162
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
163
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
164
|
+
|
|
165
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
184
166
|
|
|
185
167
|
Examples:
|
|
168
|
+
>>> import mindspore.dataset as ds
|
|
186
169
|
>>> caltech101_dataset_directory = "/path/to/caltech101_dataset_directory"
|
|
187
170
|
>>>
|
|
188
171
|
>>> # 1) Read all samples (image files) in caltech101_dataset_directory with 8 threads
|
|
@@ -289,23 +272,23 @@ class Caltech256Dataset(MappableDataset, VisionBaseDataset):
|
|
|
289
272
|
Args:
|
|
290
273
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
291
274
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
292
|
-
Default: None, all images.
|
|
275
|
+
Default: ``None`` , all images.
|
|
293
276
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
294
|
-
Default: None, will use global default workers(8), it can be set
|
|
295
|
-
by
|
|
277
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
278
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
296
279
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
297
|
-
Default: None, expected order behavior shown in the table below.
|
|
298
|
-
decode (bool, optional): Whether or not to decode the images after reading. Default: False
|
|
280
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
281
|
+
decode (bool, optional): Whether or not to decode the images after reading. Default: ``False``.
|
|
299
282
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
300
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
283
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
301
284
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
302
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
285
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
303
286
|
the maximum sample number of per shard.
|
|
304
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
287
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
305
288
|
argument can only be specified when `num_shards` is also specified.
|
|
306
289
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
307
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
308
|
-
Default: None, which means no cache is used.
|
|
290
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
291
|
+
Default: ``None`` , which means no cache is used.
|
|
309
292
|
|
|
310
293
|
Raises:
|
|
311
294
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -314,40 +297,21 @@ class Caltech256Dataset(MappableDataset, VisionBaseDataset):
|
|
|
314
297
|
RuntimeError: If `num_shards` is specified but `shard_id` is None.
|
|
315
298
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
316
299
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
317
|
-
ValueError: If `target_type` is not 'category'
|
|
300
|
+
ValueError: If `target_type` is not ``'category'``, ``'annotation'`` or ``'all'``.
|
|
318
301
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
319
302
|
|
|
303
|
+
Tutorial Examples:
|
|
304
|
+
- `Load & Process Data With Dataset Pipeline
|
|
305
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
306
|
+
|
|
320
307
|
Note:
|
|
321
|
-
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
..
|
|
325
|
-
:widths: 25 25 50
|
|
326
|
-
:header-rows: 1
|
|
327
|
-
|
|
328
|
-
* - Parameter `sampler`
|
|
329
|
-
- Parameter `shuffle`
|
|
330
|
-
- Expected Order Behavior
|
|
331
|
-
* - None
|
|
332
|
-
- None
|
|
333
|
-
- random order
|
|
334
|
-
* - None
|
|
335
|
-
- True
|
|
336
|
-
- random order
|
|
337
|
-
* - None
|
|
338
|
-
- False
|
|
339
|
-
- sequential order
|
|
340
|
-
* - Sampler object
|
|
341
|
-
- None
|
|
342
|
-
- order defined by sampler
|
|
343
|
-
* - Sampler object
|
|
344
|
-
- True
|
|
345
|
-
- not allowed
|
|
346
|
-
* - Sampler object
|
|
347
|
-
- False
|
|
348
|
-
- not allowed
|
|
308
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
309
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
310
|
+
|
|
311
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
349
312
|
|
|
350
313
|
Examples:
|
|
314
|
+
>>> import mindspore.dataset as ds
|
|
351
315
|
>>> caltech256_dataset_dir = "/path/to/caltech256_dataset_directory"
|
|
352
316
|
>>>
|
|
353
317
|
>>> # 1) Read all samples (image files) in caltech256_dataset_dir with 8 threads
|
|
@@ -419,26 +383,26 @@ class CelebADataset(MappableDataset, VisionBaseDataset):
|
|
|
419
383
|
Args:
|
|
420
384
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
421
385
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
422
|
-
Default: None, will use global default workers(8), it can be set
|
|
423
|
-
by
|
|
424
|
-
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None.
|
|
425
|
-
usage (str, optional): Specify the 'train'
|
|
426
|
-
Default: 'all'
|
|
427
|
-
sampler (Sampler, optional): Object used to choose samples from the dataset. Default: None.
|
|
428
|
-
decode (bool, optional): Whether to decode the images after reading. Default: False
|
|
429
|
-
extensions (list[str], optional): List of file extensions to be included in the dataset. Default: None.
|
|
386
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
387
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
388
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` .
|
|
389
|
+
usage (str, optional): Specify the ``'train'``, ``'valid'``, ``'test'`` part or ``'all'``
|
|
390
|
+
parts of dataset. Default: ``'all'``, will read all samples.
|
|
391
|
+
sampler (Sampler, optional): Object used to choose samples from the dataset. Default: ``None`` .
|
|
392
|
+
decode (bool, optional): Whether to decode the images after reading. Default: ``False``.
|
|
393
|
+
extensions (list[str], optional): List of file extensions to be included in the dataset. Default: ``None`` .
|
|
430
394
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
431
|
-
Default: None, will include all images.
|
|
395
|
+
Default: ``None`` , will include all images.
|
|
432
396
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
433
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
397
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
434
398
|
the maximum sample number of per shard.
|
|
435
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
399
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
436
400
|
argument can only be specified when `num_shards` is also specified.
|
|
437
401
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
438
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
439
|
-
Default: None, which means no cache is used.
|
|
402
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
403
|
+
Default: ``None`` , which means no cache is used.
|
|
440
404
|
decrypt (callable, optional): Image decryption function, which accepts the path of the encrypted image file
|
|
441
|
-
and returns the decrypted bytes data. Default: None, no decryption.
|
|
405
|
+
and returns the decrypted bytes data. Default: ``None`` , no decryption.
|
|
442
406
|
|
|
443
407
|
Raises:
|
|
444
408
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -448,39 +412,20 @@ class CelebADataset(MappableDataset, VisionBaseDataset):
|
|
|
448
412
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
449
413
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
450
414
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
451
|
-
ValueError: If `usage` is not 'train'
|
|
415
|
+
ValueError: If `usage` is not ``'train'``, ``'valid'``, ``'test'`` or ``'all'``.
|
|
416
|
+
|
|
417
|
+
Tutorial Examples:
|
|
418
|
+
- `Load & Process Data With Dataset Pipeline
|
|
419
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
452
420
|
|
|
453
421
|
Note:
|
|
454
|
-
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
..
|
|
458
|
-
:widths: 25 25 50
|
|
459
|
-
:header-rows: 1
|
|
460
|
-
|
|
461
|
-
* - Parameter `sampler`
|
|
462
|
-
- Parameter `shuffle`
|
|
463
|
-
- Expected Order Behavior
|
|
464
|
-
* - None
|
|
465
|
-
- None
|
|
466
|
-
- random order
|
|
467
|
-
* - None
|
|
468
|
-
- True
|
|
469
|
-
- random order
|
|
470
|
-
* - None
|
|
471
|
-
- False
|
|
472
|
-
- sequential order
|
|
473
|
-
* - Sampler object
|
|
474
|
-
- None
|
|
475
|
-
- order defined by sampler
|
|
476
|
-
* - Sampler object
|
|
477
|
-
- True
|
|
478
|
-
- not allowed
|
|
479
|
-
* - Sampler object
|
|
480
|
-
- False
|
|
481
|
-
- not allowed
|
|
422
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
423
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
424
|
+
|
|
425
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
482
426
|
|
|
483
427
|
Examples:
|
|
428
|
+
>>> import mindspore.dataset as ds
|
|
484
429
|
>>> celeba_dataset_dir = "/path/to/celeba_dataset_directory"
|
|
485
430
|
>>>
|
|
486
431
|
>>> # Read 5 samples from CelebA dataset
|
|
@@ -589,26 +534,26 @@ class Cifar10Dataset(MappableDataset, VisionBaseDataset):
|
|
|
589
534
|
|
|
590
535
|
Args:
|
|
591
536
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
592
|
-
usage (str, optional): Usage of this dataset, can be 'train', 'test' or 'all' .
|
|
593
|
-
train
|
|
594
|
-
Default: None, all samples.
|
|
537
|
+
usage (str, optional): Usage of this dataset, can be ``'train'`` , ``'test'`` or ``'all'`` .
|
|
538
|
+
``'train'`` will read from 50,000 train samples, ``'test'`` will read from 10,000 test
|
|
539
|
+
samples, ``'all'`` will read from all 60,000 samples. Default: ``None`` , all samples.
|
|
595
540
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
596
|
-
Default: None, all images.
|
|
541
|
+
Default: ``None`` , all images.
|
|
597
542
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
598
|
-
Default: None, will use global default workers(8), it can be set
|
|
599
|
-
by
|
|
600
|
-
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None, expected
|
|
543
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
544
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
545
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected
|
|
601
546
|
order behavior shown in the table below.
|
|
602
547
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
603
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
548
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
604
549
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
605
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
550
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
606
551
|
the maximum sample number of per shard.
|
|
607
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
552
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
608
553
|
argument can only be specified when `num_shards` is also specified.
|
|
609
554
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
610
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
611
|
-
Default: None, which means no cache is used.
|
|
555
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
556
|
+
Default: ``None`` , which means no cache is used.
|
|
612
557
|
|
|
613
558
|
Raises:
|
|
614
559
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -618,39 +563,20 @@ class Cifar10Dataset(MappableDataset, VisionBaseDataset):
|
|
|
618
563
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
619
564
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
620
565
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
621
|
-
ValueError: If `usage` is not 'train', 'test' or 'all'.
|
|
566
|
+
ValueError: If `usage` is not ``'train'`` , ``'test'`` or ``'all'`` .
|
|
567
|
+
|
|
568
|
+
Tutorial Examples:
|
|
569
|
+
- `Load & Process Data With Dataset Pipeline
|
|
570
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
622
571
|
|
|
623
572
|
Note:
|
|
624
|
-
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
..
|
|
628
|
-
:widths: 25 25 50
|
|
629
|
-
:header-rows: 1
|
|
630
|
-
|
|
631
|
-
* - Parameter `sampler`
|
|
632
|
-
- Parameter `shuffle`
|
|
633
|
-
- Expected Order Behavior
|
|
634
|
-
* - None
|
|
635
|
-
- None
|
|
636
|
-
- random order
|
|
637
|
-
* - None
|
|
638
|
-
- True
|
|
639
|
-
- random order
|
|
640
|
-
* - None
|
|
641
|
-
- False
|
|
642
|
-
- sequential order
|
|
643
|
-
* - Sampler object
|
|
644
|
-
- None
|
|
645
|
-
- order defined by sampler
|
|
646
|
-
* - Sampler object
|
|
647
|
-
- True
|
|
648
|
-
- not allowed
|
|
649
|
-
* - Sampler object
|
|
650
|
-
- False
|
|
651
|
-
- not allowed
|
|
573
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
574
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
575
|
+
|
|
576
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
652
577
|
|
|
653
578
|
Examples:
|
|
579
|
+
>>> import mindspore.dataset as ds
|
|
654
580
|
>>> cifar10_dataset_dir = "/path/to/cifar10_dataset_directory"
|
|
655
581
|
>>>
|
|
656
582
|
>>> # 1) Get all samples from CIFAR10 dataset in sequence
|
|
@@ -722,26 +648,26 @@ class Cifar100Dataset(MappableDataset, VisionBaseDataset):
|
|
|
722
648
|
|
|
723
649
|
Args:
|
|
724
650
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
725
|
-
usage (str, optional): Usage of this dataset, can be 'train', 'test' or 'all' .
|
|
726
|
-
train
|
|
727
|
-
Default: None, all samples.
|
|
651
|
+
usage (str, optional): Usage of this dataset, can be ``'train'`` , ``'test'`` or ``'all'`` .
|
|
652
|
+
``'train'`` will read from 50,000 train samples, ``'test'`` will read from 10,000 test samples,
|
|
653
|
+
``'all'`` will read from all 60,000 samples. Default: ``None`` , all samples.
|
|
728
654
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
729
|
-
Default: None, all images.
|
|
655
|
+
Default: ``None`` , all images.
|
|
730
656
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
731
|
-
Default: None, will use global default workers(8), it can be set
|
|
732
|
-
by
|
|
733
|
-
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None, expected
|
|
657
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
658
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
659
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected
|
|
734
660
|
order behavior shown in the table below.
|
|
735
661
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
736
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
662
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
737
663
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
738
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
664
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
739
665
|
the maximum sample number of per shard.
|
|
740
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
666
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
741
667
|
argument can only be specified when `num_shards` is also specified.
|
|
742
668
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
743
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
744
|
-
Default: None, which means no cache is used.
|
|
669
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
670
|
+
Default: ``None`` , which means no cache is used.
|
|
745
671
|
|
|
746
672
|
Raises:
|
|
747
673
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -751,39 +677,20 @@ class Cifar100Dataset(MappableDataset, VisionBaseDataset):
|
|
|
751
677
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
752
678
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
753
679
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
754
|
-
ValueError: If `usage` is not 'train', 'test' or 'all'.
|
|
680
|
+
ValueError: If `usage` is not ``'train'`` , ``'test'`` or ``'all'`` .
|
|
681
|
+
|
|
682
|
+
Tutorial Examples:
|
|
683
|
+
- `Load & Process Data With Dataset Pipeline
|
|
684
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
755
685
|
|
|
756
686
|
Note:
|
|
757
|
-
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
..
|
|
761
|
-
:widths: 25 25 50
|
|
762
|
-
:header-rows: 1
|
|
763
|
-
|
|
764
|
-
* - Parameter `sampler`
|
|
765
|
-
- Parameter `shuffle`
|
|
766
|
-
- Expected Order Behavior
|
|
767
|
-
* - None
|
|
768
|
-
- None
|
|
769
|
-
- random order
|
|
770
|
-
* - None
|
|
771
|
-
- True
|
|
772
|
-
- random order
|
|
773
|
-
* - None
|
|
774
|
-
- False
|
|
775
|
-
- sequential order
|
|
776
|
-
* - Sampler object
|
|
777
|
-
- None
|
|
778
|
-
- order defined by sampler
|
|
779
|
-
* - Sampler object
|
|
780
|
-
- True
|
|
781
|
-
- not allowed
|
|
782
|
-
* - Sampler object
|
|
783
|
-
- False
|
|
784
|
-
- not allowed
|
|
687
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
688
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
689
|
+
|
|
690
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
785
691
|
|
|
786
692
|
Examples:
|
|
693
|
+
>>> import mindspore.dataset as ds
|
|
787
694
|
>>> cifar100_dataset_dir = "/path/to/cifar100_dataset_directory"
|
|
788
695
|
>>>
|
|
789
696
|
>>> # 1) Get all samples from CIFAR100 dataset in sequence
|
|
@@ -845,34 +752,36 @@ class CityscapesDataset(MappableDataset, VisionBaseDataset):
|
|
|
845
752
|
|
|
846
753
|
The generated dataset has two columns :py:obj:`[image, task]` .
|
|
847
754
|
The tensor of column :py:obj:`image` is of the uint8 type.
|
|
848
|
-
The tensor of column :py:obj:`task` is of the uint8 type if task is not 'polygon' otherwise task is
|
|
755
|
+
The tensor of column :py:obj:`task` is of the uint8 type if `task` is not ``'polygon'`` otherwise task is
|
|
849
756
|
a string tensor with serialize json.
|
|
850
757
|
|
|
851
758
|
Args:
|
|
852
759
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
853
|
-
usage (str, optional): Acceptable usages include 'train'
|
|
854
|
-
otherwise 'train'
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
760
|
+
usage (str, optional): Acceptable usages include ``'train'``, ``'test'``, ``'val'`` or ``'all'``
|
|
761
|
+
if `quality_mode` is ``'fine'`` otherwise ``'train'``, ``'train_extra'``, ``'val'`` or
|
|
762
|
+
``'all'``. Default: ``'train'``.
|
|
763
|
+
quality_mode (str, optional): Acceptable quality_modes include ``'fine'`` or ``'coarse'``.
|
|
764
|
+
Default: ``'fine'``.
|
|
765
|
+
task (str, optional): Acceptable tasks include ``'instance'``,
|
|
766
|
+
``'semantic'``, ``'polygon'`` or ``'color'``. Default: ``'instance'``.
|
|
858
767
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
859
|
-
Default: None, all images.
|
|
768
|
+
Default: ``None`` , all images.
|
|
860
769
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
861
|
-
Default: None, will use global default workers(8), it can be set
|
|
862
|
-
by
|
|
863
|
-
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None, expected
|
|
770
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
771
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
772
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected
|
|
864
773
|
order behavior shown in the table below.
|
|
865
|
-
decode (bool, optional): Decode the images after reading. Default: False
|
|
774
|
+
decode (bool, optional): Decode the images after reading. Default: ``None``, default to be ``False``.
|
|
866
775
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
867
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
776
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
868
777
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
869
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
778
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
870
779
|
the max sample number of per shard.
|
|
871
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
780
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
872
781
|
argument can only be specified when `num_shards` is also specified.
|
|
873
782
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
874
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
875
|
-
Default: None, which means no cache is used.
|
|
783
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
784
|
+
Default: ``None`` , which means no cache is used.
|
|
876
785
|
|
|
877
786
|
Raises:
|
|
878
787
|
RuntimeError: If `dataset_dir` is invalid or does not contain data files.
|
|
@@ -882,42 +791,23 @@ class CityscapesDataset(MappableDataset, VisionBaseDataset):
|
|
|
882
791
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
883
792
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
884
793
|
ValueError: If `dataset_dir` is not exist.
|
|
885
|
-
ValueError: If `task` is
|
|
886
|
-
ValueError: If `quality_mode` is
|
|
794
|
+
ValueError: If `task` is not ``'instance'``, ``'semantic'``, ``'polygon'`` or ``'color'``.
|
|
795
|
+
ValueError: If `quality_mode` is not ``'fine'`` or ``'coarse'``.
|
|
887
796
|
ValueError: If `usage` is invalid.
|
|
888
797
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
889
798
|
|
|
799
|
+
Tutorial Examples:
|
|
800
|
+
- `Load & Process Data With Dataset Pipeline
|
|
801
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
802
|
+
|
|
890
803
|
Note:
|
|
891
|
-
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
..
|
|
895
|
-
:widths: 25 25 50
|
|
896
|
-
:header-rows: 1
|
|
897
|
-
|
|
898
|
-
* - Parameter `sampler`
|
|
899
|
-
- Parameter `shuffle`
|
|
900
|
-
- Expected Order Behavior
|
|
901
|
-
* - None
|
|
902
|
-
- None
|
|
903
|
-
- random order
|
|
904
|
-
* - None
|
|
905
|
-
- True
|
|
906
|
-
- random order
|
|
907
|
-
* - None
|
|
908
|
-
- False
|
|
909
|
-
- sequential order
|
|
910
|
-
* - Sampler object
|
|
911
|
-
- None
|
|
912
|
-
- order defined by sampler
|
|
913
|
-
* - Sampler object
|
|
914
|
-
- True
|
|
915
|
-
- not allowed
|
|
916
|
-
* - Sampler object
|
|
917
|
-
- False
|
|
918
|
-
- not allowed
|
|
804
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
805
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
806
|
+
|
|
807
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
919
808
|
|
|
920
809
|
Examples:
|
|
810
|
+
>>> import mindspore.dataset as ds
|
|
921
811
|
>>> cityscapes_dataset_dir = "/path/to/cityscapes_dataset_directory"
|
|
922
812
|
>>>
|
|
923
813
|
>>> # 1) Get all samples from Cityscapes dataset in sequence
|
|
@@ -1023,29 +913,30 @@ class CocoDataset(MappableDataset, VisionBaseDataset):
|
|
|
1023
913
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
1024
914
|
annotation_file (str): Path to the annotation JSON file.
|
|
1025
915
|
task (str, optional): Set the task type for reading COCO data. Supported task types:
|
|
1026
|
-
'Detection'
|
|
916
|
+
``'Detection'``, ``'Stuff'``, ``'Panoptic'``, ``'Keypoint'`` and ``'Captioning'``.
|
|
917
|
+
Default: ``'Detection'``.
|
|
1027
918
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
1028
|
-
Default: None, all images.
|
|
919
|
+
Default: ``None`` , all images.
|
|
1029
920
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
1030
|
-
Default: None, will use global default workers(8), it can be set
|
|
1031
|
-
by
|
|
1032
|
-
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None, expected
|
|
921
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
922
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
923
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected
|
|
1033
924
|
order behavior shown in the table below.
|
|
1034
|
-
decode (bool, optional): Decode the images after reading. Default: False
|
|
925
|
+
decode (bool, optional): Decode the images after reading. Default: ``False``.
|
|
1035
926
|
sampler (Sampler, optional): Object used to choose samples from the dataset.
|
|
1036
|
-
Default: None, expected order behavior shown in the table below.
|
|
927
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
1037
928
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
1038
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
929
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
1039
930
|
the maximum sample number of per shard.
|
|
1040
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
931
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
1041
932
|
argument can only be specified when `num_shards` is also specified.
|
|
1042
933
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
1043
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
1044
|
-
Default: None, which means no cache is used.
|
|
934
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
935
|
+
Default: ``None`` , which means no cache is used.
|
|
1045
936
|
extra_metadata(bool, optional): Flag to add extra meta-data to row. If True, an additional column will be
|
|
1046
|
-
output at the end :py:obj:`[_meta-filename, dtype=string]` . Default: False
|
|
937
|
+
output at the end :py:obj:`[_meta-filename, dtype=string]` . Default: ``False``.
|
|
1047
938
|
decrypt (callable, optional): Image decryption function, which accepts the path of the encrypted image file
|
|
1048
|
-
and returns the decrypted bytes data. Default: None, no decryption.
|
|
939
|
+
and returns the decrypted bytes data. Default: ``None`` , no decryption.
|
|
1049
940
|
|
|
1050
941
|
The generated dataset with different task setting has different output columns:
|
|
1051
942
|
|
|
@@ -1091,45 +982,27 @@ class CocoDataset(MappableDataset, VisionBaseDataset):
|
|
|
1091
982
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
1092
983
|
RuntimeError: If parse JSON file failed.
|
|
1093
984
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
1094
|
-
ValueError: If `task` is not
|
|
985
|
+
ValueError: If `task` is not ``'Detection'``, ``'Stuff'``, ``'Panoptic'``, ``'Keypoint'``
|
|
986
|
+
or ``'Captioning'``.
|
|
1095
987
|
ValueError: If `annotation_file` is not exist.
|
|
1096
988
|
ValueError: If `dataset_dir` is not exist.
|
|
1097
989
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
1098
990
|
|
|
991
|
+
Tutorial Examples:
|
|
992
|
+
- `Load & Process Data With Dataset Pipeline
|
|
993
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
994
|
+
|
|
1099
995
|
Note:
|
|
1100
996
|
- Column '[_meta-filename, dtype=string]' won't be output unless an explicit rename dataset op is added
|
|
1101
997
|
to remove the prefix('_meta-').
|
|
1102
|
-
- Not support
|
|
1103
|
-
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
..
|
|
1107
|
-
:widths: 25 25 50
|
|
1108
|
-
:header-rows: 1
|
|
1109
|
-
|
|
1110
|
-
* - Parameter `sampler`
|
|
1111
|
-
- Parameter `shuffle`
|
|
1112
|
-
- Expected Order Behavior
|
|
1113
|
-
* - None
|
|
1114
|
-
- None
|
|
1115
|
-
- random order
|
|
1116
|
-
* - None
|
|
1117
|
-
- True
|
|
1118
|
-
- random order
|
|
1119
|
-
* - None
|
|
1120
|
-
- False
|
|
1121
|
-
- sequential order
|
|
1122
|
-
* - Sampler object
|
|
1123
|
-
- None
|
|
1124
|
-
- order defined by sampler
|
|
1125
|
-
* - Sampler object
|
|
1126
|
-
- True
|
|
1127
|
-
- not allowed
|
|
1128
|
-
* - Sampler object
|
|
1129
|
-
- False
|
|
1130
|
-
- not allowed
|
|
998
|
+
- Not support :class:`mindspore.dataset.PKSampler` for `sampler` parameter yet.
|
|
999
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
1000
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
1001
|
+
|
|
1002
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
1131
1003
|
|
|
1132
1004
|
Examples:
|
|
1005
|
+
>>> import mindspore.dataset as ds
|
|
1133
1006
|
>>> coco_dataset_dir = "/path/to/coco_dataset_directory/images"
|
|
1134
1007
|
>>> coco_annotation_file = "/path/to/coco_dataset_directory/annotation_file"
|
|
1135
1008
|
>>>
|
|
@@ -1234,12 +1107,17 @@ class CocoDataset(MappableDataset, VisionBaseDataset):
|
|
|
1234
1107
|
|
|
1235
1108
|
def get_class_indexing(self):
|
|
1236
1109
|
"""
|
|
1237
|
-
Get the
|
|
1110
|
+
Get the mapping dictionary from category names to category indexes.
|
|
1111
|
+
|
|
1112
|
+
This dictionary can be used to look up which category name corresponds to a particular category index.
|
|
1238
1113
|
|
|
1239
1114
|
Returns:
|
|
1240
|
-
|
|
1115
|
+
Dict[str, List[int]], the mappings from category names to category index list. The first
|
|
1116
|
+
element of the list is always the category ID. Only in Panoptic tasks, the second element
|
|
1117
|
+
of the list indicates whether the category is a thing or a stuff.
|
|
1241
1118
|
|
|
1242
1119
|
Examples:
|
|
1120
|
+
>>> import mindspore.dataset as ds
|
|
1243
1121
|
>>> coco_dataset_dir = "/path/to/coco_dataset_directory/images"
|
|
1244
1122
|
>>> coco_annotation_file = "/path/to/coco_dataset_directory/annotation_file"
|
|
1245
1123
|
>>>
|
|
@@ -1267,31 +1145,32 @@ class DIV2KDataset(MappableDataset, VisionBaseDataset):
|
|
|
1267
1145
|
|
|
1268
1146
|
Args:
|
|
1269
1147
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
1270
|
-
usage (str, optional): Acceptable usages include 'train'
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
When `downgrade` is '
|
|
1276
|
-
When `downgrade` is '
|
|
1148
|
+
usage (str, optional): Acceptable usages include ``'train'``, ``'valid'`` or ``'all'``.
|
|
1149
|
+
Default: ``'train'``.
|
|
1150
|
+
downgrade (str, optional): Acceptable downgrades include ``'bicubic'``, ``'unknown'``, ``'mild'``,
|
|
1151
|
+
``'difficult'`` or ``'wild'``. Default: ``'bicubic'``.
|
|
1152
|
+
scale (int, optional): Acceptable scales include ``2``, ``3``, ``4`` or ``8``. Default: ``2``.
|
|
1153
|
+
When `downgrade` is ``'bicubic'``, scale can be ``2``, ``3``, ``4``, ``8``.
|
|
1154
|
+
When `downgrade` is ``'unknown'``, scale can only be ``2``, ``3``, ``4``.
|
|
1155
|
+
When `downgrade` is ``'mild'``, ``'difficult'`` or ``'wild'``, scale can only be ``4``.
|
|
1277
1156
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
1278
|
-
Default: None, all images.
|
|
1157
|
+
Default: ``None`` , all images.
|
|
1279
1158
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
1280
|
-
Default: None, will use global default workers(8), it can be set
|
|
1281
|
-
by
|
|
1282
|
-
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None, expected
|
|
1159
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
1160
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
1161
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected
|
|
1283
1162
|
order behavior shown in the table below.
|
|
1284
|
-
decode (bool, optional): Decode the images after reading. Default: False
|
|
1163
|
+
decode (bool, optional): Decode the images after reading. Default: ``None``, set to ``False``.
|
|
1285
1164
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
1286
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
1165
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
1287
1166
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
1288
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
1167
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
1289
1168
|
the max sample number of per shard.
|
|
1290
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
1169
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
1291
1170
|
argument can only be specified when `num_shards` is also specified.
|
|
1292
1171
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
1293
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
1294
|
-
Default: None, which means no cache is used.
|
|
1172
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
1173
|
+
Default: ``None`` , which means no cache is used.
|
|
1295
1174
|
|
|
1296
1175
|
Raises:
|
|
1297
1176
|
RuntimeError: If `dataset_dir` is invalid or does not contain data files.
|
|
@@ -1304,41 +1183,23 @@ class DIV2KDataset(MappableDataset, VisionBaseDataset):
|
|
|
1304
1183
|
ValueError: If `usage` is invalid.
|
|
1305
1184
|
ValueError: If `downgrade` is invalid.
|
|
1306
1185
|
ValueError: If `scale` is invalid.
|
|
1307
|
-
ValueError: If `scale` equal to 8 and downgrade not equal to 'bicubic'
|
|
1308
|
-
ValueError: If `downgrade`
|
|
1186
|
+
ValueError: If `scale` equal to ``8`` and downgrade not equal to ``'bicubic'``.
|
|
1187
|
+
ValueError: If `downgrade` is ``'mild'``, ``'difficult'`` or ``'wild'``, and `scale`
|
|
1188
|
+
not equal to ``4``.
|
|
1309
1189
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
1310
1190
|
|
|
1191
|
+
Tutorial Examples:
|
|
1192
|
+
- `Load & Process Data With Dataset Pipeline
|
|
1193
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
1194
|
+
|
|
1311
1195
|
Note:
|
|
1312
|
-
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
..
|
|
1316
|
-
:widths: 25 25 50
|
|
1317
|
-
:header-rows: 1
|
|
1318
|
-
|
|
1319
|
-
* - Parameter `sampler`
|
|
1320
|
-
- Parameter `shuffle`
|
|
1321
|
-
- Expected Order Behavior
|
|
1322
|
-
* - None
|
|
1323
|
-
- None
|
|
1324
|
-
- random order
|
|
1325
|
-
* - None
|
|
1326
|
-
- True
|
|
1327
|
-
- random order
|
|
1328
|
-
* - None
|
|
1329
|
-
- False
|
|
1330
|
-
- sequential order
|
|
1331
|
-
* - Sampler object
|
|
1332
|
-
- None
|
|
1333
|
-
- order defined by sampler
|
|
1334
|
-
* - Sampler object
|
|
1335
|
-
- True
|
|
1336
|
-
- not allowed
|
|
1337
|
-
* - Sampler object
|
|
1338
|
-
- False
|
|
1339
|
-
- not allowed
|
|
1196
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
1197
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
1198
|
+
|
|
1199
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
1340
1200
|
|
|
1341
1201
|
Examples:
|
|
1202
|
+
>>> import mindspore.dataset as ds
|
|
1342
1203
|
>>> div2k_dataset_dir = "/path/to/div2k_dataset_directory"
|
|
1343
1204
|
>>>
|
|
1344
1205
|
>>> # 1) Get all samples from DIV2K dataset in sequence
|
|
@@ -1457,27 +1318,27 @@ class EMnistDataset(MappableDataset, VisionBaseDataset):
|
|
|
1457
1318
|
|
|
1458
1319
|
Args:
|
|
1459
1320
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
1460
|
-
name (str): Name of splits for this dataset, can be 'byclass'
|
|
1461
|
-
or 'mnist'
|
|
1462
|
-
usage (str, optional): Usage of this dataset, can be 'train'
|
|
1463
|
-
train
|
|
1464
|
-
Default: None, will read all samples.
|
|
1321
|
+
name (str): Name of splits for this dataset, can be ``'byclass'``, ``'bymerge'``, ``'balanced'``,
|
|
1322
|
+
``'letters'``, ``'digits'`` or ``'mnist'``.
|
|
1323
|
+
usage (str, optional): Usage of this dataset, can be ``'train'``, ``'test'`` or ``'all'``.
|
|
1324
|
+
``'train'`` will read from 60,000 train samples, ``'test'`` will read from 10,000 test samples,
|
|
1325
|
+
``'all'`` will read from all 70,000 samples. Default: ``None`` , will read all samples.
|
|
1465
1326
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
1466
|
-
Default: None, will read all images.
|
|
1327
|
+
Default: ``None`` , will read all images.
|
|
1467
1328
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
1468
|
-
Default: None, will use global default workers(8), it can be set
|
|
1469
|
-
by
|
|
1329
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
1330
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
1470
1331
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
1471
|
-
Default: None, expected order behavior shown in the table below.
|
|
1332
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
1472
1333
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
1473
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
1474
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
1334
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
1335
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
1475
1336
|
When this argument is specified, `num_samples` reflects the max sample number of per shard.
|
|
1476
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
1337
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
1477
1338
|
argument can only be specified when `num_shards` is also specified.
|
|
1478
1339
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
1479
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
1480
|
-
Default: None, which means no cache is used.
|
|
1340
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
1341
|
+
Default: ``None`` , which means no cache is used.
|
|
1481
1342
|
|
|
1482
1343
|
Raises:
|
|
1483
1344
|
RuntimeError: If `sampler` and `shuffle` are specified at the same time.
|
|
@@ -1486,37 +1347,18 @@ class EMnistDataset(MappableDataset, VisionBaseDataset):
|
|
|
1486
1347
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
1487
1348
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
1488
1349
|
|
|
1350
|
+
Tutorial Examples:
|
|
1351
|
+
- `Load & Process Data With Dataset Pipeline
|
|
1352
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
1353
|
+
|
|
1489
1354
|
Note:
|
|
1490
|
-
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
..
|
|
1494
|
-
:widths: 25 25 50
|
|
1495
|
-
:header-rows: 1
|
|
1496
|
-
|
|
1497
|
-
* - Parameter `sampler`
|
|
1498
|
-
- Parameter `shuffle`
|
|
1499
|
-
- Expected Order Behavior
|
|
1500
|
-
* - None
|
|
1501
|
-
- None
|
|
1502
|
-
- random order
|
|
1503
|
-
* - None
|
|
1504
|
-
- True
|
|
1505
|
-
- random order
|
|
1506
|
-
* - None
|
|
1507
|
-
- False
|
|
1508
|
-
- sequential order
|
|
1509
|
-
* - Sampler object
|
|
1510
|
-
- None
|
|
1511
|
-
- order defined by sampler
|
|
1512
|
-
* - Sampler object
|
|
1513
|
-
- True
|
|
1514
|
-
- not allowed
|
|
1515
|
-
* - Sampler object
|
|
1516
|
-
- False
|
|
1517
|
-
- not allowed
|
|
1355
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
1356
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
1357
|
+
|
|
1358
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
1518
1359
|
|
|
1519
1360
|
Examples:
|
|
1361
|
+
>>> import mindspore.dataset as ds
|
|
1520
1362
|
>>> emnist_dataset_dir = "/path/to/emnist_dataset_directory"
|
|
1521
1363
|
>>>
|
|
1522
1364
|
>>> # Read 3 samples from EMNIST dataset
|
|
@@ -1591,26 +1433,27 @@ class FakeImageDataset(MappableDataset, VisionBaseDataset):
|
|
|
1591
1433
|
The column :py:obj:`label` is a scalar of the uint32 type.
|
|
1592
1434
|
|
|
1593
1435
|
Args:
|
|
1594
|
-
num_images (int, optional): Number of images to generate in the dataset. Default: 1000
|
|
1595
|
-
image_size (tuple, optional): Size of the fake image. Default: (224, 224, 3)
|
|
1596
|
-
num_classes (int, optional): Number of classes in the dataset. Default: 10
|
|
1597
|
-
base_seed (int, optional): Offsets the index-based random seed used to generate each image.
|
|
1436
|
+
num_images (int, optional): Number of images to generate in the dataset. Default: ``1000``.
|
|
1437
|
+
image_size (tuple, optional): Size of the fake image. Default: ``(224, 224, 3)``.
|
|
1438
|
+
num_classes (int, optional): Number of classes in the dataset. Default: ``10``.
|
|
1439
|
+
base_seed (int, optional): Offsets the index-based random seed used to generate each image.
|
|
1440
|
+
Default: ``0``.
|
|
1598
1441
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
1599
|
-
Default: None, will read all images.
|
|
1442
|
+
Default: ``None`` , will read all images.
|
|
1600
1443
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
1601
|
-
Default: None, will use global default workers(8), it can be set
|
|
1602
|
-
by
|
|
1444
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
1445
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
1603
1446
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
1604
|
-
Default: None, expected order behavior shown in the table below.
|
|
1447
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
1605
1448
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
1606
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
1607
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
1449
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
1450
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
1608
1451
|
When this argument is specified, `num_samples` reflects the max sample number of per shard.
|
|
1609
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
1452
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
1610
1453
|
argument can only be specified when `num_shards` is also specified.
|
|
1611
1454
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
1612
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
1613
|
-
Default: None, which means no cache is used.
|
|
1455
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
1456
|
+
Default: ``None`` , which means no cache is used.
|
|
1614
1457
|
|
|
1615
1458
|
Raises:
|
|
1616
1459
|
RuntimeError: If `sampler` and `shuffle` are specified at the same time.
|
|
@@ -1620,37 +1463,18 @@ class FakeImageDataset(MappableDataset, VisionBaseDataset):
|
|
|
1620
1463
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
1621
1464
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
1622
1465
|
|
|
1466
|
+
Tutorial Examples:
|
|
1467
|
+
- `Load & Process Data With Dataset Pipeline
|
|
1468
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
1469
|
+
|
|
1623
1470
|
Note:
|
|
1624
|
-
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
..
|
|
1628
|
-
:widths: 25 25 50
|
|
1629
|
-
:header-rows: 1
|
|
1630
|
-
|
|
1631
|
-
* - Parameter 'sampler'
|
|
1632
|
-
- Parameter 'shuffle'
|
|
1633
|
-
- Expected Order Behavior
|
|
1634
|
-
* - None
|
|
1635
|
-
- None
|
|
1636
|
-
- random order
|
|
1637
|
-
* - None
|
|
1638
|
-
- True
|
|
1639
|
-
- random order
|
|
1640
|
-
* - None
|
|
1641
|
-
- False
|
|
1642
|
-
- sequential order
|
|
1643
|
-
* - Sampler object
|
|
1644
|
-
- None
|
|
1645
|
-
- order defined by sampler
|
|
1646
|
-
* - Sampler object
|
|
1647
|
-
- True
|
|
1648
|
-
- not allowed
|
|
1649
|
-
* - Sampler object
|
|
1650
|
-
- False
|
|
1651
|
-
- not allowed
|
|
1471
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
1472
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
1473
|
+
|
|
1474
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
1652
1475
|
|
|
1653
1476
|
Examples:
|
|
1477
|
+
>>> import mindspore.dataset as ds
|
|
1654
1478
|
>>> # Read 3 samples from FakeImage dataset
|
|
1655
1479
|
>>> dataset = ds.FakeImageDataset(num_images=1000, image_size=(224,224,3),
|
|
1656
1480
|
... num_classes=10, base_seed=0, num_samples=3)
|
|
@@ -1681,25 +1505,26 @@ class FashionMnistDataset(MappableDataset, VisionBaseDataset):
|
|
|
1681
1505
|
|
|
1682
1506
|
Args:
|
|
1683
1507
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
1684
|
-
usage (str, optional): Usage of this dataset, can be 'train'
|
|
1685
|
-
train
|
|
1686
|
-
|
|
1508
|
+
usage (str, optional): Usage of this dataset, can be ``'train'``, ``'test'`` or ``'all'`` .
|
|
1509
|
+
``'train'`` will read from 60,000 train samples, ``'test'`` will read from 10,000 test
|
|
1510
|
+
samples, ``'all'`` will read from all 70,000 samples.
|
|
1511
|
+
Default: ``None`` , will read all samples.
|
|
1687
1512
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
1688
|
-
Default: None, will read all images.
|
|
1513
|
+
Default: ``None`` , will read all images.
|
|
1689
1514
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
1690
|
-
Default: None, will use global default workers(8), it can be set
|
|
1691
|
-
by
|
|
1515
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
1516
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
1692
1517
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
1693
|
-
Default: None, expected order behavior shown in the table below.
|
|
1518
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
1694
1519
|
sampler (Sampler, optional): Object used to choose samples from the dataset.
|
|
1695
|
-
Default: None, expected order behavior shown in the table below.
|
|
1696
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
1520
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
1521
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
1697
1522
|
When this argument is specified, `num_samples` reflects the maximum sample number of per shard.
|
|
1698
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
1523
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
1699
1524
|
argument can only be specified when `num_shards` is also specified.
|
|
1700
1525
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
1701
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
1702
|
-
Default: None, which means no cache is used.
|
|
1526
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
1527
|
+
Default: ``None`` , which means no cache is used.
|
|
1703
1528
|
|
|
1704
1529
|
Raises:
|
|
1705
1530
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -1710,37 +1535,18 @@ class FashionMnistDataset(MappableDataset, VisionBaseDataset):
|
|
|
1710
1535
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
1711
1536
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
1712
1537
|
|
|
1538
|
+
Tutorial Examples:
|
|
1539
|
+
- `Load & Process Data With Dataset Pipeline
|
|
1540
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
1541
|
+
|
|
1713
1542
|
Note:
|
|
1714
|
-
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
..
|
|
1718
|
-
:widths: 25 25 50
|
|
1719
|
-
:header-rows: 1
|
|
1720
|
-
|
|
1721
|
-
* - Parameter `sampler`
|
|
1722
|
-
- Parameter `shuffle`
|
|
1723
|
-
- Expected Order Behavior
|
|
1724
|
-
* - None
|
|
1725
|
-
- None
|
|
1726
|
-
- random order
|
|
1727
|
-
* - None
|
|
1728
|
-
- True
|
|
1729
|
-
- random order
|
|
1730
|
-
* - None
|
|
1731
|
-
- False
|
|
1732
|
-
- sequential order
|
|
1733
|
-
* - Sampler object
|
|
1734
|
-
- None
|
|
1735
|
-
- order defined by sampler
|
|
1736
|
-
* - Sampler object
|
|
1737
|
-
- True
|
|
1738
|
-
- not allowed
|
|
1739
|
-
* - Sampler object
|
|
1740
|
-
- False
|
|
1741
|
-
- not allowed
|
|
1543
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
1544
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
1545
|
+
|
|
1546
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
1742
1547
|
|
|
1743
1548
|
Examples:
|
|
1549
|
+
>>> import mindspore.dataset as ds
|
|
1744
1550
|
>>> fashion_mnist_dataset_dir = "/path/to/fashion_mnist_dataset_directory"
|
|
1745
1551
|
>>>
|
|
1746
1552
|
>>> # Read 3 samples from FASHIONMNIST dataset
|
|
@@ -1807,23 +1613,23 @@ class FlickrDataset(MappableDataset, VisionBaseDataset):
|
|
|
1807
1613
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
1808
1614
|
annotation_file (str): Path to the root directory that contains the annotation.
|
|
1809
1615
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
1810
|
-
Default: None, all images.
|
|
1616
|
+
Default: ``None`` , all images.
|
|
1811
1617
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
1812
|
-
Default: None, will use global default workers(8), it can be set
|
|
1813
|
-
by
|
|
1814
|
-
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None, expected
|
|
1618
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
1619
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
1620
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected
|
|
1815
1621
|
order behavior shown in the table below.
|
|
1816
|
-
decode (bool, optional): Decode the images after reading. Default: None.
|
|
1622
|
+
decode (bool, optional): Decode the images after reading. Default: ``None`` .
|
|
1817
1623
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
1818
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
1624
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
1819
1625
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
1820
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
1626
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
1821
1627
|
the max sample number of per shard.
|
|
1822
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
1628
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
1823
1629
|
argument can only be specified when `num_shards` is also specified.
|
|
1824
1630
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
1825
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
1826
|
-
Default: None, which means no cache is used.
|
|
1631
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
1632
|
+
Default: ``None`` , which means no cache is used.
|
|
1827
1633
|
|
|
1828
1634
|
Raises:
|
|
1829
1635
|
RuntimeError: If `dataset_dir` is not valid or does not contain data files.
|
|
@@ -1836,37 +1642,18 @@ class FlickrDataset(MappableDataset, VisionBaseDataset):
|
|
|
1836
1642
|
ValueError: If `annotation_file` is not exist.
|
|
1837
1643
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
1838
1644
|
|
|
1645
|
+
Tutorial Examples:
|
|
1646
|
+
- `Load & Process Data With Dataset Pipeline
|
|
1647
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
1648
|
+
|
|
1839
1649
|
Note:
|
|
1840
|
-
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
..
|
|
1844
|
-
:widths: 25 25 50
|
|
1845
|
-
:header-rows: 1
|
|
1846
|
-
|
|
1847
|
-
* - Parameter `sampler`
|
|
1848
|
-
- Parameter `shuffle`
|
|
1849
|
-
- Expected Order Behavior
|
|
1850
|
-
* - None
|
|
1851
|
-
- None
|
|
1852
|
-
- random order
|
|
1853
|
-
* - None
|
|
1854
|
-
- True
|
|
1855
|
-
- random order
|
|
1856
|
-
* - None
|
|
1857
|
-
- False
|
|
1858
|
-
- sequential order
|
|
1859
|
-
* - Sampler object
|
|
1860
|
-
- None
|
|
1861
|
-
- order defined by sampler
|
|
1862
|
-
* - Sampler object
|
|
1863
|
-
- True
|
|
1864
|
-
- not allowed
|
|
1865
|
-
* - Sampler object
|
|
1866
|
-
- False
|
|
1867
|
-
- not allowed
|
|
1650
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
1651
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
1652
|
+
|
|
1653
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
1868
1654
|
|
|
1869
1655
|
Examples:
|
|
1656
|
+
>>> import mindspore.dataset as ds
|
|
1870
1657
|
>>> flickr_dataset_dir = "/path/to/flickr_dataset_directory"
|
|
1871
1658
|
>>> annotation_file = "/path/to/flickr_annotation_file"
|
|
1872
1659
|
>>>
|
|
@@ -2050,21 +1837,24 @@ class Flowers102Dataset(GeneratorDataset):
|
|
|
2050
1837
|
|
|
2051
1838
|
Args:
|
|
2052
1839
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
2053
|
-
task (str, optional): Specify the 'Classification' or 'Segmentation' task.
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
1840
|
+
task (str, optional): Specify the ``'Classification'`` or ``'Segmentation'`` task.
|
|
1841
|
+
Default: ``'Classification'``.
|
|
1842
|
+
usage (str, optional): Specify the ``'train'``, ``'valid'``, ``'test'`` part or ``'all'``
|
|
1843
|
+
parts of dataset. Default: 'all', will read all samples.
|
|
1844
|
+
num_samples (int, optional): The number of samples to be included in the dataset.
|
|
1845
|
+
Default: ``None`` , all images.
|
|
2057
1846
|
num_parallel_workers (int, optional): Number of worker subprocesses used to
|
|
2058
|
-
fetch the dataset in parallel. Default: 1
|
|
1847
|
+
fetch the dataset in parallel. Default: ``1``.
|
|
2059
1848
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
2060
|
-
Default: None, expected order behavior shown in the table below.
|
|
2061
|
-
decode (bool, optional): Whether or not to decode the images and segmentations after reading.
|
|
1849
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
1850
|
+
decode (bool, optional): Whether or not to decode the images and segmentations after reading.
|
|
1851
|
+
Default: ``False``.
|
|
2062
1852
|
sampler (Union[Sampler, Iterable], optional): Object used to choose samples from the dataset.
|
|
2063
|
-
Default: None, expected order behavior shown in the table below.
|
|
2064
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
1853
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
1854
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
2065
1855
|
When this argument is specified, `num_samples` reflects the max sample number of per shard.
|
|
2066
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None.
|
|
2067
|
-
when `num_shards` is also specified.
|
|
1856
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` .
|
|
1857
|
+
This argument must be specified only when `num_shards` is also specified.
|
|
2068
1858
|
|
|
2069
1859
|
Raises:
|
|
2070
1860
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -2075,37 +1865,18 @@ class Flowers102Dataset(GeneratorDataset):
|
|
|
2075
1865
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
2076
1866
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
2077
1867
|
|
|
1868
|
+
Tutorial Examples:
|
|
1869
|
+
- `Load & Process Data With Dataset Pipeline
|
|
1870
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
1871
|
+
|
|
2078
1872
|
Note:
|
|
2079
|
-
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
..
|
|
2083
|
-
:widths: 25 25 50
|
|
2084
|
-
:header-rows: 1
|
|
2085
|
-
|
|
2086
|
-
* - Parameter 'sampler'
|
|
2087
|
-
- Parameter 'shuffle'
|
|
2088
|
-
- Expected Order Behavior
|
|
2089
|
-
* - None
|
|
2090
|
-
- None
|
|
2091
|
-
- random order
|
|
2092
|
-
* - None
|
|
2093
|
-
- True
|
|
2094
|
-
- random order
|
|
2095
|
-
* - None
|
|
2096
|
-
- False
|
|
2097
|
-
- sequential order
|
|
2098
|
-
* - Sampler object
|
|
2099
|
-
- None
|
|
2100
|
-
- order defined by sampler
|
|
2101
|
-
* - Sampler object
|
|
2102
|
-
- True
|
|
2103
|
-
- not allowed
|
|
2104
|
-
* - Sampler object
|
|
2105
|
-
- False
|
|
2106
|
-
- not allowed
|
|
1873
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
1874
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
1875
|
+
|
|
1876
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
2107
1877
|
|
|
2108
1878
|
Examples:
|
|
1879
|
+
>>> import mindspore.dataset as ds
|
|
2109
1880
|
>>> flowers102_dataset_dir = "/path/to/flowers102_dataset_directory"
|
|
2110
1881
|
>>> dataset = ds.Flowers102Dataset(dataset_dir=flowers102_dataset_dir,
|
|
2111
1882
|
... task="Classification",
|
|
@@ -2212,26 +1983,26 @@ class Food101Dataset(MappableDataset, VisionBaseDataset):
|
|
|
2212
1983
|
|
|
2213
1984
|
Args:
|
|
2214
1985
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
2215
|
-
usage (str, optional): Usage of this dataset, can be 'train', 'test', or 'all'.
|
|
2216
|
-
from 75,750 samples, 'test' will read from 25,250 samples, and 'all'
|
|
2217
|
-
and 'test' samples. Default: None, will be set to 'all'
|
|
1986
|
+
usage (str, optional): Usage of this dataset, can be ``'train'`` , ``'test'`` , or ``'all'`` .
|
|
1987
|
+
``'train'`` will read from 75,750 samples, ``'test'`` will read from 25,250 samples, and ``'all'``
|
|
1988
|
+
will read all ``'train'`` and ``'test'`` samples. Default: ``None`` , will be set to ``'all'``.
|
|
2218
1989
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
2219
|
-
Default: None, will read all images.
|
|
1990
|
+
Default: ``None`` , will read all images.
|
|
2220
1991
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
2221
|
-
Default: None, will use global default workers(8), it can be set
|
|
2222
|
-
by
|
|
1992
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
1993
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
2223
1994
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
2224
|
-
Default: None, expected order behavior shown in the table below.
|
|
2225
|
-
decode (bool, optional): Decode the images after reading. Default: False
|
|
1995
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
1996
|
+
decode (bool, optional): Decode the images after reading. Default: ``False``.
|
|
2226
1997
|
sampler (Sampler, optional): Object used to choose samples from the dataset.
|
|
2227
|
-
Default: None, expected order behavior shown in the table below.
|
|
1998
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
2228
1999
|
num_shards (int, optional): Number of shards that the dataset will be divided into. When this argument
|
|
2229
|
-
is specified, `num_samples` reflects the maximum sample number of per shard. Default: None.
|
|
2000
|
+
is specified, `num_samples` reflects the maximum sample number of per shard. Default: ``None`` .
|
|
2230
2001
|
shard_id (int, optional): The shard ID within `num_shards` . This argument can only be specified
|
|
2231
|
-
when `num_shards` is also specified. Default: None.
|
|
2002
|
+
when `num_shards` is also specified. Default: ``None`` .
|
|
2232
2003
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
2233
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
2234
|
-
Default: None, which means no cache is used.
|
|
2004
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
2005
|
+
Default: ``None`` , which means no cache is used.
|
|
2235
2006
|
|
|
2236
2007
|
Raises:
|
|
2237
2008
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -2241,40 +2012,21 @@ class Food101Dataset(MappableDataset, VisionBaseDataset):
|
|
|
2241
2012
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
2242
2013
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
2243
2014
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
2244
|
-
ValueError: If the value of `usage` is not 'train'
|
|
2015
|
+
ValueError: If the value of `usage` is not ``'train'``, ``'test'``, or ``'all'``.
|
|
2245
2016
|
ValueError: If `dataset_dir` is not exist.
|
|
2246
2017
|
|
|
2018
|
+
Tutorial Examples:
|
|
2019
|
+
- `Load & Process Data With Dataset Pipeline
|
|
2020
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
2021
|
+
|
|
2247
2022
|
Note:
|
|
2248
|
-
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
..
|
|
2252
|
-
:widths: 25 25 50
|
|
2253
|
-
:header-rows: 1
|
|
2254
|
-
|
|
2255
|
-
* - Parameter `sampler`
|
|
2256
|
-
- Parameter `shuffle`
|
|
2257
|
-
- Expected Order Behavior
|
|
2258
|
-
* - None
|
|
2259
|
-
- None
|
|
2260
|
-
- random order
|
|
2261
|
-
* - None
|
|
2262
|
-
- True
|
|
2263
|
-
- random order
|
|
2264
|
-
* - None
|
|
2265
|
-
- False
|
|
2266
|
-
- sequential order
|
|
2267
|
-
* - Sampler object
|
|
2268
|
-
- None
|
|
2269
|
-
- order defined by sampler
|
|
2270
|
-
* - Sampler object
|
|
2271
|
-
- True
|
|
2272
|
-
- not allowed
|
|
2273
|
-
* - Sampler object
|
|
2274
|
-
- False
|
|
2275
|
-
- not allowed
|
|
2023
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
2024
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
2025
|
+
|
|
2026
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
2276
2027
|
|
|
2277
2028
|
Examples:
|
|
2029
|
+
>>> import mindspore.dataset as ds
|
|
2278
2030
|
>>> food101_dataset_dir = "/path/to/food101_dataset_directory"
|
|
2279
2031
|
>>>
|
|
2280
2032
|
>>> # Read 3 samples from Food101 dataset
|
|
@@ -2349,31 +2101,31 @@ class ImageFolderDataset(MappableDataset, VisionBaseDataset):
|
|
|
2349
2101
|
Args:
|
|
2350
2102
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
2351
2103
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
2352
|
-
Default: None, all images.
|
|
2104
|
+
Default: ``None`` , all images.
|
|
2353
2105
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
2354
|
-
Default: None, will use global default workers(8), it can be set
|
|
2355
|
-
by
|
|
2106
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
2107
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
2356
2108
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
2357
|
-
Default: None, expected order behavior shown in the table below.
|
|
2109
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
2358
2110
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
2359
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
2111
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
2360
2112
|
extensions (list[str], optional): List of file extensions to be
|
|
2361
|
-
included in the dataset. Default: None.
|
|
2113
|
+
included in the dataset. Default: ``None`` .
|
|
2362
2114
|
class_indexing (dict, optional): A str-to-int mapping from folder name to index
|
|
2363
|
-
Default: None, the folder names will be sorted
|
|
2115
|
+
Default: ``None`` , the folder names will be sorted
|
|
2364
2116
|
alphabetically and each class will be given a
|
|
2365
2117
|
unique index starting from 0.
|
|
2366
|
-
decode (bool, optional): Decode the images after reading. Default: False
|
|
2118
|
+
decode (bool, optional): Decode the images after reading. Default: ``False``.
|
|
2367
2119
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
2368
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
2120
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
2369
2121
|
the maximum sample number of per shard.
|
|
2370
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
2122
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
2371
2123
|
argument can only be specified when `num_shards` is also specified.
|
|
2372
2124
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
2373
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
2374
|
-
Default: None, which means no cache is used.
|
|
2125
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
2126
|
+
Default: ``None`` , which means no cache is used.
|
|
2375
2127
|
decrypt (callable, optional): Image decryption function, which accepts the path of the encrypted image file
|
|
2376
|
-
and returns the decrypted bytes data. Default: None, no decryption.
|
|
2128
|
+
and returns the decrypted bytes data. Default: ``None`` , no decryption.
|
|
2377
2129
|
|
|
2378
2130
|
Raises:
|
|
2379
2131
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -2385,38 +2137,19 @@ class ImageFolderDataset(MappableDataset, VisionBaseDataset):
|
|
|
2385
2137
|
RuntimeError: If `class_indexing` is not a dictionary.
|
|
2386
2138
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
2387
2139
|
|
|
2140
|
+
Tutorial Examples:
|
|
2141
|
+
- `Load & Process Data With Dataset Pipeline
|
|
2142
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
2143
|
+
|
|
2388
2144
|
Note:
|
|
2389
|
-
- The shape of the image column is [image_size] if decode flag is False
|
|
2390
|
-
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
..
|
|
2394
|
-
:widths: 25 25 50
|
|
2395
|
-
:header-rows: 1
|
|
2396
|
-
|
|
2397
|
-
* - Parameter `sampler`
|
|
2398
|
-
- Parameter `shuffle`
|
|
2399
|
-
- Expected Order Behavior
|
|
2400
|
-
* - None
|
|
2401
|
-
- None
|
|
2402
|
-
- random order
|
|
2403
|
-
* - None
|
|
2404
|
-
- True
|
|
2405
|
-
- random order
|
|
2406
|
-
* - None
|
|
2407
|
-
- False
|
|
2408
|
-
- sequential order
|
|
2409
|
-
* - Sampler object
|
|
2410
|
-
- None
|
|
2411
|
-
- order defined by sampler
|
|
2412
|
-
* - Sampler object
|
|
2413
|
-
- True
|
|
2414
|
-
- not allowed
|
|
2415
|
-
* - Sampler object
|
|
2416
|
-
- False
|
|
2417
|
-
- not allowed
|
|
2145
|
+
- The shape of the image column is [image_size] if `decode` flag is ``False``, or [H,W,C] otherwise.
|
|
2146
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
2147
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
2148
|
+
|
|
2149
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
2418
2150
|
|
|
2419
2151
|
Examples:
|
|
2152
|
+
>>> import mindspore.dataset as ds
|
|
2420
2153
|
>>> image_folder_dataset_dir = "/path/to/image_folder_dataset_directory"
|
|
2421
2154
|
>>>
|
|
2422
2155
|
>>> # 1) Read all samples (image files) in image_folder_dataset_dir with 8 threads
|
|
@@ -2481,6 +2214,7 @@ class ImageFolderDataset(MappableDataset, VisionBaseDataset):
|
|
|
2481
2214
|
dict, a str-to-int mapping from label name to index.
|
|
2482
2215
|
|
|
2483
2216
|
Examples:
|
|
2217
|
+
>>> import mindspore.dataset as ds
|
|
2484
2218
|
>>> image_folder_dataset_dir = "/path/to/image_folder_dataset_directory"
|
|
2485
2219
|
>>>
|
|
2486
2220
|
>>> dataset = ds.ImageFolderDataset(dataset_dir=image_folder_dataset_dir)
|
|
@@ -2498,7 +2232,7 @@ class KITTIDataset(MappableDataset, VisionBaseDataset):
|
|
|
2498
2232
|
"""
|
|
2499
2233
|
KITTI dataset.
|
|
2500
2234
|
|
|
2501
|
-
When `usage` is "train"
|
|
2235
|
+
When `usage` is ``"train"``, the generated dataset has multiple columns: :py:obj:`[image, label, truncated,
|
|
2502
2236
|
occluded, alpha, bbox, dimensions, location, rotation_y]` ; When `usage` is "test", the generated dataset
|
|
2503
2237
|
has only one column: :py:obj:`[image]` .
|
|
2504
2238
|
The tensor of column :py:obj:`image` is of the uint8 type.
|
|
@@ -2513,26 +2247,27 @@ class KITTIDataset(MappableDataset, VisionBaseDataset):
|
|
|
2513
2247
|
|
|
2514
2248
|
Args:
|
|
2515
2249
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
2516
|
-
usage (str, optional): Usage of this dataset, can be
|
|
2517
|
-
train samples,
|
|
2250
|
+
usage (str, optional): Usage of this dataset, can be ``"train"`` or ``"test"`` .
|
|
2251
|
+
``"train"`` will read 7481 train samples, ``"test"`` will read from 7518 test samples
|
|
2252
|
+
without label. Default: ``None`` , will use ``"train"`` .
|
|
2518
2253
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
2519
|
-
Default: None, will include all images.
|
|
2254
|
+
Default: ``None`` , will include all images.
|
|
2520
2255
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
2521
|
-
Default: None, will use global default workers(8), it can be set
|
|
2522
|
-
by
|
|
2523
|
-
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None, expected
|
|
2256
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
2257
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
2258
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected
|
|
2524
2259
|
order behavior shown in the table below.
|
|
2525
|
-
decode (bool, optional): Decode the images after reading. Default: False
|
|
2260
|
+
decode (bool, optional): Decode the images after reading. Default: ``False``.
|
|
2526
2261
|
sampler (Sampler, optional): Object used to choose samples from the dataset.
|
|
2527
|
-
Default: None, expected order behavior shown in the table below.
|
|
2262
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
2528
2263
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
2529
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
2264
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
2530
2265
|
the max sample number of per shard.
|
|
2531
|
-
shard_id (int, optional): The shard ID within `num_shards`. Default: None. This
|
|
2266
|
+
shard_id (int, optional): The shard ID within `num_shards`. Default: ``None`` . This
|
|
2532
2267
|
argument can only be specified when `num_shards` is also specified.
|
|
2533
2268
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
2534
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
2535
|
-
Default: None, which means no cache is used.
|
|
2269
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
2270
|
+
Default: ``None`` , which means no cache is used.
|
|
2536
2271
|
|
|
2537
2272
|
Raises:
|
|
2538
2273
|
RuntimeError: If `sampler` and `shuffle` are specified at the same time.
|
|
@@ -2542,37 +2277,18 @@ class KITTIDataset(MappableDataset, VisionBaseDataset):
|
|
|
2542
2277
|
ValueError: If `dataset_dir` is not exist.
|
|
2543
2278
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
2544
2279
|
|
|
2280
|
+
Tutorial Examples:
|
|
2281
|
+
- `Load & Process Data With Dataset Pipeline
|
|
2282
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
2283
|
+
|
|
2545
2284
|
Note:
|
|
2546
|
-
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
..
|
|
2550
|
-
:widths: 25 25 50
|
|
2551
|
-
:header-rows: 1
|
|
2552
|
-
|
|
2553
|
-
* - Parameter `sampler`
|
|
2554
|
-
- Parameter `shuffle`
|
|
2555
|
-
- Expected Order Behavior
|
|
2556
|
-
* - None
|
|
2557
|
-
- None
|
|
2558
|
-
- random order
|
|
2559
|
-
* - None
|
|
2560
|
-
- True
|
|
2561
|
-
- random order
|
|
2562
|
-
* - None
|
|
2563
|
-
- False
|
|
2564
|
-
- sequential order
|
|
2565
|
-
* - Sampler object
|
|
2566
|
-
- None
|
|
2567
|
-
- order defined by sampler
|
|
2568
|
-
* - Sampler object
|
|
2569
|
-
- True
|
|
2570
|
-
- not allowed
|
|
2571
|
-
* - Sampler object
|
|
2572
|
-
- False
|
|
2573
|
-
- not allowed
|
|
2285
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
2286
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
2287
|
+
|
|
2288
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
2574
2289
|
|
|
2575
2290
|
Examples:
|
|
2291
|
+
>>> import mindspore.dataset as ds
|
|
2576
2292
|
>>> kitti_dataset_dir = "/path/to/kitti_dataset_directory"
|
|
2577
2293
|
>>>
|
|
2578
2294
|
>>> # 1) Read all KITTI train dataset samples in kitti_dataset_dir in sequence
|
|
@@ -2653,25 +2369,25 @@ class KMnistDataset(MappableDataset, VisionBaseDataset):
|
|
|
2653
2369
|
|
|
2654
2370
|
Args:
|
|
2655
2371
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
2656
|
-
usage (str, optional): Usage of this dataset, can be 'train', 'test' or 'all' .
|
|
2657
|
-
train
|
|
2658
|
-
Default: None, will read all samples.
|
|
2372
|
+
usage (str, optional): Usage of this dataset, can be ``'train'`` , ``'test'`` or ``'all'`` .
|
|
2373
|
+
``'train'`` will read from 60,000 train samples, ``'test'`` will read from 10,000 test samples,
|
|
2374
|
+
``'all'`` will read from all 70,000 samples. Default: ``None`` , will read all samples.
|
|
2659
2375
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
2660
|
-
Default: None, will read all images.
|
|
2376
|
+
Default: ``None`` , will read all images.
|
|
2661
2377
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
2662
|
-
Default: None, will use global default workers(8), it can be set
|
|
2663
|
-
by
|
|
2378
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
2379
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
2664
2380
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
2665
|
-
Default: None, expected order behavior shown in the table below.
|
|
2381
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
2666
2382
|
sampler (Sampler, optional): Object used to choose samples from the dataset.
|
|
2667
|
-
Default: None, expected order behavior shown in the table below.
|
|
2668
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
2383
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
2384
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
2669
2385
|
When this argument is specified, `num_samples` reflects the maximum sample number of per shard.
|
|
2670
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
2386
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
2671
2387
|
argument can only be specified when `num_shards` is also specified.
|
|
2672
2388
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
2673
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
2674
|
-
Default: None, which means no cache is used.
|
|
2389
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
2390
|
+
Default: ``None`` , which means no cache is used.
|
|
2675
2391
|
|
|
2676
2392
|
Raises:
|
|
2677
2393
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -2682,37 +2398,18 @@ class KMnistDataset(MappableDataset, VisionBaseDataset):
|
|
|
2682
2398
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
2683
2399
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
2684
2400
|
|
|
2401
|
+
Tutorial Examples:
|
|
2402
|
+
- `Load & Process Data With Dataset Pipeline
|
|
2403
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
2404
|
+
|
|
2685
2405
|
Note:
|
|
2686
|
-
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
..
|
|
2690
|
-
:widths: 25 25 50
|
|
2691
|
-
:header-rows: 1
|
|
2692
|
-
|
|
2693
|
-
* - Parameter `sampler`
|
|
2694
|
-
- Parameter `shuffle`
|
|
2695
|
-
- Expected Order Behavior
|
|
2696
|
-
* - None
|
|
2697
|
-
- None
|
|
2698
|
-
- random order
|
|
2699
|
-
* - None
|
|
2700
|
-
- True
|
|
2701
|
-
- random order
|
|
2702
|
-
* - None
|
|
2703
|
-
- False
|
|
2704
|
-
- sequential order
|
|
2705
|
-
* - Sampler object
|
|
2706
|
-
- None
|
|
2707
|
-
- order defined by sampler
|
|
2708
|
-
* - Sampler object
|
|
2709
|
-
- True
|
|
2710
|
-
- not allowed
|
|
2711
|
-
* - Sampler object
|
|
2712
|
-
- False
|
|
2713
|
-
- not allowed
|
|
2406
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
2407
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
2408
|
+
|
|
2409
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
2714
2410
|
|
|
2715
2411
|
Examples:
|
|
2412
|
+
>>> import mindspore.dataset as ds
|
|
2716
2413
|
>>> kmnist_dataset_dir = "/path/to/kmnist_dataset_directory"
|
|
2717
2414
|
>>>
|
|
2718
2415
|
>>> # Read 3 samples from KMNIST dataset
|
|
@@ -2777,30 +2474,30 @@ class LFWDataset(MappableDataset, VisionBaseDataset):
|
|
|
2777
2474
|
|
|
2778
2475
|
Args:
|
|
2779
2476
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
2780
|
-
task (str, optional): Set the task type of reading lfw data, support 'people' and 'pairs'
|
|
2781
|
-
Default: None, means 'people'
|
|
2782
|
-
usage (str, optional): The image split to use, support '10fold'
|
|
2783
|
-
Default: None, will read samples including train and test
|
|
2784
|
-
image_set (str, optional): Type of image funneling to use, support 'original'
|
|
2785
|
-
'deepfunneled'
|
|
2477
|
+
task (str, optional): Set the task type of reading lfw data, support ``'people'`` and ``'pairs'``.
|
|
2478
|
+
Default: ``None`` , means ``'people'``.
|
|
2479
|
+
usage (str, optional): The image split to use, support '``10fold'``, ``'train'``, ``'test'`` and ``'all'``.
|
|
2480
|
+
Default: ``None`` , will read samples including ``'train'`` and ``'test'``.
|
|
2481
|
+
image_set (str, optional): Type of image funneling to use, support ``'original'``, ``'funneled'`` or
|
|
2482
|
+
``'deepfunneled'``. Default: ``None`` , will use ``'funneled'``.
|
|
2786
2483
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
2787
|
-
Default: None, all images.
|
|
2484
|
+
Default: ``None`` , all images.
|
|
2788
2485
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
2789
|
-
Default: None, will use global default workers(8), it can be set
|
|
2790
|
-
by
|
|
2486
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
2487
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
2791
2488
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
2792
|
-
Default: None, expected order behavior shown in the table below.
|
|
2793
|
-
decode (bool, optional): Decode the images after reading. Default: False
|
|
2489
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
2490
|
+
decode (bool, optional): Decode the images after reading. Default: ``False``.
|
|
2794
2491
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
2795
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
2492
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
2796
2493
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
2797
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
2494
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
2798
2495
|
the max sample number of per shard.
|
|
2799
|
-
shard_id (int, optional): The shard ID within `num_shards`. Default: None. This
|
|
2496
|
+
shard_id (int, optional): The shard ID within `num_shards`. Default: ``None`` . This
|
|
2800
2497
|
argument can only be specified when `num_shards` is also specified.
|
|
2801
2498
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
2802
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
2803
|
-
Default: None, which means no cache is used.
|
|
2499
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
2500
|
+
Default: ``None`` , which means no cache is used.
|
|
2804
2501
|
|
|
2805
2502
|
Raises:
|
|
2806
2503
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -2810,37 +2507,18 @@ class LFWDataset(MappableDataset, VisionBaseDataset):
|
|
|
2810
2507
|
RuntimeError: If `shard_id` is specified but num_shards is None.
|
|
2811
2508
|
ValueError: If `shard_id` is invalid (< 0 or >= `num_shards` ).
|
|
2812
2509
|
|
|
2510
|
+
Tutorial Examples:
|
|
2511
|
+
- `Load & Process Data With Dataset Pipeline
|
|
2512
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
2513
|
+
|
|
2813
2514
|
Note:
|
|
2814
|
-
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
..
|
|
2818
|
-
:widths: 25 25 50
|
|
2819
|
-
:header-rows: 1
|
|
2820
|
-
|
|
2821
|
-
* - Parameter 'sampler'
|
|
2822
|
-
- Parameter 'shuffle'
|
|
2823
|
-
- Expected Order Behavior
|
|
2824
|
-
* - None
|
|
2825
|
-
- None
|
|
2826
|
-
- random order
|
|
2827
|
-
* - None
|
|
2828
|
-
- True
|
|
2829
|
-
- random order
|
|
2830
|
-
* - None
|
|
2831
|
-
- False
|
|
2832
|
-
- sequential order
|
|
2833
|
-
* - Sampler object
|
|
2834
|
-
- None
|
|
2835
|
-
- order defined by sampler
|
|
2836
|
-
* - Sampler object
|
|
2837
|
-
- True
|
|
2838
|
-
- not allowed
|
|
2839
|
-
* - Sampler object
|
|
2840
|
-
- False
|
|
2841
|
-
- not allowed
|
|
2515
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
2516
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
2517
|
+
|
|
2518
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
2842
2519
|
|
|
2843
2520
|
Examples:
|
|
2521
|
+
>>> import mindspore.dataset as ds
|
|
2844
2522
|
>>> # 1) Read LFW People dataset
|
|
2845
2523
|
>>> lfw_people_dataset_dir = "/path/to/lfw_people_dataset_directory"
|
|
2846
2524
|
>>> dataset = ds.LFWDataset(dataset_dir=lfw_people_dataset_dir, task="people", usage="10fold",
|
|
@@ -2937,28 +2615,28 @@ class LSUNDataset(MappableDataset, VisionBaseDataset):
|
|
|
2937
2615
|
|
|
2938
2616
|
Args:
|
|
2939
2617
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
2940
|
-
usage (str, optional): Usage of this dataset, can be
|
|
2941
|
-
Default: None, will be set to
|
|
2942
|
-
classes (Union[str, list[str]], optional): Choose the specific classes to load. Default: None,
|
|
2943
|
-
all classes in root directory.
|
|
2618
|
+
usage (str, optional): Usage of this dataset, can be ``"train"`` , ``"test"`` , ``"valid"`` or ``"all"``
|
|
2619
|
+
Default: ``None`` , will be set to ``"all"`` .
|
|
2620
|
+
classes (Union[str, list[str]], optional): Choose the specific classes to load. Default: ``None`` ,
|
|
2621
|
+
means loading all classes in root directory.
|
|
2944
2622
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
2945
|
-
Default: None, all images.
|
|
2623
|
+
Default: ``None`` , all images.
|
|
2946
2624
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
2947
|
-
Default: None, will use global default workers(8), it can be set
|
|
2948
|
-
by
|
|
2625
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
2626
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
2949
2627
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
2950
|
-
Default: None, expected order behavior shown in the table below.
|
|
2951
|
-
decode (bool, optional): Decode the images after reading. Default: False
|
|
2628
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
2629
|
+
decode (bool, optional): Decode the images after reading. Default: ``False``.
|
|
2952
2630
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
2953
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
2631
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
2954
2632
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
2955
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
2633
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
2956
2634
|
the max sample number of per shard.
|
|
2957
|
-
shard_id (int, optional): The shard ID within `num_shards`. Default: None. This
|
|
2635
|
+
shard_id (int, optional): The shard ID within `num_shards`. Default: ``None`` . This
|
|
2958
2636
|
argument can only be specified when `num_shards` is also specified.
|
|
2959
2637
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
2960
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
2961
|
-
Default: None, which means no cache is used.
|
|
2638
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
2639
|
+
Default: ``None`` , which means no cache is used.
|
|
2962
2640
|
|
|
2963
2641
|
Raises:
|
|
2964
2642
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -2969,37 +2647,18 @@ class LSUNDataset(MappableDataset, VisionBaseDataset):
|
|
|
2969
2647
|
ValueError: If `shard_id` is invalid (< 0 or >= `num_shards` ).
|
|
2970
2648
|
ValueError: If `usage` or `classes` is invalid (not in specific types).
|
|
2971
2649
|
|
|
2650
|
+
Tutorial Examples:
|
|
2651
|
+
- `Load & Process Data With Dataset Pipeline
|
|
2652
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
2653
|
+
|
|
2972
2654
|
Note:
|
|
2973
|
-
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
..
|
|
2977
|
-
:widths: 25 25 50
|
|
2978
|
-
:header-rows: 1
|
|
2979
|
-
|
|
2980
|
-
* - Parameter 'sampler'
|
|
2981
|
-
- Parameter 'shuffle'
|
|
2982
|
-
- Expected Order Behavior
|
|
2983
|
-
* - None
|
|
2984
|
-
- None
|
|
2985
|
-
- random order
|
|
2986
|
-
* - None
|
|
2987
|
-
- True
|
|
2988
|
-
- random order
|
|
2989
|
-
* - None
|
|
2990
|
-
- False
|
|
2991
|
-
- sequential order
|
|
2992
|
-
* - Sampler object
|
|
2993
|
-
- None
|
|
2994
|
-
- order defined by sampler
|
|
2995
|
-
* - Sampler object
|
|
2996
|
-
- True
|
|
2997
|
-
- not allowed
|
|
2998
|
-
* - Sampler object
|
|
2999
|
-
- False
|
|
3000
|
-
- not allowed
|
|
2655
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
2656
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
2657
|
+
|
|
2658
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
3001
2659
|
|
|
3002
2660
|
Examples:
|
|
2661
|
+
>>> import mindspore.dataset as ds
|
|
3003
2662
|
>>> lsun_dataset_dir = "/path/to/lsun_dataset_directory"
|
|
3004
2663
|
>>>
|
|
3005
2664
|
>>> # 1) Read all samples (image files) in lsun_dataset_dir with 8 threads
|
|
@@ -3076,28 +2735,29 @@ class ManifestDataset(MappableDataset, VisionBaseDataset):
|
|
|
3076
2735
|
|
|
3077
2736
|
Args:
|
|
3078
2737
|
dataset_file (str): File to be read.
|
|
3079
|
-
usage (str, optional): Acceptable usages include 'train'
|
|
2738
|
+
usage (str, optional): Acceptable usages include ``'train'``, ``'eval'`` and ``'inference'``.
|
|
2739
|
+
Default: ``'train'``.
|
|
3080
2740
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
3081
|
-
Default: None, will include all images.
|
|
2741
|
+
Default: ``None`` , will include all images.
|
|
3082
2742
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
3083
|
-
Default: None, will use global default workers(8), it can be set
|
|
3084
|
-
by
|
|
3085
|
-
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None, expected
|
|
2743
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
2744
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
2745
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected
|
|
3086
2746
|
order behavior shown in the table below.
|
|
3087
2747
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
3088
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
2748
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
3089
2749
|
class_indexing (dict, optional): A str-to-int mapping from label name to index.
|
|
3090
|
-
Default: None, the folder names will be sorted alphabetically and each
|
|
2750
|
+
Default: ``None`` , the folder names will be sorted alphabetically and each
|
|
3091
2751
|
class will be given a unique index starting from 0.
|
|
3092
|
-
decode (bool, optional): decode the images after reading. Default: False
|
|
2752
|
+
decode (bool, optional): decode the images after reading. Default: ``False``.
|
|
3093
2753
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
3094
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
2754
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
3095
2755
|
the max number of samples per shard.
|
|
3096
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
2756
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
3097
2757
|
argument can only be specified when `num_shards` is also specified.
|
|
3098
2758
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
3099
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
3100
|
-
Default: None, which means no cache is used.
|
|
2759
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
2760
|
+
Default: ``None`` , which means no cache is used.
|
|
3101
2761
|
|
|
3102
2762
|
Raises:
|
|
3103
2763
|
RuntimeError: If dataset_files are not valid or do not exist.
|
|
@@ -3109,38 +2769,20 @@ class ManifestDataset(MappableDataset, VisionBaseDataset):
|
|
|
3109
2769
|
RuntimeError: If class_indexing is not a dictionary.
|
|
3110
2770
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
3111
2771
|
|
|
2772
|
+
Tutorial Examples:
|
|
2773
|
+
- `Load & Process Data With Dataset Pipeline
|
|
2774
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
2775
|
+
|
|
3112
2776
|
Note:
|
|
3113
|
-
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
:header-rows: 1
|
|
3120
|
-
|
|
3121
|
-
* - Parameter `sampler`
|
|
3122
|
-
- Parameter `shuffle`
|
|
3123
|
-
- Expected Order Behavior
|
|
3124
|
-
* - None
|
|
3125
|
-
- None
|
|
3126
|
-
- random order
|
|
3127
|
-
* - None
|
|
3128
|
-
- True
|
|
3129
|
-
- random order
|
|
3130
|
-
* - None
|
|
3131
|
-
- False
|
|
3132
|
-
- sequential order
|
|
3133
|
-
* - Sampler object
|
|
3134
|
-
- None
|
|
3135
|
-
- order defined by sampler
|
|
3136
|
-
* - Sampler object
|
|
3137
|
-
- True
|
|
3138
|
-
- not allowed
|
|
3139
|
-
* - Sampler object
|
|
3140
|
-
- False
|
|
3141
|
-
- not allowed
|
|
2777
|
+
- If `decode` is ``False`` , the "image" column will get the 1D raw bytes of the image.
|
|
2778
|
+
Otherwise, a decoded image with shape :math:`[H,W,C]` will be returned.
|
|
2779
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
2780
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
2781
|
+
|
|
2782
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
3142
2783
|
|
|
3143
2784
|
Examples:
|
|
2785
|
+
>>> import mindspore.dataset as ds
|
|
3144
2786
|
>>> manifest_dataset_dir = "/path/to/manifest_dataset_file"
|
|
3145
2787
|
>>>
|
|
3146
2788
|
>>> # 1) Read all samples specified in manifest_dataset_dir dataset with 8 threads for training
|
|
@@ -3192,6 +2834,7 @@ class ManifestDataset(MappableDataset, VisionBaseDataset):
|
|
|
3192
2834
|
dict, a str-to-int mapping from label name to index.
|
|
3193
2835
|
|
|
3194
2836
|
Examples:
|
|
2837
|
+
>>> import mindspore.dataset as ds
|
|
3195
2838
|
>>> manifest_dataset_dir = "/path/to/manifest_dataset_file"
|
|
3196
2839
|
>>>
|
|
3197
2840
|
>>> dataset = ds.ManifestDataset(dataset_file=manifest_dataset_dir)
|
|
@@ -3217,67 +2860,48 @@ class MnistDataset(MappableDataset, VisionBaseDataset):
|
|
|
3217
2860
|
|
|
3218
2861
|
Args:
|
|
3219
2862
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
3220
|
-
usage (str, optional): Usage of this dataset, can be 'train', 'test' or 'all' .
|
|
3221
|
-
train
|
|
3222
|
-
Default: None, will read all samples.
|
|
2863
|
+
usage (str, optional): Usage of this dataset, can be ``'train'`` , ``'test'`` or ``'all'`` .
|
|
2864
|
+
``'train'`` will read from 60,000 train samples, ``'test'`` will read from 10,000 test samples,
|
|
2865
|
+
``'all'`` will read from all 70,000 samples. Default: ``None`` , will read all samples.
|
|
3223
2866
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
3224
|
-
Default: None, will read all images.
|
|
2867
|
+
Default: ``None`` , will read all images.
|
|
3225
2868
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
3226
|
-
Default: None, will use global default workers(8), it can be set
|
|
3227
|
-
by
|
|
2869
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
2870
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
3228
2871
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
3229
|
-
Default: None, expected order behavior shown in the table below.
|
|
2872
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
3230
2873
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
3231
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
3232
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
2874
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
2875
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
3233
2876
|
When this argument is specified, `num_samples` reflects the maximum sample number of per shard.
|
|
3234
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
2877
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
3235
2878
|
argument can only be specified when `num_shards` is also specified.
|
|
3236
2879
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
3237
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
3238
|
-
Default: None, which means no cache is used.
|
|
2880
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
2881
|
+
Default: ``None`` , which means no cache is used.
|
|
3239
2882
|
|
|
3240
2883
|
Raises:
|
|
3241
2884
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
3242
2885
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
3243
|
-
ValueError: If `usage` is not 'train'
|
|
2886
|
+
ValueError: If `usage` is not ``'train'``、``'test'`` or ``'all'``.
|
|
3244
2887
|
RuntimeError: If `sampler` and `shuffle` are specified at the same time.
|
|
3245
2888
|
RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time.
|
|
3246
2889
|
RuntimeError: If `num_shards` is specified but shard_id is None.
|
|
3247
2890
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
3248
2891
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
3249
2892
|
|
|
2893
|
+
Tutorial Examples:
|
|
2894
|
+
- `Load & Process Data With Dataset Pipeline
|
|
2895
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
2896
|
+
|
|
3250
2897
|
Note:
|
|
3251
|
-
-
|
|
3252
|
-
|
|
3253
|
-
|
|
3254
|
-
..
|
|
3255
|
-
:widths: 25 25 50
|
|
3256
|
-
:header-rows: 1
|
|
3257
|
-
|
|
3258
|
-
* - Parameter `sampler`
|
|
3259
|
-
- Parameter `shuffle`
|
|
3260
|
-
- Expected Order Behavior
|
|
3261
|
-
* - None
|
|
3262
|
-
- None
|
|
3263
|
-
- random order
|
|
3264
|
-
* - None
|
|
3265
|
-
- True
|
|
3266
|
-
- random order
|
|
3267
|
-
* - None
|
|
3268
|
-
- False
|
|
3269
|
-
- sequential order
|
|
3270
|
-
* - Sampler object
|
|
3271
|
-
- None
|
|
3272
|
-
- order defined by sampler
|
|
3273
|
-
* - Sampler object
|
|
3274
|
-
- True
|
|
3275
|
-
- not allowed
|
|
3276
|
-
* - Sampler object
|
|
3277
|
-
- False
|
|
3278
|
-
- not allowed
|
|
2898
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
2899
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
2900
|
+
|
|
2901
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
3279
2902
|
|
|
3280
2903
|
Examples:
|
|
2904
|
+
>>> import mindspore.dataset as ds
|
|
3281
2905
|
>>> mnist_dataset_dir = "/path/to/mnist_dataset_directory"
|
|
3282
2906
|
>>>
|
|
3283
2907
|
>>> # Read 3 samples from MNIST dataset
|
|
@@ -3341,25 +2965,25 @@ class OmniglotDataset(MappableDataset, VisionBaseDataset):
|
|
|
3341
2965
|
Args:
|
|
3342
2966
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
3343
2967
|
background (bool, optional): Whether to create dataset from the "background" set.
|
|
3344
|
-
Otherwise create from the "evaluation" set. Default: None, set to True
|
|
2968
|
+
Otherwise create from the "evaluation" set. Default: ``None`` , set to ``True``.
|
|
3345
2969
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
3346
|
-
Default: None, all images.
|
|
2970
|
+
Default: ``None`` , all images.
|
|
3347
2971
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
3348
|
-
Default: None, will use global default workers(8), it can be set
|
|
3349
|
-
by
|
|
2972
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
2973
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
3350
2974
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
3351
|
-
Default: None, expected order behavior shown in the table below.
|
|
3352
|
-
decode (bool, optional): Decode the images after reading. Default: False
|
|
2975
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
2976
|
+
decode (bool, optional): Decode the images after reading. Default: ``False``.
|
|
3353
2977
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
3354
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
2978
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
3355
2979
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
3356
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
2980
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
3357
2981
|
the max sample number of per shard.
|
|
3358
|
-
shard_id (int, optional): The shard ID within `num_shards`. Default: None. This
|
|
2982
|
+
shard_id (int, optional): The shard ID within `num_shards`. Default: ``None`` . This
|
|
3359
2983
|
argument can only be specified when `num_shards` is also specified.
|
|
3360
2984
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
3361
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
3362
|
-
Default: None, which means no cache is used.
|
|
2985
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
2986
|
+
Default: ``None`` , which means no cache is used.
|
|
3363
2987
|
|
|
3364
2988
|
Raises:
|
|
3365
2989
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -3369,37 +2993,18 @@ class OmniglotDataset(MappableDataset, VisionBaseDataset):
|
|
|
3369
2993
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
3370
2994
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
3371
2995
|
|
|
2996
|
+
Tutorial Examples:
|
|
2997
|
+
- `Load & Process Data With Dataset Pipeline
|
|
2998
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
2999
|
+
|
|
3372
3000
|
Note:
|
|
3373
|
-
-
|
|
3374
|
-
|
|
3375
|
-
|
|
3376
|
-
..
|
|
3377
|
-
:widths: 25 25 50
|
|
3378
|
-
:header-rows: 1
|
|
3379
|
-
|
|
3380
|
-
* - Parameter `sampler`
|
|
3381
|
-
- Parameter `shuffle`
|
|
3382
|
-
- Expected Order Behavior
|
|
3383
|
-
* - None
|
|
3384
|
-
- None
|
|
3385
|
-
- random order
|
|
3386
|
-
* - None
|
|
3387
|
-
- True
|
|
3388
|
-
- random order
|
|
3389
|
-
* - None
|
|
3390
|
-
- False
|
|
3391
|
-
- sequential order
|
|
3392
|
-
* - Sampler object
|
|
3393
|
-
- None
|
|
3394
|
-
- order defined by sampler
|
|
3395
|
-
* - Sampler object
|
|
3396
|
-
- True
|
|
3397
|
-
- not allowed
|
|
3398
|
-
* - Sampler object
|
|
3399
|
-
- False
|
|
3400
|
-
- not allowed
|
|
3001
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
3002
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
3003
|
+
|
|
3004
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
3401
3005
|
|
|
3402
3006
|
Examples:
|
|
3007
|
+
>>> import mindspore.dataset as ds
|
|
3403
3008
|
>>> omniglot_dataset_dir = "/path/to/omniglot_dataset_directory"
|
|
3404
3009
|
>>> dataset = ds.OmniglotDataset(dataset_dir=omniglot_dataset_dir,
|
|
3405
3010
|
... num_parallel_workers=8)
|
|
@@ -3468,35 +3073,37 @@ class PhotoTourDataset(MappableDataset, VisionBaseDataset):
|
|
|
3468
3073
|
PhotoTour dataset.
|
|
3469
3074
|
|
|
3470
3075
|
According to the given `usage` configuration, the generated dataset has different output columns:
|
|
3076
|
+
|
|
3471
3077
|
- `usage` = 'train', output columns: `[image, dtype=uint8]` .
|
|
3472
3078
|
- `usage` ≠ 'train', output columns: `[image1, dtype=uint8]` , `[image2, dtype=uint8]` , `[matches, dtype=uint32]` .
|
|
3473
3079
|
|
|
3474
3080
|
Args:
|
|
3475
3081
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
3476
3082
|
name (str): Name of the dataset to load,
|
|
3477
|
-
should be one of 'notredame'
|
|
3478
|
-
'yosemite_harris' or 'liberty_harris'
|
|
3479
|
-
usage (str, optional): Usage of the dataset, can be 'train' or 'test'
|
|
3083
|
+
should be one of ``'notredame'``, ``'yosemite'``, ``'liberty'``, ``'notredame_harris'``,
|
|
3084
|
+
``'yosemite_harris'`` or ``'liberty_harris'``.
|
|
3085
|
+
usage (str, optional): Usage of the dataset, can be ``'train'`` or ``'test'``. Default: ``None`` ,
|
|
3086
|
+
will be set to 'train'.
|
|
3480
3087
|
When usage is 'train', number of samples for each `name` is
|
|
3481
3088
|
{'notredame': 468159, 'yosemite': 633587, 'liberty': 450092, 'liberty_harris': 379587,
|
|
3482
3089
|
'yosemite_harris': 450912, 'notredame_harris': 325295}.
|
|
3483
3090
|
When usage is 'test', will read 100,000 samples for testing.
|
|
3484
3091
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
3485
|
-
Default: None, will read all images.
|
|
3092
|
+
Default: ``None`` , will read all images.
|
|
3486
3093
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
3487
|
-
Default: None, will use global default workers(8), it can be set
|
|
3488
|
-
by
|
|
3094
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
3095
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
3489
3096
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
3490
|
-
Default: None, expected order behavior shown in the table below.
|
|
3097
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
3491
3098
|
sampler (Sampler, optional): Object used to choose samples from the dataset.
|
|
3492
|
-
Default: None, expected order behavior shown in the table below.
|
|
3493
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
3099
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
3100
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
3494
3101
|
When this argument is specified, `num_samples` reflects the max sample number of per shard.
|
|
3495
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
3102
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
3496
3103
|
argument can only be specified when `num_shards` is also specified.
|
|
3497
3104
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
3498
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
3499
|
-
Default: None, which means no cache is used.
|
|
3105
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
3106
|
+
Default: ``None`` , which means no cache is used.
|
|
3500
3107
|
|
|
3501
3108
|
Raises:
|
|
3502
3109
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -3505,43 +3112,24 @@ class PhotoTourDataset(MappableDataset, VisionBaseDataset):
|
|
|
3505
3112
|
RuntimeError: If `num_shards` is specified but `shard_id` is None.
|
|
3506
3113
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
3507
3114
|
ValueError: If `dataset_dir` is not exist.
|
|
3508
|
-
ValueError: If `usage` is not
|
|
3509
|
-
ValueError: If name is not
|
|
3510
|
-
|
|
3115
|
+
ValueError: If `usage` is not ``'train'`` or ``'test'``.
|
|
3116
|
+
ValueError: If name is not ``'notredame'``, ``'yosemite'``, ``'liberty'``,
|
|
3117
|
+
``'notredame_harris'``, ``'yosemite_harris'`` or ``'liberty_harris'``.
|
|
3511
3118
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
3512
3119
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
3513
3120
|
|
|
3121
|
+
Tutorial Examples:
|
|
3122
|
+
- `Load & Process Data With Dataset Pipeline
|
|
3123
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
3124
|
+
|
|
3514
3125
|
Note:
|
|
3515
|
-
-
|
|
3516
|
-
|
|
3517
|
-
|
|
3518
|
-
..
|
|
3519
|
-
:widths: 64 64 1
|
|
3520
|
-
:header-rows: 1
|
|
3521
|
-
|
|
3522
|
-
* - Parameter `sampler`
|
|
3523
|
-
- Parameter `shuffle`
|
|
3524
|
-
- Expected Order Behavior
|
|
3525
|
-
* - None
|
|
3526
|
-
- None
|
|
3527
|
-
- random order
|
|
3528
|
-
* - None
|
|
3529
|
-
- True
|
|
3530
|
-
- random order
|
|
3531
|
-
* - None
|
|
3532
|
-
- False
|
|
3533
|
-
- sequential order
|
|
3534
|
-
* - Sampler object
|
|
3535
|
-
- None
|
|
3536
|
-
- order defined by sampler
|
|
3537
|
-
* - Sampler object
|
|
3538
|
-
- True
|
|
3539
|
-
- not allowed
|
|
3540
|
-
* - Sampler object
|
|
3541
|
-
- False
|
|
3542
|
-
- not allowed
|
|
3126
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
3127
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
3128
|
+
|
|
3129
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
3543
3130
|
|
|
3544
3131
|
Examples:
|
|
3132
|
+
>>> import mindspore.dataset as ds
|
|
3545
3133
|
>>> # Read 3 samples from PhotoTour dataset.
|
|
3546
3134
|
>>> dataset = ds.PhotoTourDataset(dataset_dir="/path/to/photo_tour_dataset_directory",
|
|
3547
3135
|
... name='liberty', usage='train', num_samples=3)
|
|
@@ -3624,26 +3212,26 @@ class Places365Dataset(MappableDataset, VisionBaseDataset):
|
|
|
3624
3212
|
|
|
3625
3213
|
Args:
|
|
3626
3214
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
3627
|
-
usage (str, optional): Usage of this dataset, can be 'train-standard'
|
|
3628
|
-
Default: None, will be set to 'train-standard'
|
|
3629
|
-
small (bool, optional): Use 256 * 256 images (True) or high resolution images (False). Default:
|
|
3630
|
-
decode (bool, optional): Decode the images after reading. Default: False
|
|
3215
|
+
usage (str, optional): Usage of this dataset, can be ``'train-standard'``, ``'train-challenge'``
|
|
3216
|
+
or ``'val'``. Default: ``None`` , will be set to ``'train-standard'``.
|
|
3217
|
+
small (bool, optional): Use 256 * 256 images (True) or high resolution images (False). Default: ``True``.
|
|
3218
|
+
decode (bool, optional): Decode the images after reading. Default: ``False``.
|
|
3631
3219
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
3632
|
-
Default: None, will read all images.
|
|
3220
|
+
Default: ``None`` , will read all images.
|
|
3633
3221
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
3634
|
-
Default: None, will use global default workers(8), it can be set
|
|
3635
|
-
by
|
|
3222
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
3223
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
3636
3224
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
3637
|
-
Default: None, expected order behavior shown in the table below.
|
|
3225
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
3638
3226
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
3639
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
3640
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
3227
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
3228
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
3641
3229
|
When this argument is specified, `num_samples` reflects the max sample number of per shard.
|
|
3642
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
3230
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
3643
3231
|
argument can only be specified when `num_shards` is also specified.
|
|
3644
3232
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
3645
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
3646
|
-
Default: None, which means no cache is used.
|
|
3233
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
3234
|
+
Default: ``None`` , which means no cache is used.
|
|
3647
3235
|
|
|
3648
3236
|
Raises:
|
|
3649
3237
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -3653,39 +3241,20 @@ class Places365Dataset(MappableDataset, VisionBaseDataset):
|
|
|
3653
3241
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
3654
3242
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
3655
3243
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
3656
|
-
ValueError: If `usage` is not
|
|
3244
|
+
ValueError: If `usage` is not ``"train-standard"``, ``"train-challenge"`` or ``"val"``.
|
|
3245
|
+
|
|
3246
|
+
Tutorial Examples:
|
|
3247
|
+
- `Load & Process Data With Dataset Pipeline
|
|
3248
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
3657
3249
|
|
|
3658
3250
|
Note:
|
|
3659
|
-
-
|
|
3660
|
-
|
|
3661
|
-
|
|
3662
|
-
..
|
|
3663
|
-
:widths: 25 25 50
|
|
3664
|
-
:header-rows: 1
|
|
3665
|
-
|
|
3666
|
-
* - Parameter `sampler`
|
|
3667
|
-
- Parameter `shuffle`
|
|
3668
|
-
- Expected Order Behavior
|
|
3669
|
-
* - None
|
|
3670
|
-
- None
|
|
3671
|
-
- random order
|
|
3672
|
-
* - None
|
|
3673
|
-
- True
|
|
3674
|
-
- random order
|
|
3675
|
-
* - None
|
|
3676
|
-
- False
|
|
3677
|
-
- sequential order
|
|
3678
|
-
* - Sampler object
|
|
3679
|
-
- None
|
|
3680
|
-
- order defined by sampler
|
|
3681
|
-
* - Sampler object
|
|
3682
|
-
- True
|
|
3683
|
-
- not allowed
|
|
3684
|
-
* - Sampler object
|
|
3685
|
-
- False
|
|
3686
|
-
- not allowed
|
|
3251
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
3252
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
3253
|
+
|
|
3254
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
3687
3255
|
|
|
3688
3256
|
Examples:
|
|
3257
|
+
>>> import mindspore.dataset as ds
|
|
3689
3258
|
>>> place365_dataset_dir = "/path/to/place365_dataset_directory"
|
|
3690
3259
|
>>>
|
|
3691
3260
|
>>> # Read 3 samples from Places365 dataset
|
|
@@ -3765,26 +3334,26 @@ class QMnistDataset(MappableDataset, VisionBaseDataset):
|
|
|
3765
3334
|
|
|
3766
3335
|
Args:
|
|
3767
3336
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
3768
|
-
usage (str, optional): Usage of this dataset, can be 'train'
|
|
3769
|
-
or 'all'
|
|
3770
|
-
compat (bool, optional): Whether the label for each example is class number (compat=True)
|
|
3771
|
-
information (compat=False). Default: True
|
|
3337
|
+
usage (str, optional): Usage of this dataset, can be ``'train'``, ``'test'``, ``'test10k'``,
|
|
3338
|
+
``'test50k'``, ``'nist'`` or ``'all'``. Default: ``None`` , will read all samples.
|
|
3339
|
+
compat (bool, optional): Whether the label for each example is class number (compat= ``True`` )
|
|
3340
|
+
or the full QMNIST information (compat= ``False`` ). Default: ``True``.
|
|
3772
3341
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
3773
|
-
Default: None, will read all images.
|
|
3342
|
+
Default: ``None`` , will read all images.
|
|
3774
3343
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
3775
|
-
Default: None, will use global default workers(8), it can be set
|
|
3776
|
-
by
|
|
3344
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
3345
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
3777
3346
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
3778
|
-
Default: None, expected order behavior shown in the table below.
|
|
3347
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
3779
3348
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
3780
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
3781
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
3349
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
3350
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
3782
3351
|
When this argument is specified, `num_samples` reflects the maximum sample number of per shard.
|
|
3783
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
3352
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
3784
3353
|
argument can only be specified when `num_shards` is also specified.
|
|
3785
3354
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
3786
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
3787
|
-
Default: None, which means no cache is used.
|
|
3355
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
3356
|
+
Default: ``None`` , which means no cache is used.
|
|
3788
3357
|
|
|
3789
3358
|
Raises:
|
|
3790
3359
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -3795,37 +3364,18 @@ class QMnistDataset(MappableDataset, VisionBaseDataset):
|
|
|
3795
3364
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
3796
3365
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
3797
3366
|
|
|
3367
|
+
Tutorial Examples:
|
|
3368
|
+
- `Load & Process Data With Dataset Pipeline
|
|
3369
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
3370
|
+
|
|
3798
3371
|
Note:
|
|
3799
|
-
-
|
|
3800
|
-
|
|
3801
|
-
|
|
3802
|
-
..
|
|
3803
|
-
:widths: 25 25 50
|
|
3804
|
-
:header-rows: 1
|
|
3805
|
-
|
|
3806
|
-
* - Parameter `sampler`
|
|
3807
|
-
- Parameter `shuffle`
|
|
3808
|
-
- Expected Order Behavior
|
|
3809
|
-
* - None
|
|
3810
|
-
- None
|
|
3811
|
-
- random order
|
|
3812
|
-
* - None
|
|
3813
|
-
- True
|
|
3814
|
-
- random order
|
|
3815
|
-
* - None
|
|
3816
|
-
- False
|
|
3817
|
-
- sequential order
|
|
3818
|
-
* - Sampler object
|
|
3819
|
-
- None
|
|
3820
|
-
- order defined by sampler
|
|
3821
|
-
* - Sampler object
|
|
3822
|
-
- True
|
|
3823
|
-
- not allowed
|
|
3824
|
-
* - Sampler object
|
|
3825
|
-
- False
|
|
3826
|
-
- not allowed
|
|
3372
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
3373
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
3374
|
+
|
|
3375
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
3827
3376
|
|
|
3828
3377
|
Examples:
|
|
3378
|
+
>>> import mindspore.dataset as ds
|
|
3829
3379
|
>>> qmnist_dataset_dir = "/path/to/qmnist_dataset_directory"
|
|
3830
3380
|
>>>
|
|
3831
3381
|
>>> # Read 3 samples from QMNIST train dataset
|
|
@@ -3888,26 +3438,26 @@ class RandomDataset(SourceDataset, VisionBaseDataset):
|
|
|
3888
3438
|
|
|
3889
3439
|
Args:
|
|
3890
3440
|
total_rows (int, optional): Number of samples for the dataset to generate.
|
|
3891
|
-
Default: None, number of samples is random.
|
|
3441
|
+
Default: ``None`` , number of samples is random.
|
|
3892
3442
|
schema (Union[str, Schema], optional): Data format policy, which specifies the data types and shapes of the data
|
|
3893
|
-
column to be read. Both JSON file path and objects constructed by mindspore.dataset.Schema are
|
|
3894
|
-
Default: None.
|
|
3443
|
+
column to be read. Both JSON file path and objects constructed by :class:`mindspore.dataset.Schema` are
|
|
3444
|
+
acceptable. Default: ``None`` .
|
|
3895
3445
|
columns_list (list[str], optional): List of column names of the dataset.
|
|
3896
|
-
Default: None, the columns will be named like this "c0", "c1", "c2" etc.
|
|
3446
|
+
Default: ``None`` , the columns will be named like this "c0", "c1", "c2" etc.
|
|
3897
3447
|
num_samples (int, optional): The number of samples to be included in the dataset.
|
|
3898
|
-
Default: None, all samples.
|
|
3448
|
+
Default: ``None`` , all samples.
|
|
3899
3449
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
3900
|
-
Default: None, will use global default workers(8), it can be set
|
|
3901
|
-
by
|
|
3450
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
3451
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
3902
3452
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
3903
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
3904
|
-
Default: None, which means no cache is used.
|
|
3453
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
3454
|
+
Default: ``None`` , which means no cache is used.
|
|
3905
3455
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
3906
|
-
Default: None, expected order behavior shown in the table below.
|
|
3456
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
3907
3457
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
3908
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
3458
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
3909
3459
|
the maximum sample number of per shard.
|
|
3910
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
3460
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
3911
3461
|
argument can only be specified when `num_shards` is also specified.
|
|
3912
3462
|
|
|
3913
3463
|
Raises:
|
|
@@ -3921,6 +3471,10 @@ class RandomDataset(SourceDataset, VisionBaseDataset):
|
|
|
3921
3471
|
TypeError: If `shuffle` is not of type bool.
|
|
3922
3472
|
TypeError: If `columns_list` is not of type list.
|
|
3923
3473
|
|
|
3474
|
+
Tutorial Examples:
|
|
3475
|
+
- `Load & Process Data With Dataset Pipeline
|
|
3476
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
3477
|
+
|
|
3924
3478
|
Examples:
|
|
3925
3479
|
>>> from mindspore import dtype as mstype
|
|
3926
3480
|
>>> import mindspore.dataset as ds
|
|
@@ -3963,68 +3517,49 @@ class RenderedSST2Dataset(MappableDataset, VisionBaseDataset):
|
|
|
3963
3517
|
|
|
3964
3518
|
Args:
|
|
3965
3519
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
3966
|
-
usage (str, optional): Usage of this dataset, can be 'train'
|
|
3967
|
-
Default: None, will read all samples.
|
|
3520
|
+
usage (str, optional): Usage of this dataset, can be ``'train'``, ``'val'``, ``'test'``
|
|
3521
|
+
or ``'all'``. Default: ``None`` , will read all samples.
|
|
3968
3522
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
3969
|
-
Default: None, will include all images.
|
|
3523
|
+
Default: ``None`` , will include all images.
|
|
3970
3524
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
3971
|
-
Default: None, will use global default workers(8), it can be set
|
|
3972
|
-
by
|
|
3525
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
3526
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
3973
3527
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
3974
|
-
Default: None, expected order behavior shown in the table below.
|
|
3975
|
-
decode (bool, optional): Whether or not to decode the images after reading. Default: False
|
|
3528
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
3529
|
+
decode (bool, optional): Whether or not to decode the images after reading. Default: ``False``.
|
|
3976
3530
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
3977
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
3531
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
3978
3532
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
3979
3533
|
into. When this argument is specified, `num_samples` reflects
|
|
3980
|
-
the maximum sample number of per shard. Default: None.
|
|
3534
|
+
the maximum sample number of per shard. Default: ``None`` .
|
|
3981
3535
|
shard_id (int, optional): The shard ID within `num_shards` . This
|
|
3982
|
-
argument can only be specified when `num_shards` is also specified. Default: None.
|
|
3536
|
+
argument can only be specified when `num_shards` is also specified. Default: ``None`` .
|
|
3983
3537
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
3984
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
3985
|
-
Default: None, which means no cache is used.
|
|
3538
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
3539
|
+
Default: ``None`` , which means no cache is used.
|
|
3986
3540
|
|
|
3987
3541
|
Raises:
|
|
3988
3542
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
3989
|
-
ValueError: If `usage` is not 'train', 'test', 'val' or 'all'.
|
|
3990
|
-
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
3991
3543
|
RuntimeError: If `sampler` and `shuffle` are specified at the same time.
|
|
3992
3544
|
RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time.
|
|
3993
3545
|
RuntimeError: If `num_shards` is specified but `shard_id` is None.
|
|
3994
3546
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
3547
|
+
ValueError: If `usage` is not ``'train'``, ``'test'``, ``'val'`` or ``'all'``.
|
|
3548
|
+
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
3995
3549
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
3996
3550
|
|
|
3551
|
+
Tutorial Examples:
|
|
3552
|
+
- `Load & Process Data With Dataset Pipeline
|
|
3553
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
3554
|
+
|
|
3997
3555
|
Note:
|
|
3998
|
-
-
|
|
3999
|
-
|
|
4000
|
-
|
|
4001
|
-
..
|
|
4002
|
-
:widths: 25 25 50
|
|
4003
|
-
:header-rows: 1
|
|
4004
|
-
|
|
4005
|
-
* - Parameter `sampler`
|
|
4006
|
-
- Parameter `shuffle`
|
|
4007
|
-
- Expected Order Behavior
|
|
4008
|
-
* - None
|
|
4009
|
-
- None
|
|
4010
|
-
- random order
|
|
4011
|
-
* - None
|
|
4012
|
-
- True
|
|
4013
|
-
- random order
|
|
4014
|
-
* - None
|
|
4015
|
-
- False
|
|
4016
|
-
- sequential order
|
|
4017
|
-
* - Sampler object
|
|
4018
|
-
- None
|
|
4019
|
-
- order defined by sampler
|
|
4020
|
-
* - Sampler object
|
|
4021
|
-
- True
|
|
4022
|
-
- not allowed
|
|
4023
|
-
* - Sampler object
|
|
4024
|
-
- False
|
|
4025
|
-
- not allowed
|
|
3556
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
3557
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
3558
|
+
|
|
3559
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
4026
3560
|
|
|
4027
3561
|
Examples:
|
|
3562
|
+
>>> import mindspore.dataset as ds
|
|
4028
3563
|
>>> rendered_sst2_dataset_dir = "/path/to/rendered_sst2_dataset_directory"
|
|
4029
3564
|
>>>
|
|
4030
3565
|
>>> # 1) Read all samples (image files) in rendered_sst2_dataset_dir with 8 threads
|
|
@@ -4170,29 +3705,31 @@ class SBDataset(GeneratorDataset):
|
|
|
4170
3705
|
"""
|
|
4171
3706
|
SB(Semantic Boundaries) Dataset.
|
|
4172
3707
|
|
|
4173
|
-
By configuring the
|
|
3708
|
+
By configuring the `task` parameter, the generated dataset has different output columns.
|
|
4174
3709
|
|
|
4175
|
-
-
|
|
3710
|
+
- `task` is ``'Boundaries'`` , there are two output columns: the 'image' column has the data type uint8 and
|
|
4176
3711
|
the 'label' column contains one image of the data type uint8.
|
|
4177
|
-
-
|
|
3712
|
+
- `task` is ``'Segmentation'`` , there are two output columns: the 'image' column has the data type uint8 and
|
|
4178
3713
|
the 'label' column contains 20 images of the data type uint8.
|
|
4179
3714
|
|
|
4180
3715
|
Args:
|
|
4181
3716
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
4182
|
-
task (str, optional): Acceptable tasks include 'Boundaries' or 'Segmentation'
|
|
4183
|
-
|
|
3717
|
+
task (str, optional): Acceptable tasks include ``'Boundaries'`` or ``'Segmentation'``.
|
|
3718
|
+
Default: ``'Boundaries'``.
|
|
3719
|
+
usage (str, optional): Acceptable usages include ``'train'``, ``'val'``, ``'train_noval'``
|
|
3720
|
+
and ``'all'``. Default: ``'all'``.
|
|
4184
3721
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
4185
|
-
Default: None, all images.
|
|
4186
|
-
num_parallel_workers (int, optional): Number of worker subprocesses to read the data. Default: 1
|
|
4187
|
-
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None, expected
|
|
3722
|
+
Default: ``None`` , all images.
|
|
3723
|
+
num_parallel_workers (int, optional): Number of worker subprocesses to read the data. Default: ``1``.
|
|
3724
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected
|
|
4188
3725
|
order behavior shown in the table below.
|
|
4189
|
-
decode (bool, optional): Decode the images after reading. Default: None
|
|
3726
|
+
decode (bool, optional): Decode the images after reading. Default: ``None`` , means ``False``.
|
|
4190
3727
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
4191
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
3728
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
4192
3729
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
4193
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
3730
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
4194
3731
|
the max sample number of per shard.
|
|
4195
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
3732
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
4196
3733
|
argument can only be specified when `num_shards` is also specified.
|
|
4197
3734
|
|
|
4198
3735
|
Raises:
|
|
@@ -4203,41 +3740,22 @@ class SBDataset(GeneratorDataset):
|
|
|
4203
3740
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
4204
3741
|
ValueError: If `dataset_dir` is not exist.
|
|
4205
3742
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
4206
|
-
ValueError: If `task` is not
|
|
4207
|
-
ValueError: If `usage` is not
|
|
3743
|
+
ValueError: If `task` is not ``'Boundaries'`` or ``'Segmentation'``.
|
|
3744
|
+
ValueError: If `usage` is not ``'train'``, ``'val'``, ``'train_noval'`` or ``'all'``.
|
|
4208
3745
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
4209
3746
|
|
|
3747
|
+
Tutorial Examples:
|
|
3748
|
+
- `Load & Process Data With Dataset Pipeline
|
|
3749
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
3750
|
+
|
|
4210
3751
|
Note:
|
|
4211
|
-
-
|
|
4212
|
-
|
|
4213
|
-
|
|
4214
|
-
..
|
|
4215
|
-
:widths: 25 25 50
|
|
4216
|
-
:header-rows: 1
|
|
4217
|
-
|
|
4218
|
-
* - Parameter `sampler`
|
|
4219
|
-
- Parameter `shuffle`
|
|
4220
|
-
- Expected Order Behavior
|
|
4221
|
-
* - None
|
|
4222
|
-
- None
|
|
4223
|
-
- random order
|
|
4224
|
-
* - None
|
|
4225
|
-
- True
|
|
4226
|
-
- random order
|
|
4227
|
-
* - None
|
|
4228
|
-
- False
|
|
4229
|
-
- sequential order
|
|
4230
|
-
* - Sampler object
|
|
4231
|
-
- None
|
|
4232
|
-
- order defined by sampler
|
|
4233
|
-
* - Sampler object
|
|
4234
|
-
- True
|
|
4235
|
-
- not allowed
|
|
4236
|
-
* - Sampler object
|
|
4237
|
-
- False
|
|
4238
|
-
- not allowed
|
|
3752
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
3753
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
3754
|
+
|
|
3755
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
4239
3756
|
|
|
4240
3757
|
Examples:
|
|
3758
|
+
>>> import mindspore.dataset as ds
|
|
4241
3759
|
>>> sb_dataset_dir = "/path/to/sb_dataset_directory"
|
|
4242
3760
|
>>>
|
|
4243
3761
|
>>> # 1) Get all samples from Semantic Boundaries Dataset in sequence
|
|
@@ -4311,22 +3829,22 @@ class SBUDataset(MappableDataset, VisionBaseDataset):
|
|
|
4311
3829
|
Args:
|
|
4312
3830
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
4313
3831
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
4314
|
-
Default: None, will read all images.
|
|
3832
|
+
Default: ``None`` , will read all images.
|
|
4315
3833
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
4316
|
-
Default: None, will use global default workers(8), it can be set
|
|
4317
|
-
by
|
|
3834
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
3835
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
4318
3836
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
4319
|
-
Default: None, expected order behavior shown in the table below.
|
|
4320
|
-
decode (bool, optional): Decode the images after reading. Default: False
|
|
3837
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
3838
|
+
decode (bool, optional): Decode the images after reading. Default: ``False``.
|
|
4321
3839
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
4322
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
4323
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
3840
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
3841
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
4324
3842
|
When this argument is specified, `num_samples` reflects the max sample number of per shard.
|
|
4325
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
3843
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
4326
3844
|
argument can only be specified when `num_shards` is also specified.
|
|
4327
3845
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
4328
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
4329
|
-
Default: None, which means no cache is used.
|
|
3846
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
3847
|
+
Default: ``None`` , which means no cache is used.
|
|
4330
3848
|
|
|
4331
3849
|
Raises:
|
|
4332
3850
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -4337,37 +3855,18 @@ class SBUDataset(MappableDataset, VisionBaseDataset):
|
|
|
4337
3855
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
4338
3856
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
4339
3857
|
|
|
3858
|
+
Tutorial Examples:
|
|
3859
|
+
- `Load & Process Data With Dataset Pipeline
|
|
3860
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
3861
|
+
|
|
4340
3862
|
Note:
|
|
4341
|
-
-
|
|
4342
|
-
|
|
4343
|
-
|
|
4344
|
-
..
|
|
4345
|
-
:widths: 25 25 50
|
|
4346
|
-
:header-rows: 1
|
|
4347
|
-
|
|
4348
|
-
* - Parameter 'sampler'
|
|
4349
|
-
- Parameter 'shuffle'
|
|
4350
|
-
- Expected Order Behavior
|
|
4351
|
-
* - None
|
|
4352
|
-
- None
|
|
4353
|
-
- random order
|
|
4354
|
-
* - None
|
|
4355
|
-
- True
|
|
4356
|
-
- random order
|
|
4357
|
-
* - None
|
|
4358
|
-
- False
|
|
4359
|
-
- sequential order
|
|
4360
|
-
* - Sampler object
|
|
4361
|
-
- None
|
|
4362
|
-
- order defined by sampler
|
|
4363
|
-
* - Sampler object
|
|
4364
|
-
- True
|
|
4365
|
-
- not allowed
|
|
4366
|
-
* - Sampler object
|
|
4367
|
-
- False
|
|
4368
|
-
- not allowed
|
|
3863
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
3864
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
3865
|
+
|
|
3866
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
4369
3867
|
|
|
4370
3868
|
Examples:
|
|
3869
|
+
>>> import mindspore.dataset as ds
|
|
4371
3870
|
>>> sbu_dataset_dir = "/path/to/sbu_dataset_directory"
|
|
4372
3871
|
>>> # Read 3 samples from SBU dataset
|
|
4373
3872
|
>>> dataset = ds.SBUDataset(dataset_dir=sbu_dataset_dir, num_samples=3)
|
|
@@ -4427,22 +3926,22 @@ class SemeionDataset(MappableDataset, VisionBaseDataset):
|
|
|
4427
3926
|
Args:
|
|
4428
3927
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
4429
3928
|
num_samples (int, optional): The number of samples to be included in the dataset.
|
|
4430
|
-
Default: None, will read all images.
|
|
3929
|
+
Default: ``None`` , will read all images.
|
|
4431
3930
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
4432
|
-
Default: None, will use global default workers(8), it can be set
|
|
4433
|
-
by
|
|
4434
|
-
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None, expected
|
|
3931
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
3932
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
3933
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected
|
|
4435
3934
|
order behavior shown in the table below.
|
|
4436
3935
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
4437
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
3936
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
4438
3937
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
4439
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
3938
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
4440
3939
|
the maximum sample number of per shard.
|
|
4441
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
3940
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
4442
3941
|
argument can only be specified when `num_shards` is also specified.
|
|
4443
3942
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
4444
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
4445
|
-
Default: None, which means no cache is used.
|
|
3943
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
3944
|
+
Default: ``None`` , which means no cache is used.
|
|
4446
3945
|
|
|
4447
3946
|
Raises:
|
|
4448
3947
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -4453,37 +3952,18 @@ class SemeionDataset(MappableDataset, VisionBaseDataset):
|
|
|
4453
3952
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
4454
3953
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
4455
3954
|
|
|
3955
|
+
Tutorial Examples:
|
|
3956
|
+
- `Load & Process Data With Dataset Pipeline
|
|
3957
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
3958
|
+
|
|
4456
3959
|
Note:
|
|
4457
|
-
-
|
|
4458
|
-
|
|
4459
|
-
|
|
4460
|
-
..
|
|
4461
|
-
:widths: 25 25 50
|
|
4462
|
-
:header-rows: 1
|
|
4463
|
-
|
|
4464
|
-
* - Parameter `sampler`
|
|
4465
|
-
- Parameter `shuffle`
|
|
4466
|
-
- Expected Order Behavior
|
|
4467
|
-
* - None
|
|
4468
|
-
- None
|
|
4469
|
-
- random order
|
|
4470
|
-
* - None
|
|
4471
|
-
- True
|
|
4472
|
-
- random order
|
|
4473
|
-
* - None
|
|
4474
|
-
- False
|
|
4475
|
-
- sequential order
|
|
4476
|
-
* - Sampler object
|
|
4477
|
-
- None
|
|
4478
|
-
- order defined by sampler
|
|
4479
|
-
* - Sampler object
|
|
4480
|
-
- True
|
|
4481
|
-
- not allowed
|
|
4482
|
-
* - Sampler object
|
|
4483
|
-
- False
|
|
4484
|
-
- not allowed
|
|
3960
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
3961
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
3962
|
+
|
|
3963
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
4485
3964
|
|
|
4486
3965
|
Examples:
|
|
3966
|
+
>>> import mindspore.dataset as ds
|
|
4487
3967
|
>>> semeion_dataset_dir = "/path/to/semeion_dataset_directory"
|
|
4488
3968
|
>>>
|
|
4489
3969
|
>>> # 1) Get all samples from SEMEION dataset in sequence
|
|
@@ -4549,29 +4029,29 @@ class STL10Dataset(MappableDataset, VisionBaseDataset):
|
|
|
4549
4029
|
|
|
4550
4030
|
Args:
|
|
4551
4031
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
4552
|
-
usage (str, optional): Usage of this dataset, can be 'train'
|
|
4553
|
-
'unlabeled'
|
|
4554
|
-
train samples, 'test' will read from 8,000 test samples,
|
|
4555
|
-
'unlabeled' will read from all 100,000 samples, and 'train+unlabeled'
|
|
4556
|
-
will read from 105000 samples, 'all' will read all the samples
|
|
4557
|
-
Default: None, all samples.
|
|
4032
|
+
usage (str, optional): Usage of this dataset, can be ``'train'``, ``'test'``,
|
|
4033
|
+
``'unlabeled'``, ``'train+unlabeled'`` or ``'all'`` . ``'train'`` will read from 5,000
|
|
4034
|
+
train samples, ``'test'`` will read from 8,000 test samples,
|
|
4035
|
+
``'unlabeled'`` will read from all 100,000 samples, and ``'train+unlabeled'``
|
|
4036
|
+
will read from 105000 samples, ``'all'`` will read all the samples
|
|
4037
|
+
Default: ``None`` , all samples.
|
|
4558
4038
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
4559
|
-
Default: None, all images.
|
|
4039
|
+
Default: ``None`` , all images.
|
|
4560
4040
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
4561
|
-
Default: None, will use global default workers(8), it can be set
|
|
4562
|
-
by
|
|
4563
|
-
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None, expected
|
|
4041
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
4042
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
4043
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected
|
|
4564
4044
|
order behavior shown in the table below.
|
|
4565
4045
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
4566
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
4046
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
4567
4047
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
4568
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
4048
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
4569
4049
|
the max sample number of per shard.
|
|
4570
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
4050
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
4571
4051
|
argument can only be specified when `num_shards` is also specified.
|
|
4572
4052
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
4573
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
4574
|
-
Default: None, which means no cache is used.
|
|
4053
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
4054
|
+
Default: ``None`` , which means no cache is used.
|
|
4575
4055
|
|
|
4576
4056
|
Raises:
|
|
4577
4057
|
RuntimeError: If `dataset_dir` is not valid or does not exist or does not contain data files.
|
|
@@ -4583,37 +4063,18 @@ class STL10Dataset(MappableDataset, VisionBaseDataset):
|
|
|
4583
4063
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
4584
4064
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
4585
4065
|
|
|
4066
|
+
Tutorial Examples:
|
|
4067
|
+
- `Load & Process Data With Dataset Pipeline
|
|
4068
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
4069
|
+
|
|
4586
4070
|
Note:
|
|
4587
|
-
-
|
|
4588
|
-
|
|
4589
|
-
|
|
4590
|
-
..
|
|
4591
|
-
:widths: 25 25 50
|
|
4592
|
-
:header-rows: 1
|
|
4593
|
-
|
|
4594
|
-
* - Parameter 'sampler'
|
|
4595
|
-
- Parameter 'shuffle'
|
|
4596
|
-
- Expected Order Behavior
|
|
4597
|
-
* - None
|
|
4598
|
-
- None
|
|
4599
|
-
- random order
|
|
4600
|
-
* - None
|
|
4601
|
-
- True
|
|
4602
|
-
- random order
|
|
4603
|
-
* - None
|
|
4604
|
-
- False
|
|
4605
|
-
- sequential order
|
|
4606
|
-
* - Sampler object
|
|
4607
|
-
- None
|
|
4608
|
-
- order defined by sampler
|
|
4609
|
-
* - Sampler object
|
|
4610
|
-
- True
|
|
4611
|
-
- not allowed
|
|
4612
|
-
* - Sampler object
|
|
4613
|
-
- False
|
|
4614
|
-
- not allowed
|
|
4071
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
4072
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
4073
|
+
|
|
4074
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
4615
4075
|
|
|
4616
4076
|
Examples:
|
|
4077
|
+
>>> import mindspore.dataset as ds
|
|
4617
4078
|
>>> stl10_dataset_dir = "/path/to/stl10_dataset_directory"
|
|
4618
4079
|
>>>
|
|
4619
4080
|
>>> # 1) Get all samples from STL10 dataset in sequence
|
|
@@ -4687,23 +4148,23 @@ class SUN397Dataset(MappableDataset, VisionBaseDataset):
|
|
|
4687
4148
|
Args:
|
|
4688
4149
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
4689
4150
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
4690
|
-
Default: None, all images.
|
|
4151
|
+
Default: ``None`` , all images.
|
|
4691
4152
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
4692
|
-
Default: None, will use global default workers(8), it can be set
|
|
4693
|
-
by
|
|
4153
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
4154
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
4694
4155
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
4695
|
-
Default: None, expected order behavior shown in the table below.
|
|
4696
|
-
decode (bool, optional): Whether or not to decode the images after reading. Default: False
|
|
4156
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
4157
|
+
decode (bool, optional): Whether or not to decode the images after reading. Default: ``False``.
|
|
4697
4158
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
4698
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
4159
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
4699
4160
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
4700
4161
|
into. When this argument is specified, `num_samples` reflects
|
|
4701
|
-
the maximum sample number of per shard. Default: None.
|
|
4162
|
+
the maximum sample number of per shard. Default: ``None`` .
|
|
4702
4163
|
shard_id (int, optional): The shard ID within `num_shards` . This
|
|
4703
|
-
argument can only be specified when `num_shards` is also specified. Default: None.
|
|
4164
|
+
argument can only be specified when `num_shards` is also specified. Default: ``None`` .
|
|
4704
4165
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
4705
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
4706
|
-
Default: None, which means no cache is used.
|
|
4166
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
4167
|
+
Default: ``None`` , which means no cache is used.
|
|
4707
4168
|
|
|
4708
4169
|
Raises:
|
|
4709
4170
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -4714,37 +4175,18 @@ class SUN397Dataset(MappableDataset, VisionBaseDataset):
|
|
|
4714
4175
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
4715
4176
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
4716
4177
|
|
|
4178
|
+
Tutorial Examples:
|
|
4179
|
+
- `Load & Process Data With Dataset Pipeline
|
|
4180
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
4181
|
+
|
|
4717
4182
|
Note:
|
|
4718
|
-
-
|
|
4719
|
-
|
|
4720
|
-
|
|
4721
|
-
..
|
|
4722
|
-
:widths: 25 25 50
|
|
4723
|
-
:header-rows: 1
|
|
4724
|
-
|
|
4725
|
-
* - Parameter `sampler`
|
|
4726
|
-
- Parameter `shuffle`
|
|
4727
|
-
- Expected Order Behavior
|
|
4728
|
-
* - None
|
|
4729
|
-
- None
|
|
4730
|
-
- random order
|
|
4731
|
-
* - None
|
|
4732
|
-
- True
|
|
4733
|
-
- random order
|
|
4734
|
-
* - None
|
|
4735
|
-
- False
|
|
4736
|
-
- sequential order
|
|
4737
|
-
* - Sampler object
|
|
4738
|
-
- None
|
|
4739
|
-
- order defined by sampler
|
|
4740
|
-
* - Sampler object
|
|
4741
|
-
- True
|
|
4742
|
-
- not allowed
|
|
4743
|
-
* - Sampler object
|
|
4744
|
-
- False
|
|
4745
|
-
- not allowed
|
|
4183
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
4184
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
4185
|
+
|
|
4186
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
4746
4187
|
|
|
4747
4188
|
Examples:
|
|
4189
|
+
>>> import mindspore.dataset as ds
|
|
4748
4190
|
>>> sun397_dataset_dir = "/path/to/sun397_dataset_directory"
|
|
4749
4191
|
>>>
|
|
4750
4192
|
>>> # 1) Read all samples (image files) in sun397_dataset_dir with 8 threads
|
|
@@ -4867,19 +4309,20 @@ class SVHNDataset(GeneratorDataset):
|
|
|
4867
4309
|
|
|
4868
4310
|
Args:
|
|
4869
4311
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
4870
|
-
usage (str, optional): Specify the 'train'
|
|
4871
|
-
Default: None, will read all samples.
|
|
4872
|
-
num_samples (int, optional): The number of samples to be included in the dataset. Default: None,
|
|
4312
|
+
usage (str, optional): Specify the ``'train'``, ``'test'``, ``'extra'`` or ``'all'`` parts of dataset.
|
|
4313
|
+
Default: ``None`` , will read all samples.
|
|
4314
|
+
num_samples (int, optional): The number of samples to be included in the dataset. Default: ``None`` ,
|
|
4315
|
+
all images.
|
|
4873
4316
|
num_parallel_workers (int, optional): Number of worker subprocesses used to
|
|
4874
|
-
fetch the dataset in parallel. Default: 1
|
|
4317
|
+
fetch the dataset in parallel. Default: ``1``.
|
|
4875
4318
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
4876
|
-
Default: None, expected order behavior shown in the table below.
|
|
4319
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
4877
4320
|
sampler (Sampler, optional): Object used to choose samples from the dataset. Random accessible
|
|
4878
|
-
input is required. Default: None, expected order behavior shown in the table below.
|
|
4879
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
4321
|
+
input is required. Default: ``None`` , expected order behavior shown in the table below.
|
|
4322
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
4880
4323
|
When this argument is specified, `num_samples` reflects the max sample number of per shard.
|
|
4881
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None.
|
|
4882
|
-
when `num_shards` is also specified.
|
|
4324
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` .
|
|
4325
|
+
This argument must be specified only when `num_shards` is also specified.
|
|
4883
4326
|
|
|
4884
4327
|
Raises:
|
|
4885
4328
|
RuntimeError: If `dataset_dir` is not valid or does not exist or does not contain data files.
|
|
@@ -4891,37 +4334,18 @@ class SVHNDataset(GeneratorDataset):
|
|
|
4891
4334
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
4892
4335
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
4893
4336
|
|
|
4337
|
+
Tutorial Examples:
|
|
4338
|
+
- `Load & Process Data With Dataset Pipeline
|
|
4339
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
4340
|
+
|
|
4894
4341
|
Note:
|
|
4895
|
-
-
|
|
4896
|
-
|
|
4897
|
-
|
|
4898
|
-
..
|
|
4899
|
-
:widths: 25 25 50
|
|
4900
|
-
:header-rows: 1
|
|
4901
|
-
|
|
4902
|
-
* - Parameter 'sampler'
|
|
4903
|
-
- Parameter 'shuffle'
|
|
4904
|
-
- Expected Order Behavior
|
|
4905
|
-
* - None
|
|
4906
|
-
- None
|
|
4907
|
-
- random order
|
|
4908
|
-
* - None
|
|
4909
|
-
- True
|
|
4910
|
-
- random order
|
|
4911
|
-
* - None
|
|
4912
|
-
- False
|
|
4913
|
-
- sequential order
|
|
4914
|
-
* - Sampler object
|
|
4915
|
-
- None
|
|
4916
|
-
- order defined by sampler
|
|
4917
|
-
* - Sampler object
|
|
4918
|
-
- True
|
|
4919
|
-
- not allowed
|
|
4920
|
-
* - Sampler object
|
|
4921
|
-
- False
|
|
4922
|
-
- not allowed
|
|
4342
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
4343
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
4344
|
+
|
|
4345
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
4923
4346
|
|
|
4924
4347
|
Examples:
|
|
4348
|
+
>>> import mindspore.dataset as ds
|
|
4925
4349
|
>>> svhn_dataset_dir = "/path/to/svhn_dataset_directory"
|
|
4926
4350
|
>>> dataset = ds.SVHNDataset(dataset_dir=svhn_dataset_dir, usage="train")
|
|
4927
4351
|
|
|
@@ -4977,31 +4401,31 @@ class USPSDataset(SourceDataset, VisionBaseDataset):
|
|
|
4977
4401
|
|
|
4978
4402
|
Args:
|
|
4979
4403
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
4980
|
-
usage (str, optional): Usage of this dataset, can be 'train'
|
|
4981
|
-
train
|
|
4982
|
-
Default: None, will read all samples.
|
|
4404
|
+
usage (str, optional): Usage of this dataset, can be ``'train'``, ``'test'`` or ``'all'`` .
|
|
4405
|
+
``'train'`` will read from 7,291 train samples, ``'test'`` will read from 2,007 test samples,
|
|
4406
|
+
``'all'`` will read from all 9,298 samples. Default: ``None`` , will read all samples.
|
|
4983
4407
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
4984
|
-
Default: None, will read all images.
|
|
4408
|
+
Default: ``None`` , will read all images.
|
|
4985
4409
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
4986
|
-
Default: None, will use global default workers(8), it can be set
|
|
4987
|
-
by
|
|
4410
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
4411
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
4988
4412
|
shuffle (Union[bool, Shuffle], optional): Perform reshuffling of the data every epoch.
|
|
4989
|
-
Bool type and Shuffle enum are both supported to pass in. Default:
|
|
4990
|
-
If shuffle is False, no shuffling will be performed.
|
|
4991
|
-
If shuffle is True, it is equivalent to setting `shuffle` to mindspore.dataset.Shuffle.GLOBAL
|
|
4413
|
+
Bool type and Shuffle enum are both supported to pass in. Default: ``Shuffle.GLOBAL`` .
|
|
4414
|
+
If `shuffle` is ``False`` , no shuffling will be performed.
|
|
4415
|
+
If `shuffle` is ``True`` , it is equivalent to setting `shuffle` to ``mindspore.dataset.Shuffle.GLOBAL``.
|
|
4992
4416
|
Set the mode of data shuffling by passing in enumeration variables:
|
|
4993
4417
|
|
|
4994
|
-
- Shuffle.GLOBAL: Shuffle both the files and samples.
|
|
4418
|
+
- ``Shuffle.GLOBAL`` : Shuffle both the files and samples.
|
|
4995
4419
|
|
|
4996
|
-
- Shuffle.FILES: Shuffle files only.
|
|
4420
|
+
- ``Shuffle.FILES`` : Shuffle files only.
|
|
4997
4421
|
|
|
4998
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
4422
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
4999
4423
|
When this argument is specified, `num_samples` reflects the max sample number of per shard.
|
|
5000
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
4424
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
5001
4425
|
argument can only be specified when `num_shards` is also specified.
|
|
5002
4426
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
5003
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
5004
|
-
Default: None, which means no cache is used.
|
|
4427
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
4428
|
+
Default: ``None`` , which means no cache is used.
|
|
5005
4429
|
|
|
5006
4430
|
Raises:
|
|
5007
4431
|
RuntimeError: If `dataset_dir` is not valid or does not exist or does not contain data files.
|
|
@@ -5011,7 +4435,12 @@ class USPSDataset(SourceDataset, VisionBaseDataset):
|
|
|
5011
4435
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
5012
4436
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
5013
4437
|
|
|
4438
|
+
Tutorial Examples:
|
|
4439
|
+
- `Load & Process Data With Dataset Pipeline
|
|
4440
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
4441
|
+
|
|
5014
4442
|
Examples:
|
|
4443
|
+
>>> import mindspore.dataset as ds
|
|
5015
4444
|
>>> usps_dataset_dir = "/path/to/usps_dataset_directory"
|
|
5016
4445
|
>>>
|
|
5017
4446
|
>>> # Read 3 samples from USPS dataset
|
|
@@ -5067,45 +4496,48 @@ class VOCDataset(MappableDataset, VisionBaseDataset):
|
|
|
5067
4496
|
"""
|
|
5068
4497
|
VOC(Visual Object Classes) dataset.
|
|
5069
4498
|
|
|
5070
|
-
The generated dataset with different task setting has different output columns:
|
|
4499
|
+
The generated dataset with different `task` setting has different output columns:
|
|
5071
4500
|
|
|
5072
|
-
- task = :py:obj:`Detection` , output columns: :py:obj:`[image, dtype=uint8]` ,
|
|
5073
|
-
:py:obj:`[
|
|
5074
|
-
|
|
4501
|
+
- `task` = :py:obj:`Detection` , output columns: :py:obj:`[image, dtype=uint8]` ,
|
|
4502
|
+
:py:obj:`[bbox, dtype=float32]` , :py:obj:`[label, dtype=uint32]` ,
|
|
4503
|
+
:py:obj:`[difficult, dtype=uint32]` , :py:obj:`[truncate, dtype=uint32]` .
|
|
4504
|
+
- `task` = :py:obj:`Segmentation` , output columns: :py:obj:`[image, dtype=uint8]` ,
|
|
4505
|
+
:py:obj:`[target,dtype=uint8]` .
|
|
5075
4506
|
|
|
5076
4507
|
Args:
|
|
5077
4508
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
5078
|
-
task (str, optional): Set the task type of reading voc data, now only support 'Segmentation' or
|
|
5079
|
-
Default: 'Segmentation'
|
|
5080
|
-
usage (str, optional): Set the task type of ImageSets. Default: 'train'
|
|
5081
|
-
annotation list will be loaded in ./ImageSets/Segmentation/usage + ".txt";
|
|
5082
|
-
annotation list will be loaded in ./ImageSets/Main/usage + ".txt";
|
|
5083
|
-
annotation list will be loaded in
|
|
4509
|
+
task (str, optional): Set the task type of reading voc data, now only support ``'Segmentation'`` or
|
|
4510
|
+
``'Detection'``. Default: ``'Segmentation'``.
|
|
4511
|
+
usage (str, optional): Set the task type of ImageSets. Default: ``'train'``. If `task` is ``'Segmentation'``,
|
|
4512
|
+
image and annotation list will be loaded in ./ImageSets/Segmentation/usage + ".txt";
|
|
4513
|
+
If `task` is 'Detection', image and annotation list will be loaded in ./ImageSets/Main/usage + ".txt";
|
|
4514
|
+
if `task` and `usage` are not set, image and annotation list will be loaded in
|
|
4515
|
+
./ImageSets/Segmentation/train.txt as default.
|
|
5084
4516
|
class_indexing (dict, optional): A str-to-int mapping from label name to index, only valid in
|
|
5085
|
-
'Detection' task. Default: None, the folder names will be sorted alphabetically and each
|
|
4517
|
+
'Detection' task. Default: ``None`` , the folder names will be sorted alphabetically and each
|
|
5086
4518
|
class will be given a unique index starting from 0.
|
|
5087
4519
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
5088
|
-
Default: None, all images.
|
|
4520
|
+
Default: ``None`` , all images.
|
|
5089
4521
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
5090
|
-
Default: None, will use global default workers(8), it can be set
|
|
5091
|
-
by
|
|
5092
|
-
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None, expected
|
|
4522
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
4523
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
4524
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected
|
|
5093
4525
|
order behavior shown in the table below.
|
|
5094
|
-
decode (bool, optional): Decode the images after reading. Default: False
|
|
4526
|
+
decode (bool, optional): Decode the images after reading. Default: ``False``.
|
|
5095
4527
|
sampler (Sampler, optional): Object used to choose samples from the dataset.
|
|
5096
|
-
Default: None, expected order behavior shown in the table below.
|
|
4528
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
5097
4529
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
5098
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
4530
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
5099
4531
|
the maximum sample number of per shard.
|
|
5100
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
4532
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
5101
4533
|
argument can only be specified when `num_shards` is also specified.
|
|
5102
4534
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
5103
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
5104
|
-
Default: None, which means no cache is used.
|
|
4535
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
4536
|
+
Default: ``None`` , which means no cache is used.
|
|
5105
4537
|
extra_metadata(bool, optional): Flag to add extra meta-data to row. If True, an additional column named
|
|
5106
|
-
:py:obj:`[_meta-filename, dtype=string]` will be output at the end. Default: False
|
|
4538
|
+
:py:obj:`[_meta-filename, dtype=string]` will be output at the end. Default: ``False``.
|
|
5107
4539
|
decrypt (callable, optional): Image decryption function, which accepts the path of the encrypted image file
|
|
5108
|
-
and returns the decrypted bytes data. Default: None, no decryption.
|
|
4540
|
+
and returns the decrypted bytes data. Default: ``None`` , no decryption.
|
|
5109
4541
|
|
|
5110
4542
|
Raises:
|
|
5111
4543
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -5117,44 +4549,25 @@ class VOCDataset(MappableDataset, VisionBaseDataset):
|
|
|
5117
4549
|
RuntimeError: If `num_shards` is specified but `shard_id` is None.
|
|
5118
4550
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
5119
4551
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
5120
|
-
ValueError: If task is not equal 'Segmentation' or 'Detection'
|
|
5121
|
-
ValueError: If task
|
|
4552
|
+
ValueError: If `task` is not equal ``'Segmentation'`` or ``'Detection'``.
|
|
4553
|
+
ValueError: If `task` is ``'Segmentation'`` but `class_indexing` is not ``None``.
|
|
5122
4554
|
ValueError: If txt related to mode is not exist.
|
|
5123
4555
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
5124
4556
|
|
|
4557
|
+
Tutorial Examples:
|
|
4558
|
+
- `Load & Process Data With Dataset Pipeline
|
|
4559
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
4560
|
+
|
|
5125
4561
|
Note:
|
|
5126
4562
|
- Column '[_meta-filename, dtype=string]' won't be output unless an explicit rename dataset op
|
|
5127
4563
|
is added to remove the prefix('_meta-').
|
|
5128
|
-
-
|
|
5129
|
-
|
|
5130
|
-
|
|
5131
|
-
..
|
|
5132
|
-
:widths: 25 25 50
|
|
5133
|
-
:header-rows: 1
|
|
5134
|
-
|
|
5135
|
-
* - Parameter `sampler`
|
|
5136
|
-
- Parameter `shuffle`
|
|
5137
|
-
- Expected Order Behavior
|
|
5138
|
-
* - None
|
|
5139
|
-
- None
|
|
5140
|
-
- random order
|
|
5141
|
-
* - None
|
|
5142
|
-
- True
|
|
5143
|
-
- random order
|
|
5144
|
-
* - None
|
|
5145
|
-
- False
|
|
5146
|
-
- sequential order
|
|
5147
|
-
* - Sampler object
|
|
5148
|
-
- None
|
|
5149
|
-
- order defined by sampler
|
|
5150
|
-
* - Sampler object
|
|
5151
|
-
- True
|
|
5152
|
-
- not allowed
|
|
5153
|
-
* - Sampler object
|
|
5154
|
-
- False
|
|
5155
|
-
- not allowed
|
|
4564
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
4565
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
4566
|
+
|
|
4567
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
5156
4568
|
|
|
5157
4569
|
Examples:
|
|
4570
|
+
>>> import mindspore.dataset as ds
|
|
5158
4571
|
>>> voc_dataset_dir = "/path/to/voc_dataset_directory"
|
|
5159
4572
|
>>>
|
|
5160
4573
|
>>> # 1) Read VOC data for segmentation training
|
|
@@ -5253,6 +4666,7 @@ class VOCDataset(MappableDataset, VisionBaseDataset):
|
|
|
5253
4666
|
dict, a str-to-int mapping from label name to index.
|
|
5254
4667
|
|
|
5255
4668
|
Examples:
|
|
4669
|
+
>>> import mindspore.dataset as ds
|
|
5256
4670
|
>>> voc_dataset_dir = "/path/to/voc_dataset_directory"
|
|
5257
4671
|
>>>
|
|
5258
4672
|
>>> dataset = ds.VOCDataset(dataset_dir=voc_dataset_dir, task="Detection")
|
|
@@ -5281,26 +4695,27 @@ class WIDERFaceDataset(MappableDataset, VisionBaseDataset):
|
|
|
5281
4695
|
|
|
5282
4696
|
Args:
|
|
5283
4697
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
5284
|
-
usage (str, optional): Usage of this dataset, can be 'train'
|
|
5285
|
-
|
|
5286
|
-
|
|
4698
|
+
usage (str, optional): Usage of this dataset, can be ``'train'``, ``'test'``, ``'valid'`` or
|
|
4699
|
+
``'all'``. ``'train'`` will read from 12,880 samples, ``'test'`` will read from 16,097 samples,
|
|
4700
|
+
``'valid'`` will read from 3,226 test samples and ``'all'`` will read all ``'train'``
|
|
4701
|
+
and ``'valid'`` samples. Default: ``None`` , will be set to ``'all'``.
|
|
5287
4702
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
5288
|
-
Default: None, will read all images.
|
|
4703
|
+
Default: ``None`` , will read all images.
|
|
5289
4704
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
5290
|
-
Default: None, will use global default workers(8), it can be set
|
|
5291
|
-
by
|
|
4705
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
4706
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
5292
4707
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
5293
|
-
Default: None, expected order behavior shown in the table below.
|
|
5294
|
-
decode (bool, optional): Decode the images after reading. Default: False
|
|
4708
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
4709
|
+
decode (bool, optional): Decode the images after reading. Default: ``False``.
|
|
5295
4710
|
sampler (Sampler, optional): Object used to choose samples from the dataset.
|
|
5296
|
-
Default: None, expected order behavior shown in the table below.
|
|
5297
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
4711
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
4712
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
5298
4713
|
When this argument is specified, `num_samples` reflects the maximum sample number of per shard.
|
|
5299
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None.
|
|
5300
|
-
when `num_shards` is also specified.
|
|
4714
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` .
|
|
4715
|
+
This argument can only be specified when `num_shards` is also specified.
|
|
5301
4716
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
5302
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
5303
|
-
Default: None, which means no cache is used.
|
|
4717
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
4718
|
+
Default: ``None`` , which means no cache is used.
|
|
5304
4719
|
|
|
5305
4720
|
Raises:
|
|
5306
4721
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -5309,42 +4724,23 @@ class WIDERFaceDataset(MappableDataset, VisionBaseDataset):
|
|
|
5309
4724
|
RuntimeError: If `num_shards` is specified but `shard_id` is None.
|
|
5310
4725
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
5311
4726
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
5312
|
-
ValueError: If `usage` is not
|
|
4727
|
+
ValueError: If `usage` is not ``'train'``, ``'test'``, ``'valid'``, ``'all'``.
|
|
5313
4728
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
5314
4729
|
ValueError: If `annotation_file` is not exist.
|
|
5315
4730
|
ValueError: If `dataset_dir` is not exist.
|
|
5316
4731
|
|
|
4732
|
+
Tutorial Examples:
|
|
4733
|
+
- `Load & Process Data With Dataset Pipeline
|
|
4734
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
4735
|
+
|
|
5317
4736
|
Note:
|
|
5318
|
-
-
|
|
5319
|
-
|
|
5320
|
-
|
|
5321
|
-
..
|
|
5322
|
-
:widths: 25 25 50
|
|
5323
|
-
:header-rows: 1
|
|
5324
|
-
|
|
5325
|
-
* - Parameter `sampler`
|
|
5326
|
-
- Parameter `shuffle`
|
|
5327
|
-
- Expected Order Behavior
|
|
5328
|
-
* - None
|
|
5329
|
-
- None
|
|
5330
|
-
- random order
|
|
5331
|
-
* - None
|
|
5332
|
-
- True
|
|
5333
|
-
- random order
|
|
5334
|
-
* - None
|
|
5335
|
-
- False
|
|
5336
|
-
- sequential order
|
|
5337
|
-
* - Sampler object
|
|
5338
|
-
- None
|
|
5339
|
-
- order defined by sampler
|
|
5340
|
-
* - Sampler object
|
|
5341
|
-
- True
|
|
5342
|
-
- not allowed
|
|
5343
|
-
* - Sampler object
|
|
5344
|
-
- False
|
|
5345
|
-
- not allowed
|
|
4737
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
4738
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
4739
|
+
|
|
4740
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
5346
4741
|
|
|
5347
4742
|
Examples:
|
|
4743
|
+
>>> import mindspore.dataset as ds
|
|
5348
4744
|
>>> wider_face_dir = "/path/to/wider_face_dataset"
|
|
5349
4745
|
>>>
|
|
5350
4746
|
>>> # Read 3 samples from WIDERFace dataset
|