mindspore 2.0.0rc1__cp38-none-any.whl → 2.2.0__cp38-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mindspore might be problematic. Click here for more details.
- mindspore/.commit_id +1 -1
- mindspore/Third_Party_Open_Source_Software_Notice +2 -2
- mindspore/__init__.py +5 -2
- mindspore/_akg/akg/build_module.py +5 -6
- mindspore/_akg/akg/composite/build_module.py +49 -16
- mindspore/_akg/akg/composite/split_stitch.py +10 -11
- mindspore/_akg/akg/config/repository.json +195 -0
- mindspore/_akg/akg/global_configs.py +5 -1
- mindspore/_akg/akg/ms/info_version_adapt.py +67 -1
- mindspore/_akg/akg/tvm/api.py +4 -3
- mindspore/_akg/akg/tvm/autotvm/__init__.py +1 -2
- mindspore/_akg/akg/tvm/autotvm/graph_tuner/base_graph_tuner.py +1 -5
- mindspore/_akg/akg/tvm/autotvm/measure/__init__.py +1 -1
- mindspore/_akg/akg/tvm/autotvm/measure/measure.py +1 -10
- mindspore/_akg/akg/tvm/autotvm/measure/measure_methods.py +1 -372
- mindspore/_akg/akg/tvm/build_module.py +16 -1
- mindspore/_akg/akg/tvm/contrib/graph_runtime.py +0 -53
- mindspore/_akg/akg/tvm/hybrid/parser.py +7 -6
- mindspore/_akg/akg/tvm/ir_builder.py +1 -1
- mindspore/_akg/akg/tvm/module.py +1 -2
- mindspore/_akg/akg/tvm/stmt.py +2 -2
- mindspore/_akg/akg/utils/composite_op_helper.py +9 -10
- mindspore/_akg/akg/utils/kernel_exec.py +58 -260
- mindspore/_akg/akg/utils/op_dsl.py +17 -1
- mindspore/_akg/akg/utils/result_analysis.py +4 -24
- mindspore/_akg/akg/utils/tbe_codegen_utils.py +198 -0
- mindspore/_c_dataengine.cpython-38-aarch64-linux-gnu.so +0 -0
- mindspore/_c_expression.cpython-38-aarch64-linux-gnu.so +0 -0
- mindspore/_c_mindrecord.cpython-38-aarch64-linux-gnu.so +0 -0
- mindspore/_check_jit_forbidden_api.py +5 -1
- mindspore/_checkparam.py +79 -62
- mindspore/_extends/graph_kernel/__init__.py +0 -1
- mindspore/_extends/graph_kernel/model/graph_split.py +2 -0
- mindspore/_extends/graph_kernel/model/model_builder.py +9 -50
- mindspore/_extends/graph_kernel/splitter.py +1 -9
- mindspore/_extends/parallel_compile/akg_compiler/akg_process.py +128 -21
- mindspore/_extends/parallel_compile/akg_compiler/build_tbe_kernel.py +2 -2
- mindspore/_extends/parallel_compile/akg_compiler/tbe_topi.py +4 -2
- mindspore/_extends/parallel_compile/tbe_compiler/tbe_adapter.py +18 -13
- mindspore/_extends/parallel_compile/tbe_compiler/tbe_helper.py +13 -9
- mindspore/_extends/parallel_compile/tbe_compiler/tbe_job.py +1 -1
- mindspore/_extends/parallel_compile/tbe_compiler/tbe_job_manager.py +1 -1
- mindspore/_extends/parse/__init__.py +19 -17
- mindspore/_extends/parse/namespace.py +7 -36
- mindspore/_extends/parse/parser.py +375 -189
- mindspore/_extends/parse/resources.py +36 -41
- mindspore/_extends/parse/standard_method.py +350 -245
- mindspore/_extends/parse/trope.py +2 -12
- mindspore/_extends/remote/kernel_build_server.py +24 -7
- mindspore/_extends/remote/kernel_build_server_akg_v2.py +55 -0
- mindspore/_install_custom.py +43 -0
- mindspore/_mindspore_offline_debug.cpython-38-aarch64-linux-gnu.so +0 -0
- mindspore/amp.py +85 -19
- mindspore/bin/cache_admin +0 -0
- mindspore/bin/cache_server +0 -0
- mindspore/boost/base.py +2 -2
- mindspore/boost/boost.py +27 -32
- mindspore/boost/boost_cell_wrapper.py +37 -13
- mindspore/boost/grad_accumulation.py +1 -1
- mindspore/boost/grad_freeze.py +34 -6
- mindspore/boost/group_loss_scale_manager.py +15 -14
- mindspore/boost/less_batch_normalization.py +28 -3
- mindspore/common/__init__.py +15 -11
- mindspore/common/_auto_dynamic.py +68 -0
- mindspore/common/_jit_fallback_utils.py +111 -0
- mindspore/common/_register_for_adapter.py +17 -5
- mindspore/common/_register_for_tensor.py +2 -2
- mindspore/common/_stub_tensor.py +18 -15
- mindspore/common/_utils.py +31 -7
- mindspore/common/api.py +269 -101
- mindspore/common/auto_dynamic_shape.py +498 -0
- mindspore/common/dtype.py +61 -21
- mindspore/common/dump.py +9 -7
- mindspore/common/initializer.py +106 -76
- mindspore/common/jit_config.py +35 -14
- mindspore/common/lazy_inline.py +187 -0
- mindspore/common/mindir_util.py +101 -0
- mindspore/common/mutable.py +10 -13
- mindspore/common/parameter.py +246 -55
- mindspore/common/seed.py +13 -7
- mindspore/common/sparse_tensor.py +29 -33
- mindspore/common/tensor.py +907 -251
- mindspore/communication/__init__.py +7 -4
- mindspore/communication/_comm_helper.py +84 -4
- mindspore/communication/management.py +160 -88
- mindspore/config/op_info.config +99 -75
- mindspore/config/super_bar_config.json +36 -4
- mindspore/context.py +526 -219
- mindspore/dataset/__init__.py +9 -46
- mindspore/dataset/audio/__init__.py +4 -19
- mindspore/dataset/audio/transforms.py +545 -233
- mindspore/dataset/audio/utils.py +21 -18
- mindspore/dataset/callback/ds_callback.py +42 -13
- mindspore/dataset/core/config.py +158 -100
- mindspore/dataset/core/validator_helpers.py +1 -63
- mindspore/dataset/debug/debug_hook.py +45 -13
- mindspore/dataset/debug/pre_defined_hook.py +5 -5
- mindspore/dataset/engine/__init__.py +0 -5
- mindspore/dataset/engine/cache_client.py +38 -15
- mindspore/dataset/engine/datasets.py +615 -278
- mindspore/dataset/engine/datasets_audio.py +154 -283
- mindspore/dataset/engine/datasets_standard_format.py +104 -116
- mindspore/dataset/engine/datasets_text.py +443 -326
- mindspore/dataset/engine/datasets_user_defined.py +251 -164
- mindspore/dataset/engine/datasets_vision.py +839 -1443
- mindspore/dataset/engine/iterators.py +11 -4
- mindspore/dataset/engine/obs/obs_mindrecord_dataset.py +7 -3
- mindspore/dataset/engine/obs/util.py +3 -0
- mindspore/dataset/engine/offload.py +6 -6
- mindspore/dataset/engine/queue.py +15 -14
- mindspore/dataset/engine/samplers.py +39 -23
- mindspore/dataset/engine/serializer_deserializer.py +22 -6
- mindspore/dataset/engine/validators.py +21 -331
- mindspore/dataset/text/__init__.py +5 -33
- mindspore/dataset/text/transforms.py +334 -165
- mindspore/dataset/text/utils.py +215 -145
- mindspore/dataset/transforms/__init__.py +1 -1
- mindspore/dataset/transforms/c_transforms.py +3 -2
- mindspore/dataset/transforms/py_transforms_util.py +40 -12
- mindspore/dataset/transforms/transforms.py +174 -71
- mindspore/dataset/utils/browse_dataset.py +25 -17
- mindspore/dataset/utils/line_reader.py +24 -21
- mindspore/dataset/vision/__init__.py +5 -26
- mindspore/dataset/vision/c_transforms.py +177 -165
- mindspore/dataset/vision/py_transforms.py +114 -119
- mindspore/dataset/vision/py_transforms_util.py +54 -51
- mindspore/dataset/vision/transforms.py +1127 -381
- mindspore/dataset/vision/utils.py +54 -38
- mindspore/dataset/vision/validators.py +12 -2
- mindspore/experimental/map_parameter.py +38 -4
- mindspore/{dataset/datapreprocess → experimental/optim}/__init__.py +14 -4
- mindspore/experimental/optim/adam.py +192 -0
- mindspore/experimental/optim/adamw.py +181 -0
- mindspore/experimental/optim/lr_scheduler.py +1427 -0
- mindspore/experimental/optim/optimizer.py +252 -0
- mindspore/experimental/optim/sgd.py +147 -0
- mindspore/gen_ops.py +273 -0
- mindspore/include/OWNERS +1 -2
- mindspore/include/api/context.h +21 -1
- mindspore/include/api/data_type.h +2 -1
- mindspore/include/api/graph.h +0 -15
- mindspore/include/api/kernel.h +2 -0
- mindspore/include/api/kernel_api.h +37 -12
- mindspore/include/api/model.h +29 -42
- mindspore/include/api/model_group.h +14 -3
- mindspore/include/api/model_parallel_runner.h +18 -2
- mindspore/include/api/serialization.h +26 -0
- mindspore/include/api/status.h +1 -0
- mindspore/include/api/types.h +38 -4
- mindspore/include/c_api/ms/abstract.h +67 -0
- mindspore/include/c_api/ms/attribute.h +197 -0
- mindspore/include/c_api/ms/base/handle_types.h +43 -0
- mindspore/include/c_api/ms/base/macros.h +32 -0
- mindspore/include/c_api/ms/base/status.h +33 -0
- mindspore/include/c_api/ms/base/types.h +282 -0
- mindspore/include/c_api/ms/context.h +102 -0
- mindspore/include/c_api/ms/graph.h +160 -0
- mindspore/include/c_api/ms/node.h +606 -0
- mindspore/include/c_api/ms/tensor.h +161 -0
- mindspore/include/c_api/ms/value.h +84 -0
- mindspore/include/c_api/status_c.h +3 -0
- mindspore/include/dataset/constants.h +6 -12
- mindspore/include/dataset/execute.h +23 -13
- mindspore/include/dataset/text.h +26 -26
- mindspore/include/dataset/transforms.h +25 -31
- mindspore/include/dataset/vision.h +60 -60
- mindspore/include/dataset/vision_ascend.h +5 -6
- mindspore/include/dataset/vision_lite.h +17 -17
- mindspore/include/mindapi/base/format.h +0 -1
- mindspore/include/mindapi/base/type_id.h +2 -1
- mindspore/include/mindapi/base/types.h +5 -1
- mindspore/lib/libdnnl.so.2 +0 -0
- mindspore/lib/libjemalloc.so.2 +0 -0
- mindspore/lib/libmindspore.so +0 -0
- mindspore/lib/libmindspore_backend.so +0 -0
- mindspore/lib/libmindspore_common.so +0 -0
- mindspore/lib/libmindspore_core.so +0 -0
- mindspore/lib/libmindspore_glog.so.0 +0 -0
- mindspore/lib/libmindspore_gpr.so.15 +0 -0
- mindspore/lib/libmindspore_grpc++.so.1 +0 -0
- mindspore/lib/libmindspore_grpc.so.15 +0 -0
- mindspore/lib/libmindspore_shared_lib.so +0 -0
- mindspore/lib/libmpi_adapter.so +0 -0
- mindspore/lib/libnnacl.so +0 -0
- mindspore/lib/libopencv_core.so.4.5 +0 -0
- mindspore/lib/libopencv_imgcodecs.so.4.5 +0 -0
- mindspore/lib/libopencv_imgproc.so.4.5 +0 -0
- mindspore/lib/libps_cache.so +0 -0
- mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/aicpu_kernel/impl/libcust_aicpu_kernels.so +0 -0
- mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/aicpu_kernel/impl/libcust_cpu_kernels.so +0 -0
- mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/config/cust_aicpu_kernel.json +9000 -0
- mindspore/lib/plugin/ascend/custom_aicpu_ops/op_proto/libcust_op_proto.so +0 -0
- mindspore/lib/plugin/ascend/libakg.so +0 -0
- mindspore/lib/plugin/ascend/libascend_collective.so +0 -0
- mindspore/lib/plugin/ascend/libdvpp_utils.so +0 -0
- mindspore/lib/plugin/ascend/libhccl_plugin.so +0 -0
- mindspore/lib/plugin/ascend/libmindspore_aicpu_kernels.so +0 -0
- mindspore/lib/plugin/ascend/libmindspore_cpu_kernels.so +0 -0
- mindspore/lib/plugin/cpu/libakg.so +0 -0
- mindspore/lib/plugin/libmindspore_ascend.so.1 +0 -0
- mindspore/lib/plugin/libmindspore_ascend.so.2 +0 -0
- mindspore/log.py +9 -6
- mindspore/mindrecord/filereader.py +33 -4
- mindspore/mindrecord/filewriter.py +70 -35
- mindspore/mindrecord/mindpage.py +40 -34
- mindspore/mindrecord/shardreader.py +1 -1
- mindspore/mindrecord/shardsegment.py +1 -1
- mindspore/mindrecord/tools/cifar100_to_mr.py +25 -18
- mindspore/mindrecord/tools/cifar10_to_mr.py +25 -18
- mindspore/mindrecord/tools/csv_to_mr.py +29 -13
- mindspore/mindrecord/tools/imagenet_to_mr.py +24 -10
- mindspore/mindrecord/tools/mnist_to_mr.py +24 -11
- mindspore/mindrecord/tools/tfrecord_to_mr.py +31 -26
- mindspore/nn/cell.py +463 -169
- mindspore/nn/dynamic_lr.py +47 -43
- mindspore/nn/layer/activation.py +225 -82
- mindspore/nn/layer/basic.py +121 -79
- mindspore/nn/layer/channel_shuffle.py +21 -21
- mindspore/nn/layer/combined.py +33 -26
- mindspore/nn/layer/container.py +277 -22
- mindspore/nn/layer/conv.py +441 -304
- mindspore/nn/layer/dense.py +19 -13
- mindspore/nn/layer/embedding.py +62 -49
- mindspore/nn/layer/flash_attention.py +264 -0
- mindspore/nn/layer/image.py +50 -39
- mindspore/nn/layer/math.py +62 -51
- mindspore/nn/layer/normalization.py +219 -167
- mindspore/nn/layer/padding.py +58 -70
- mindspore/nn/layer/pooling.py +334 -287
- mindspore/nn/layer/rnn_cells.py +53 -38
- mindspore/nn/layer/rnns.py +59 -56
- mindspore/nn/layer/thor_layer.py +52 -44
- mindspore/nn/layer/timedistributed.py +6 -4
- mindspore/nn/layer/transformer.py +284 -164
- mindspore/nn/learning_rate_schedule.py +34 -25
- mindspore/nn/loss/__init__.py +3 -2
- mindspore/nn/loss/loss.py +554 -311
- mindspore/nn/optim/ada_grad.py +12 -9
- mindspore/nn/optim/adadelta.py +14 -11
- mindspore/nn/optim/adafactor.py +19 -16
- mindspore/nn/optim/adam.py +62 -47
- mindspore/nn/optim/adamax.py +13 -10
- mindspore/nn/optim/adasum.py +12 -8
- mindspore/nn/optim/asgd.py +10 -9
- mindspore/nn/optim/ftrl.py +20 -17
- mindspore/nn/optim/lamb.py +16 -12
- mindspore/nn/optim/lars.py +8 -6
- mindspore/nn/optim/lazyadam.py +25 -20
- mindspore/nn/optim/momentum.py +10 -7
- mindspore/nn/optim/optimizer.py +61 -9
- mindspore/nn/optim/proximal_ada_grad.py +14 -13
- mindspore/nn/optim/rmsprop.py +17 -13
- mindspore/nn/optim/rprop.py +30 -17
- mindspore/nn/optim/sgd.py +40 -23
- mindspore/nn/optim/thor.py +24 -26
- mindspore/nn/probability/bijector/bijector.py +11 -11
- mindspore/nn/probability/bijector/exp.py +1 -1
- mindspore/nn/probability/bijector/gumbel_cdf.py +3 -3
- mindspore/nn/probability/bijector/invert.py +1 -1
- mindspore/nn/probability/bijector/power_transform.py +29 -29
- mindspore/nn/probability/bijector/scalar_affine.py +3 -3
- mindspore/nn/probability/bijector/softplus.py +5 -5
- mindspore/nn/probability/bnn_layers/bnn_cell_wrapper.py +4 -2
- mindspore/nn/probability/bnn_layers/conv_variational.py +13 -13
- mindspore/nn/probability/bnn_layers/dense_variational.py +12 -12
- mindspore/nn/probability/bnn_layers/layer_distribution.py +9 -8
- mindspore/nn/probability/distribution/_utils/custom_ops.py +19 -3
- mindspore/nn/probability/distribution/_utils/utils.py +1 -1
- mindspore/nn/probability/distribution/bernoulli.py +9 -9
- mindspore/nn/probability/distribution/beta.py +8 -8
- mindspore/nn/probability/distribution/categorical.py +23 -15
- mindspore/nn/probability/distribution/cauchy.py +5 -6
- mindspore/nn/probability/distribution/distribution.py +3 -3
- mindspore/nn/probability/distribution/exponential.py +4 -4
- mindspore/nn/probability/distribution/gamma.py +10 -10
- mindspore/nn/probability/distribution/geometric.py +8 -8
- mindspore/nn/probability/distribution/gumbel.py +8 -9
- mindspore/nn/probability/distribution/half_normal.py +5 -5
- mindspore/nn/probability/distribution/laplace.py +5 -5
- mindspore/nn/probability/distribution/log_normal.py +12 -11
- mindspore/nn/probability/distribution/logistic.py +8 -8
- mindspore/nn/probability/distribution/normal.py +6 -5
- mindspore/nn/probability/distribution/poisson.py +10 -11
- mindspore/nn/probability/distribution/student_t.py +8 -9
- mindspore/nn/probability/distribution/transformed_distribution.py +5 -5
- mindspore/nn/probability/distribution/uniform.py +11 -11
- mindspore/nn/reinforcement/tensor_array.py +2 -2
- mindspore/nn/sparse/sparse.py +9 -9
- mindspore/nn/wrap/cell_wrapper.py +188 -63
- mindspore/nn/wrap/grad_reducer.py +21 -12
- mindspore/nn/wrap/loss_scale.py +136 -49
- mindspore/numpy/__init__.py +4 -4
- mindspore/numpy/array_creations.py +55 -56
- mindspore/numpy/array_ops.py +134 -35
- mindspore/numpy/logic_ops.py +66 -20
- mindspore/numpy/math_ops.py +142 -139
- mindspore/numpy/utils_const.py +2 -2
- mindspore/offline_debug/convert_async.py +2 -2
- mindspore/ops/_grad_experimental/__init__.py +7 -5
- mindspore/ops/_grad_experimental/grad_array_ops.py +231 -348
- mindspore/ops/{_grad → _grad_experimental}/grad_base.py +1 -33
- mindspore/ops/{_grad → _grad_experimental}/grad_comm_ops.py +25 -13
- mindspore/ops/{_grad/__init__.py → _grad_experimental/grad_debug_ops.py} +15 -7
- mindspore/ops/{_grad → _grad_experimental}/grad_implementations.py +17 -11
- mindspore/ops/_grad_experimental/grad_inner_ops.py +33 -52
- mindspore/ops/_grad_experimental/grad_math_ops.py +151 -1224
- mindspore/ops/_grad_experimental/grad_nn_ops.py +141 -414
- mindspore/ops/{_grad → _grad_experimental}/grad_quant_ops.py +10 -6
- mindspore/ops/_grad_experimental/grad_sparse.py +317 -2
- mindspore/ops/_grad_experimental/grad_sparse_ops.py +3 -13
- mindspore/ops/{_grad → _grad_experimental}/taylor_rule.py +1 -1
- mindspore/ops/_op_impl/_custom_op/dsd_back_impl.py +1 -1
- mindspore/ops/_op_impl/_custom_op/flash_attention/__init__.py +0 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/attention.py +406 -0
- mindspore/{_extends/graph_kernel/expanders/complex/__init__.py → ops/_op_impl/_custom_op/flash_attention/constants.py} +27 -8
- mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_bwd.py +467 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_fwd.py +563 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_impl.py +193 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/tik_ops_utils.py +435 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/__init__.py +0 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/sparse_tiling.py +45 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/strategy.py +67 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/wukong_tiling.py +62 -0
- mindspore/ops/_op_impl/_custom_op/matmul_cube_dense_left_impl.py +2 -2
- mindspore/ops/_op_impl/aicpu/__init__.py +41 -1
- mindspore/ops/_op_impl/aicpu/adaptive_max_pool_2d.py +37 -0
- mindspore/ops/_op_impl/aicpu/bias_add_grad.py +0 -1
- mindspore/ops/_op_impl/aicpu/cast.py +52 -0
- mindspore/ops/_op_impl/aicpu/coalesce.py +2 -0
- mindspore/ops/_op_impl/aicpu/col2im.py +3 -1
- mindspore/ops/_op_impl/aicpu/count_nonzero.py +43 -0
- mindspore/ops/_op_impl/aicpu/dropout_genmask.py +6 -0
- mindspore/ops/_op_impl/aicpu/eps.py +32 -0
- mindspore/ops/_op_impl/aicpu/eye.py +4 -4
- mindspore/ops/_op_impl/aicpu/fft_with_size.py +6 -0
- mindspore/ops/_op_impl/aicpu/fill_diagonal.py +5 -0
- mindspore/ops/_op_impl/aicpu/gamma.py +2 -2
- mindspore/ops/_op_impl/aicpu/im2col.py +3 -5
- mindspore/ops/_op_impl/aicpu/lgamma.py +1 -0
- mindspore/ops/_op_impl/aicpu/log_uniform_candidate_sampler.py +6 -3
- mindspore/ops/_op_impl/aicpu/lu.py +39 -0
- mindspore/ops/_op_impl/aicpu/lu_unpack_grad.py +0 -1
- mindspore/ops/_op_impl/aicpu/masked_scatter.py +1 -0
- mindspore/ops/_op_impl/aicpu/masked_select_grad.py +3 -0
- mindspore/ops/_op_impl/aicpu/matrix_band_part.py +59 -0
- mindspore/ops/_op_impl/aicpu/matrix_power.py +6 -1
- mindspore/ops/_op_impl/aicpu/median.py +1 -0
- mindspore/ops/_op_impl/aicpu/multinomial.py +9 -9
- mindspore/ops/_op_impl/aicpu/not_equal.py +0 -5
- mindspore/ops/_op_impl/aicpu/pad_v3.py +3 -1
- mindspore/ops/_op_impl/aicpu/pad_v3_grad.py +2 -0
- mindspore/ops/_op_impl/aicpu/parameterized_truncated_normal.py +15 -7
- mindspore/ops/_op_impl/aicpu/random_categorical.py +39 -19
- mindspore/ops/_op_impl/aicpu/random_choice_with_mask.py +5 -2
- mindspore/ops/_op_impl/aicpu/random_poisson.py +103 -52
- mindspore/ops/_op_impl/aicpu/random_shuffle.py +17 -15
- mindspore/ops/_op_impl/aicpu/resize_bilinear_grad.py +0 -1
- mindspore/ops/_op_impl/aicpu/resize_nearest_neighbor_v2.py +0 -6
- mindspore/ops/_op_impl/aicpu/resize_nearest_neighbor_v2_grad.py +0 -7
- mindspore/ops/_op_impl/aicpu/scatter_nd.py +2 -0
- mindspore/ops/_op_impl/aicpu/sequence_concat.py +40 -0
- mindspore/ops/_op_impl/aicpu/sequence_stack.py +40 -0
- mindspore/ops/_op_impl/aicpu/{sparseaddmm.py → sparse_addmm.py} +2 -2
- mindspore/ops/_op_impl/aicpu/{sparsesparsemaximum.py → sparse_sparse_maximum.py} +4 -4
- mindspore/ops/_op_impl/aicpu/standard_laplace.py +5 -4
- mindspore/ops/_op_impl/aicpu/standard_normal.py +5 -4
- mindspore/ops/_op_impl/aicpu/truncated_normal.py +9 -7
- mindspore/ops/_op_impl/aicpu/uniform.py +5 -3
- mindspore/ops/_op_impl/aicpu/uniform_candidate_sampler.py +8 -4
- mindspore/ops/_op_impl/aicpu/uniform_int.py +5 -5
- mindspore/ops/_op_impl/aicpu/uniform_real.py +4 -4
- mindspore/ops/_op_impl/aicpu/upsample_nearest_3d.py +14 -6
- mindspore/ops/_op_impl/aicpu/upsample_nearest_3d_grad.py +22 -8
- mindspore/ops/_op_impl/aicpu/upsample_trilinear_3d.py +11 -6
- mindspore/ops/_op_impl/aicpu/upsample_trilinear_3d_grad.py +21 -10
- mindspore/ops/_op_impl/tbe/__init__.py +6 -4
- mindspore/ops/_op_impl/tbe/atomic_addr_clean.py +1 -1
- mindspore/ops/_op_impl/tbe/avg_pool.py +2 -2
- mindspore/ops/_op_impl/tbe/avg_pool_3d.py +3 -3
- mindspore/ops/_op_impl/tbe/avg_pool_3d_grad.py +4 -4
- mindspore/ops/_op_impl/tbe/avg_pool_ds.py +2 -2
- mindspore/ops/_op_impl/tbe/avg_pool_grad.py +3 -3
- mindspore/ops/_op_impl/tbe/avg_pool_grad_vm.py +3 -3
- mindspore/ops/_op_impl/tbe/batch_to_space.py +1 -1
- mindspore/ops/_op_impl/tbe/batch_to_space_nd.py +2 -2
- mindspore/ops/_op_impl/tbe/bn_infer.py +2 -2
- mindspore/ops/_op_impl/tbe/bn_infer_ds.py +3 -2
- mindspore/ops/_op_impl/tbe/broadcast_to.py +1 -1
- mindspore/ops/_op_impl/tbe/depthwise_conv2d.py +3 -3
- mindspore/ops/_op_impl/tbe/expand_dims.py +1 -1
- mindspore/ops/_op_impl/tbe/gather_v2.py +56 -0
- mindspore/ops/_op_impl/tbe/im2col.py +4 -4
- mindspore/ops/_op_impl/tbe/inplace_index_add.py +7 -3
- mindspore/ops/_op_impl/tbe/mem_set.py +38 -0
- mindspore/ops/_op_impl/tbe/scatter_nd_add.py +3 -0
- mindspore/ops/_op_impl/tbe/scatter_nd_d.py +1 -1
- mindspore/ops/_op_impl/tbe/space_to_batch.py +1 -1
- mindspore/ops/_op_impl/tbe/space_to_batch_nd.py +2 -2
- mindspore/ops/_op_impl/tbe/trans_data_ds.py +2 -0
- mindspore/ops/_primitive_cache.py +1 -1
- mindspore/ops/_tracefunc.py +241 -0
- mindspore/ops/_utils/utils.py +10 -2
- mindspore/ops/_vmap/vmap_array_ops.py +5 -3
- mindspore/ops/_vmap/vmap_base.py +5 -4
- mindspore/ops/_vmap/vmap_convolution_ops.py +1 -1
- mindspore/ops/_vmap/vmap_grad_math_ops.py +6 -4
- mindspore/ops/_vmap/vmap_grad_nn_ops.py +11 -6
- mindspore/ops/_vmap/vmap_math_ops.py +5 -2
- mindspore/ops/_vmap/vmap_nn_ops.py +135 -11
- mindspore/ops/arg_dtype_cast.py +54 -0
- mindspore/ops/composite/__init__.py +7 -5
- mindspore/ops/composite/base.py +78 -34
- mindspore/ops/composite/math_ops.py +5 -695
- mindspore/ops/composite/multitype_ops/_compile_utils.py +403 -97
- mindspore/ops/composite/multitype_ops/_constexpr_utils.py +28 -22
- mindspore/ops/composite/multitype_ops/add_impl.py +69 -7
- mindspore/ops/composite/multitype_ops/bitwise_and_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/bitwise_or_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/bitwise_xor_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/div_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/floordiv_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/getitem_impl.py +48 -10
- mindspore/ops/composite/multitype_ops/greater_equal_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/greater_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/left_shift_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/less_equal_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/less_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/logic_not_impl.py +2 -2
- mindspore/ops/composite/multitype_ops/mod_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/mul_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/negative_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/not_in_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/ones_like_impl.py +6 -0
- mindspore/ops/composite/multitype_ops/pow_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/right_shift_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/setitem_impl.py +10 -7
- mindspore/ops/composite/multitype_ops/sub_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/uadd_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/zeros_like_impl.py +9 -0
- mindspore/ops/deprecated.py +304 -0
- mindspore/ops/function/__init__.py +41 -4
- mindspore/ops/function/array_func.py +1108 -467
- mindspore/ops/function/clip_func.py +94 -27
- mindspore/ops/function/debug_func.py +3 -1
- mindspore/ops/function/grad/grad_func.py +82 -73
- mindspore/ops/function/image_func.py +28 -12
- mindspore/ops/function/linalg_func.py +135 -39
- mindspore/ops/function/math_func.py +3779 -894
- mindspore/ops/function/nn_func.py +1584 -657
- mindspore/ops/function/parameter_func.py +13 -3
- mindspore/ops/function/random_func.py +247 -153
- mindspore/ops/function/sparse_func.py +14 -11
- mindspore/ops/function/sparse_unary_func.py +173 -47
- mindspore/ops/function/spectral_func.py +8 -4
- mindspore/ops/function/vmap_func.py +8 -7
- mindspore/ops/functional.py +47 -16
- mindspore/ops/op_info_register.py +346 -86
- mindspore/ops/operations/__init__.py +38 -22
- mindspore/ops/operations/_grad_ops.py +145 -149
- mindspore/ops/operations/_inner_ops.py +298 -56
- mindspore/ops/operations/_ms_kernel.py +3 -3
- mindspore/ops/operations/_quant_ops.py +24 -28
- mindspore/ops/operations/_rl_inner_ops.py +9 -7
- mindspore/ops/operations/_scalar_ops.py +115 -0
- mindspore/ops/operations/_sequence_ops.py +148 -10
- mindspore/ops/operations/_tensor_array.py +1 -1
- mindspore/ops/operations/_thor_ops.py +2 -2
- mindspore/ops/operations/array_ops.py +1239 -561
- mindspore/ops/operations/comm_ops.py +166 -90
- mindspore/ops/operations/control_ops.py +3 -3
- mindspore/ops/operations/custom_ops.py +124 -102
- mindspore/ops/operations/debug_ops.py +24 -11
- mindspore/ops/operations/image_ops.py +86 -71
- mindspore/ops/operations/inner_ops.py +18 -13
- mindspore/ops/operations/linalg_ops.py +30 -11
- mindspore/ops/operations/math_ops.py +1730 -435
- mindspore/ops/operations/nn_ops.py +1953 -943
- mindspore/ops/operations/other_ops.py +65 -43
- mindspore/ops/operations/random_ops.py +258 -98
- mindspore/ops/operations/rl_ops.py +4 -36
- mindspore/ops/operations/sparse_ops.py +38 -33
- mindspore/ops/operations/spectral_ops.py +8 -4
- mindspore/ops/primitive.py +66 -44
- mindspore/ops/signature.py +5 -5
- mindspore/parallel/_auto_parallel_context.py +80 -19
- mindspore/parallel/_cost_model_context.py +42 -0
- mindspore/parallel/_offload_context.py +162 -72
- mindspore/parallel/_parallel_serialization.py +2 -2
- mindspore/parallel/_ps_context.py +16 -4
- mindspore/parallel/_recovery_context.py +2 -1
- mindspore/parallel/_tensor.py +15 -13
- mindspore/parallel/_transformer/layers.py +8 -6
- mindspore/parallel/_transformer/loss.py +1 -0
- mindspore/parallel/_transformer/moe.py +7 -7
- mindspore/parallel/_transformer/op_parallel_config.py +12 -1
- mindspore/parallel/_transformer/transformer.py +34 -14
- mindspore/parallel/_utils.py +36 -14
- mindspore/parallel/algo_parameter_config.py +114 -20
- mindspore/parallel/checkpoint_transform.py +16 -18
- mindspore/parallel/shard.py +16 -13
- mindspore/profiler/__init__.py +1 -1
- mindspore/profiler/common/struct_type.py +3 -3
- mindspore/profiler/common/util.py +3 -2
- mindspore/profiler/envprofiling.py +11 -4
- mindspore/profiler/parser/aicpu_data_parser.py +5 -3
- mindspore/profiler/parser/ascend_flops_generator.py +94 -0
- mindspore/profiler/parser/ascend_fpbp_generator.py +76 -0
- mindspore/profiler/parser/ascend_hccl_generator.py +288 -0
- mindspore/profiler/parser/ascend_msprof_exporter.py +213 -0
- mindspore/profiler/parser/ascend_msprof_generator.py +199 -0
- mindspore/profiler/parser/ascend_op_generator.py +276 -0
- mindspore/profiler/parser/ascend_steptrace_generator.py +94 -0
- mindspore/profiler/parser/ascend_timeline_generator.py +110 -54
- mindspore/profiler/parser/base_timeline_generator.py +11 -7
- mindspore/profiler/parser/cpu_gpu_timeline_generator.py +45 -46
- mindspore/profiler/parser/flops_parser.py +15 -11
- mindspore/profiler/parser/framework_parser.py +92 -73
- mindspore/profiler/parser/hccl_parser.py +16 -12
- mindspore/profiler/parser/integrator.py +22 -11
- mindspore/profiler/parser/memory_usage_parser.py +36 -11
- mindspore/profiler/parser/minddata_analyzer.py +12 -14
- mindspore/profiler/parser/minddata_pipeline_parser.py +1 -1
- mindspore/profiler/parser/msadvisor_parser.py +8 -4
- mindspore/profiler/parser/op_intermediate_parser.py +5 -2
- mindspore/profiler/parser/optime_parser.py +1 -1
- mindspore/profiler/parser/profiler_info.py +4 -5
- mindspore/profiler/parser/step_trace_parser.py +11 -14
- mindspore/profiler/profiling.py +678 -377
- mindspore/rewrite/api/node.py +211 -54
- mindspore/rewrite/api/node_type.py +5 -0
- mindspore/rewrite/api/pattern_engine.py +22 -23
- mindspore/rewrite/api/scoped_value.py +20 -17
- mindspore/rewrite/api/symbol_tree.py +252 -106
- mindspore/rewrite/api/tree_node_helper.py +3 -0
- mindspore/rewrite/ast_helpers/__init__.py +2 -1
- mindspore/rewrite/ast_helpers/ast_finder.py +129 -0
- mindspore/rewrite/ast_helpers/ast_modifier.py +116 -104
- mindspore/rewrite/ast_transformers/flatten_recursive_stmt.py +97 -46
- mindspore/rewrite/common/rewrite_elog.py +5 -1
- mindspore/rewrite/namer.py +51 -51
- mindspore/rewrite/namespace.py +14 -5
- mindspore/{ops/bprop_mindir → rewrite/node}/__init__.py +9 -4
- mindspore/rewrite/node/call_function.py +79 -0
- mindspore/rewrite/node/cell_container.py +135 -0
- mindspore/rewrite/node/control_flow.py +88 -0
- mindspore/rewrite/{node.py → node/node.py} +313 -247
- mindspore/rewrite/node/node_manager.py +254 -0
- mindspore/rewrite/node/node_topological_manager.py +243 -0
- mindspore/rewrite/parsers/arguments_parser.py +22 -21
- mindspore/rewrite/parsers/assign_parser.py +225 -239
- mindspore/rewrite/parsers/attribute_parser.py +9 -7
- mindspore/rewrite/parsers/class_def_parser.py +179 -218
- mindspore/rewrite/parsers/constant_parser.py +9 -6
- mindspore/rewrite/parsers/container_parser.py +9 -7
- mindspore/rewrite/parsers/for_parser.py +36 -15
- mindspore/rewrite/parsers/function_def_parser.py +23 -20
- mindspore/rewrite/parsers/if_parser.py +28 -24
- mindspore/rewrite/parsers/module_parser.py +202 -25
- mindspore/rewrite/{parser.py → parsers/parser.py} +4 -2
- mindspore/rewrite/{parser_register.py → parsers/parser_register.py} +1 -1
- mindspore/rewrite/parsers/return_parser.py +6 -6
- mindspore/rewrite/sparsify/sparse_transformer.py +12 -3
- mindspore/rewrite/sparsify/sparsify.py +4 -1
- mindspore/rewrite/sparsify/utils.py +11 -5
- mindspore/rewrite/symbol_tree.py +577 -732
- mindspore/rewrite/symbol_tree_builder.py +9 -175
- mindspore/rewrite/symbol_tree_dumper.py +2 -2
- mindspore/run_check/_check_version.py +46 -39
- mindspore/run_check/run_check.py +3 -2
- mindspore/{scipy/sparse → safeguard}/__init__.py +4 -5
- mindspore/safeguard/rewrite_obfuscation.py +517 -0
- mindspore/scipy/__init__.py +1 -1
- mindspore/scipy/linalg.py +67 -61
- mindspore/scipy/ops.py +5 -41
- mindspore/scipy/ops_grad.py +3 -2
- mindspore/scipy/ops_wrapper.py +5 -5
- mindspore/scipy/optimize/line_search.py +8 -8
- mindspore/scipy/optimize/linear_sum_assignment.py +4 -4
- mindspore/scipy/optimize/minimize.py +16 -12
- mindspore/scipy/utils.py +1 -52
- mindspore/scipy/utils_const.py +4 -4
- mindspore/train/__init__.py +4 -4
- mindspore/train/_utils.py +13 -5
- mindspore/train/amp.py +410 -148
- mindspore/train/anf_ir_pb2.py +16 -4
- mindspore/train/callback/_backup_and_restore.py +8 -11
- mindspore/train/callback/_callback.py +80 -3
- mindspore/train/callback/_checkpoint.py +82 -51
- mindspore/train/callback/_early_stop.py +12 -15
- mindspore/train/callback/_history.py +1 -1
- mindspore/train/callback/_lambda_callback.py +13 -13
- mindspore/train/callback/_landscape.py +21 -17
- mindspore/train/callback/_loss_monitor.py +9 -10
- mindspore/train/callback/_on_request_exit.py +16 -33
- mindspore/train/callback/_reduce_lr_on_plateau.py +21 -24
- mindspore/train/callback/_summary_collector.py +44 -30
- mindspore/train/callback/_time_monitor.py +62 -12
- mindspore/train/data_sink.py +10 -16
- mindspore/train/dataset_helper.py +154 -86
- mindspore/train/loss_scale_manager.py +14 -9
- mindspore/train/metrics/__init__.py +10 -2
- mindspore/train/metrics/accuracy.py +1 -1
- mindspore/train/metrics/auc.py +1 -1
- mindspore/train/metrics/bleu_score.py +2 -2
- mindspore/train/metrics/confusion_matrix.py +14 -14
- mindspore/train/metrics/cosine_similarity.py +3 -3
- mindspore/train/metrics/dice.py +1 -1
- mindspore/train/metrics/fbeta.py +1 -1
- mindspore/train/metrics/hausdorff_distance.py +8 -6
- mindspore/train/metrics/mean_surface_distance.py +5 -4
- mindspore/train/metrics/metric.py +49 -17
- mindspore/train/metrics/occlusion_sensitivity.py +4 -4
- mindspore/train/metrics/perplexity.py +1 -1
- mindspore/train/metrics/precision.py +2 -2
- mindspore/train/metrics/recall.py +2 -3
- mindspore/train/metrics/roc.py +7 -7
- mindspore/train/metrics/root_mean_square_surface_distance.py +5 -4
- mindspore/train/metrics/topk.py +7 -4
- mindspore/train/mind_ir_pb2.py +193 -48
- mindspore/train/model.py +377 -133
- mindspore/train/serialization.py +697 -245
- mindspore/train/summary/_summary_adapter.py +5 -2
- mindspore/train/summary/_writer_pool.py +4 -3
- mindspore/train/summary/summary_record.py +25 -23
- mindspore/train/train_thor/convert_utils.py +39 -23
- mindspore/train/train_thor/dataset_helper.py +4 -3
- mindspore/train/train_thor/model_thor.py +8 -8
- mindspore/version.py +1 -1
- {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/METADATA +7 -8
- {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/RECORD +633 -804
- {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/entry_points.txt +0 -1
- mindspore/_akg/akg/tvm/contrib/debugger/__init__.py +0 -16
- mindspore/_akg/akg/tvm/contrib/debugger/debug_result.py +0 -274
- mindspore/_akg/akg/tvm/contrib/debugger/debug_runtime.py +0 -259
- mindspore/_akg/akg/tvm/contrib/peak.py +0 -341
- mindspore/_akg/akg/tvm/contrib/rpc.py +0 -25
- mindspore/_akg/akg/tvm/contrib/xcode.py +0 -257
- mindspore/_akg/akg/tvm/exec/__init__.py +0 -17
- mindspore/_akg/akg/tvm/exec/autotvm_log_editor.py +0 -60
- mindspore/_akg/akg/tvm/exec/measure_peak.py +0 -48
- mindspore/_akg/akg/tvm/exec/query_rpc_tracker.py +0 -48
- mindspore/_akg/akg/tvm/exec/rpc_proxy.py +0 -98
- mindspore/_akg/akg/tvm/exec/rpc_server.py +0 -88
- mindspore/_akg/akg/tvm/exec/rpc_tracker.py +0 -62
- mindspore/_akg/akg/tvm/rpc/__init__.py +0 -29
- mindspore/_akg/akg/tvm/rpc/base.py +0 -182
- mindspore/_akg/akg/tvm/rpc/client.py +0 -436
- mindspore/_akg/akg/tvm/rpc/proxy.py +0 -595
- mindspore/_akg/akg/tvm/rpc/server.py +0 -413
- mindspore/_akg/akg/tvm/rpc/tornado_util.py +0 -121
- mindspore/_akg/akg/tvm/rpc/tracker.py +0 -431
- mindspore/_extends/graph_kernel/expander.py +0 -80
- mindspore/_extends/graph_kernel/expanders/__init__.py +0 -57
- mindspore/_extends/graph_kernel/expanders/_utils.py +0 -269
- mindspore/_extends/graph_kernel/expanders/addn.py +0 -33
- mindspore/_extends/graph_kernel/expanders/batchnorm.py +0 -152
- mindspore/_extends/graph_kernel/expanders/batchnorm_grad.py +0 -105
- mindspore/_extends/graph_kernel/expanders/bias_add_grad.py +0 -49
- mindspore/_extends/graph_kernel/expanders/clip_by_norm_no_div_sum.py +0 -33
- mindspore/_extends/graph_kernel/expanders/complex/abs.py +0 -30
- mindspore/_extends/graph_kernel/expanders/complex/add.py +0 -44
- mindspore/_extends/graph_kernel/expanders/complex/div.py +0 -62
- mindspore/_extends/graph_kernel/expanders/complex/mul.py +0 -52
- mindspore/_extends/graph_kernel/expanders/complex/real_div.py +0 -62
- mindspore/_extends/graph_kernel/expanders/complex/sub.py +0 -45
- mindspore/_extends/graph_kernel/expanders/conv2d.py +0 -200
- mindspore/_extends/graph_kernel/expanders/dropout_grad.py +0 -30
- mindspore/_extends/graph_kernel/expanders/equal_count.py +0 -50
- mindspore/_extends/graph_kernel/expanders/erfc.py +0 -35
- mindspore/_extends/graph_kernel/expanders/expand_dims.py +0 -50
- mindspore/_extends/graph_kernel/expanders/fused_adam.py +0 -44
- mindspore/_extends/graph_kernel/expanders/fused_adam_weight_decay.py +0 -47
- mindspore/_extends/graph_kernel/expanders/fused_mul_add.py +0 -28
- mindspore/_extends/graph_kernel/expanders/gather.py +0 -43
- mindspore/_extends/graph_kernel/expanders/gelu_grad.py +0 -70
- mindspore/_extends/graph_kernel/expanders/gkdropout.py +0 -40
- mindspore/_extends/graph_kernel/expanders/identity.py +0 -25
- mindspore/_extends/graph_kernel/expanders/layernorm.py +0 -93
- mindspore/_extends/graph_kernel/expanders/layernorm_grad.py +0 -113
- mindspore/_extends/graph_kernel/expanders/logsoftmax.py +0 -46
- mindspore/_extends/graph_kernel/expanders/logsoftmax_grad.py +0 -36
- mindspore/_extends/graph_kernel/expanders/matmul.py +0 -80
- mindspore/_extends/graph_kernel/expanders/maximum_grad.py +0 -59
- mindspore/_extends/graph_kernel/expanders/minimum_grad.py +0 -80
- mindspore/_extends/graph_kernel/expanders/oneslike.py +0 -26
- mindspore/_extends/graph_kernel/expanders/reduce_mean.py +0 -43
- mindspore/_extends/graph_kernel/expanders/relu_grad.py +0 -32
- mindspore/_extends/graph_kernel/expanders/sigmoid_cross_entropy_with_logits.py +0 -41
- mindspore/_extends/graph_kernel/expanders/sigmoid_cross_entropy_with_logits_grad.py +0 -35
- mindspore/_extends/graph_kernel/expanders/sigmoid_grad.py +0 -31
- mindspore/_extends/graph_kernel/expanders/slice.py +0 -35
- mindspore/_extends/graph_kernel/expanders/softmax_cross_entropy_with_logits.py +0 -42
- mindspore/_extends/graph_kernel/expanders/softmax_grad_ext.py +0 -41
- mindspore/_extends/graph_kernel/expanders/softsign.py +0 -28
- mindspore/_extends/graph_kernel/expanders/sqrt_grad.py +0 -29
- mindspore/_extends/graph_kernel/expanders/square_sum_all.py +0 -44
- mindspore/_extends/graph_kernel/expanders/square_sum_v1.py +0 -37
- mindspore/_extends/graph_kernel/expanders/squared_difference.py +0 -43
- mindspore/_extends/graph_kernel/expanders/tanh_grad.py +0 -31
- mindspore/_extends/graph_kernel/expanders/tile.py +0 -54
- mindspore/_extends/graph_kernel/model/op_infer.py +0 -506
- mindspore/_extends/parse/jit_fallback_modules.py +0 -51
- mindspore/dataset/datapreprocess/preprocess_imagenet_validate_dataset.py +0 -54
- mindspore/dataset/engine/graphdata.py +0 -1586
- mindspore/include/api/net.h +0 -142
- mindspore/ops/_grad/grad_array_ops.py +0 -1347
- mindspore/ops/_grad/grad_clip_ops.py +0 -84
- mindspore/ops/_grad/grad_debug_ops.py +0 -68
- mindspore/ops/_grad/grad_inner_ops.py +0 -235
- mindspore/ops/_grad/grad_math_ops.py +0 -1684
- mindspore/ops/_grad/grad_nn_ops.py +0 -1529
- mindspore/ops/_grad/grad_other_ops.py +0 -89
- mindspore/ops/_grad/grad_sequence_ops.py +0 -296
- mindspore/ops/_grad/grad_sparse.py +0 -323
- mindspore/ops/_grad_experimental/grad_image_ops.py +0 -249
- mindspore/ops/_grad_experimental/grad_linalg_ops.py +0 -195
- mindspore/ops/_grad_experimental/grad_scalar_ops.py +0 -112
- mindspore/ops/bprop_mindir/AdaptiveAvgPool2D_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/AdaptiveMaxPool2D_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/ApproximateEqual_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/Argmax_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/Argmin_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/AssignSub_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/Assign_bprop.mindir +0 -17
- mindspore/ops/bprop_mindir/AvgPool3D_bprop.mindir +0 -150
- mindspore/ops/bprop_mindir/AvgPool_bprop.mindir +0 -66
- mindspore/ops/bprop_mindir/BCEWithLogitsLoss_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/BNTrainingReduce_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/BatchNormGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/BatchToSpaceND_bprop.mindir +0 -28
- mindspore/ops/bprop_mindir/BiasAddGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/BinaryCrossEntropy_bprop.mindir +0 -33
- mindspore/ops/bprop_mindir/BroadcastTo_bprop.mindir +0 -306
- mindspore/ops/bprop_mindir/Broadcast_bprop.mindir +0 -13
- mindspore/ops/bprop_mindir/CTCLoss_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Concat_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Conv2DBackpropFilter_bprop.mindir +0 -240
- mindspore/ops/bprop_mindir/Conv2DBackpropInput_bprop.mindir +0 -247
- mindspore/ops/bprop_mindir/Conv2DTranspose_bprop.mindir +0 -247
- mindspore/ops/bprop_mindir/Conv3DTranspose_bprop.mindir +0 -315
- mindspore/ops/bprop_mindir/Conv3D_bprop.mindir +0 -278
- mindspore/ops/bprop_mindir/DType_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/DeformableOffsets_bprop.mindir +0 -58
- mindspore/ops/bprop_mindir/Depend_bprop.mindir +0 -13
- mindspore/ops/bprop_mindir/DepthToSpace_bprop.mindir +0 -23
- mindspore/ops/bprop_mindir/DepthwiseConv2dNative_bprop.mindir +0 -138
- mindspore/ops/bprop_mindir/DiagPart_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/Dropout2D_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Dropout3D_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/DropoutDoMask_bprop.mindir +0 -25
- mindspore/ops/bprop_mindir/DropoutGenMask_bprop.mindir +0 -18
- mindspore/ops/bprop_mindir/DropoutGrad_bprop.mindir +0 -27
- mindspore/ops/bprop_mindir/Dropout_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/DynamicGRUV2_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/DynamicRNN_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/DynamicShape_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/Elu_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/EmbeddingLookup_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Equal_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/ExpandDims_bprop.mindir +0 -58
- mindspore/ops/bprop_mindir/FastGeLU_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/Flatten_bprop.mindir +0 -54
- mindspore/ops/bprop_mindir/FloorDiv_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/GatherD_bprop.mindir +0 -26
- mindspore/ops/bprop_mindir/GatherNd_bprop.mindir +0 -57
- mindspore/ops/bprop_mindir/Gather_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/GreaterEqual_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/Greater_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/HSigmoid_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/HSwish_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/IOU_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/InstanceNorm_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/IsFinite_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/IsInf_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/IsNan_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/KLDivLoss_bprop.mindir +0 -126
- mindspore/ops/bprop_mindir/L2Loss_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/L2Normalize_bprop.mindir +0 -30
- mindspore/ops/bprop_mindir/LRN_bprop.mindir +0 -43
- mindspore/ops/bprop_mindir/LayerNormGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/LessEqual_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/Less_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/LinSpace_bprop.mindir +0 -23
- mindspore/ops/bprop_mindir/Load_bprop.mindir +0 -13
- mindspore/ops/bprop_mindir/LogSoftmax_bprop.mindir +0 -23
- mindspore/ops/bprop_mindir/LogicalAnd_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/LogicalNot_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/MaskedSelect_bprop.mindir +0 -21
- mindspore/ops/bprop_mindir/MaxPool3DGradGrad_bprop.mindir +0 -74
- mindspore/ops/bprop_mindir/MaxPool3DGrad_bprop.mindir +0 -74
- mindspore/ops/bprop_mindir/MaxPool3D_bprop.mindir +0 -75
- mindspore/ops/bprop_mindir/MaxPoolGradGrad_bprop.mindir +0 -65
- mindspore/ops/bprop_mindir/MaxPoolWithArgmax_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Maximum_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Minimum_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/MirrorPad_bprop.mindir +0 -27
- mindspore/ops/bprop_mindir/Mish_bprop.mindir +0 -35
- mindspore/ops/bprop_mindir/MulNoNan_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/NLLLoss_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/NonZero_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/NotEqual_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/OneHot_bprop.mindir +0 -26
- mindspore/ops/bprop_mindir/OnesLike_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/PReLU_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Pad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Padding_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/RNNTLoss_bprop.mindir +0 -29
- mindspore/ops/bprop_mindir/ROIAlign_bprop.mindir +0 -82
- mindspore/ops/bprop_mindir/Range_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/Rank_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/ReLU6_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/ReLUV2_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/ReduceAll_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/ReduceAny_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/ReluGrad_bprop.mindir +0 -20
- mindspore/ops/bprop_mindir/Reshape_bprop.mindir +0 -60
- mindspore/ops/bprop_mindir/ResizeBilinear_bprop.mindir +0 -29
- mindspore/ops/bprop_mindir/ResizeNearestNeighbor_bprop.mindir +0 -89
- mindspore/ops/bprop_mindir/ReverseSequence_bprop.mindir +0 -52
- mindspore/ops/bprop_mindir/ReverseV2_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/Round_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/ScatterMax_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/ScatterMin_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/ScatterNdUpdate_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/ScatterNd_bprop.mindir +0 -24
- mindspore/ops/bprop_mindir/ScatterNonAliasingAdd_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/ScatterUpdate_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/SeLU_bprop.mindir +0 -21
- mindspore/ops/bprop_mindir/Select_bprop.mindir +0 -31
- mindspore/ops/bprop_mindir/Shape_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/SigmoidCrossEntropyWithLogits_bprop.mindir +0 -21
- mindspore/ops/bprop_mindir/SigmoidGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Sigmoid_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/Sign_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/Slice_bprop.mindir +0 -26
- mindspore/ops/bprop_mindir/SmoothL1Loss_bprop.mindir +0 -36
- mindspore/ops/bprop_mindir/SoftmaxCrossEntropyWithLogits_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Softplus_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/Softsign_bprop.mindir +0 -33
- mindspore/ops/bprop_mindir/Sort_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/SpaceToBatchND_bprop.mindir +0 -28
- mindspore/ops/bprop_mindir/SpaceToDepth_bprop.mindir +0 -23
- mindspore/ops/bprop_mindir/SparseGatherV2_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/SparseSoftmaxCrossEntropyWithLogits_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Split_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/Squeeze_bprop.mindir +0 -54
- mindspore/ops/bprop_mindir/StridedSliceGrad_bprop.mindir +0 -95
- mindspore/ops/bprop_mindir/StridedSlice_bprop.mindir +0 -98
- mindspore/ops/bprop_mindir/Switch_bprop.mindir +0 -29
- mindspore/ops/bprop_mindir/TanhGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Tanh_bprop.mindir +0 -66
- mindspore/ops/bprop_mindir/TensorScatterAdd_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/TensorScatterUpdate_bprop.mindir +0 -29
- mindspore/ops/bprop_mindir/TensorShape_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/Tile_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/TopK_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/TransShape_bprop.mindir +0 -23
- mindspore/ops/bprop_mindir/TruncateDiv_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/TupleGetItem_bprop.mindir +0 -20
- mindspore/ops/bprop_mindir/Unique_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/Unstack_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/UpsampleNearest3D_bprop.mindir +0 -32
- mindspore/ops/bprop_mindir/UpsampleTrilinear3D_bprop.mindir +0 -38
- mindspore/ops/bprop_mindir/ZerosLike_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/generate_mindir.py +0 -114
- mindspore/rewrite/node_visitor.py +0 -44
- mindspore/rewrite/topological_manager.py +0 -203
- mindspore/scipy/sparse/linalg.py +0 -192
- {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/WHEEL +0 -0
- {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2019-
|
|
1
|
+
# Copyright 2019-2023 Huawei Technologies Co., Ltd
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -44,25 +44,27 @@ class CMUArcticDataset(MappableDataset, AudioBaseDataset):
|
|
|
44
44
|
|
|
45
45
|
Args:
|
|
46
46
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
47
|
-
name (str, optional): Part of this dataset, can be 'aew'
|
|
48
|
-
'
|
|
49
|
-
|
|
47
|
+
name (str, optional): Part of this dataset, can be ``'aew'``, ``'ahw'``, ``'aup'``,
|
|
48
|
+
``'awb'``, ``'axb'``, ``'bdl'``, ``'clb'``, ``'eey'``, ``'fem'``, ``'gka'``, ``'jmk'``,
|
|
49
|
+
``'ksp'``, ``'ljm'``, ``'lnh'``, ``'rms'``, ``'rxr'``, ``'slp'`` or ``'slt'``.
|
|
50
|
+
Default: ``None``, means ``'aew'``.
|
|
50
51
|
num_samples (int, optional): The number of audio to be included in the dataset.
|
|
51
|
-
Default: None
|
|
52
|
+
Default: ``None``, will read all audio.
|
|
52
53
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
53
|
-
Default: None
|
|
54
|
-
by
|
|
54
|
+
Default: ``None``, will use global default workers(8), it can be set
|
|
55
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
55
56
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
56
|
-
Default: None
|
|
57
|
+
Default: ``None``, expected order behavior shown in the table below.
|
|
57
58
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
58
|
-
dataset. Default: None
|
|
59
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into.
|
|
60
|
-
When this argument is specified, `num_samples`
|
|
61
|
-
|
|
59
|
+
dataset. Default: ``None``, expected order behavior shown in the table below.
|
|
60
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into.
|
|
61
|
+
Default: ``None``, no dividing. When this argument is specified, `num_samples`
|
|
62
|
+
reflects the max sample number of per shard.
|
|
63
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None``, will use ``0``. This
|
|
62
64
|
argument can only be specified when `num_shards` is also specified.
|
|
63
65
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
64
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
65
|
-
Default: None
|
|
66
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
67
|
+
Default: ``None``, which means no cache is used.
|
|
66
68
|
|
|
67
69
|
Raises:
|
|
68
70
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -73,38 +75,19 @@ class CMUArcticDataset(MappableDataset, AudioBaseDataset):
|
|
|
73
75
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
74
76
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
75
77
|
|
|
78
|
+
Tutorial Examples:
|
|
79
|
+
- `Load & Process Data With Dataset Pipeline
|
|
80
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
81
|
+
|
|
76
82
|
Note:
|
|
77
|
-
- Not support
|
|
78
|
-
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
..
|
|
82
|
-
:widths: 25 25 50
|
|
83
|
-
:header-rows: 1
|
|
84
|
-
|
|
85
|
-
* - Parameter `sampler`
|
|
86
|
-
- Parameter `shuffle`
|
|
87
|
-
- Expected Order Behavior
|
|
88
|
-
* - None
|
|
89
|
-
- None
|
|
90
|
-
- random order
|
|
91
|
-
* - None
|
|
92
|
-
- True
|
|
93
|
-
- random order
|
|
94
|
-
* - None
|
|
95
|
-
- False
|
|
96
|
-
- sequential order
|
|
97
|
-
* - Sampler object
|
|
98
|
-
- None
|
|
99
|
-
- order defined by sampler
|
|
100
|
-
* - Sampler object
|
|
101
|
-
- True
|
|
102
|
-
- not allowed
|
|
103
|
-
* - Sampler object
|
|
104
|
-
- False
|
|
105
|
-
- not allowed
|
|
83
|
+
- Not support :class:`mindspore.dataset.PKSampler` for `sampler` parameter yet.
|
|
84
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
85
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
86
|
+
|
|
87
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
106
88
|
|
|
107
89
|
Examples:
|
|
90
|
+
>>> import mindspore.dataset as ds
|
|
108
91
|
>>> cmu_arctic_dataset_directory = "/path/to/cmu_arctic_dataset_directory"
|
|
109
92
|
>>>
|
|
110
93
|
>>> # 1) Read 500 samples (audio files) in cmu_arctic_dataset_directory
|
|
@@ -182,23 +165,23 @@ class GTZANDataset(MappableDataset, AudioBaseDataset):
|
|
|
182
165
|
Args:
|
|
183
166
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
184
167
|
usage (str, optional): Usage of this dataset, can be 'train', 'valid', 'test' or 'all'.
|
|
185
|
-
Default: None, will read all samples.
|
|
168
|
+
Default: ``None`` , will read all samples.
|
|
186
169
|
num_samples (int, optional): The number of audio to be included in the dataset.
|
|
187
|
-
Default: None, will read all audio.
|
|
170
|
+
Default: ``None`` , will read all audio.
|
|
188
171
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
189
|
-
Default: None, will use global default workers(8), it can be set
|
|
190
|
-
by
|
|
172
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
173
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
191
174
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
192
|
-
Default: None, expected order behavior shown in the table below.
|
|
175
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
193
176
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
194
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
195
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
177
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
178
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
196
179
|
When this argument is specified, `num_samples` reflects the max sample number of per shard.
|
|
197
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
180
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
198
181
|
argument can only be specified when `num_shards` is also specified.
|
|
199
182
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
200
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
201
|
-
Default: None, which means no cache is used.
|
|
183
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
184
|
+
Default: ``None`` , which means no cache is used.
|
|
202
185
|
|
|
203
186
|
Raises:
|
|
204
187
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -209,38 +192,19 @@ class GTZANDataset(MappableDataset, AudioBaseDataset):
|
|
|
209
192
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
210
193
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
211
194
|
|
|
195
|
+
Tutorial Examples:
|
|
196
|
+
- `Load & Process Data With Dataset Pipeline
|
|
197
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
198
|
+
|
|
212
199
|
Note:
|
|
213
|
-
- Not support
|
|
214
|
-
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
..
|
|
218
|
-
:widths: 25 25 50
|
|
219
|
-
:header-rows: 1
|
|
220
|
-
|
|
221
|
-
* - Parameter `sampler`
|
|
222
|
-
- Parameter `shuffle`
|
|
223
|
-
- Expected Order Behavior
|
|
224
|
-
* - None
|
|
225
|
-
- None
|
|
226
|
-
- random order
|
|
227
|
-
* - None
|
|
228
|
-
- True
|
|
229
|
-
- random order
|
|
230
|
-
* - None
|
|
231
|
-
- False
|
|
232
|
-
- sequential order
|
|
233
|
-
* - Sampler object
|
|
234
|
-
- None
|
|
235
|
-
- order defined by sampler
|
|
236
|
-
* - Sampler object
|
|
237
|
-
- True
|
|
238
|
-
- not allowed
|
|
239
|
-
* - Sampler object
|
|
240
|
-
- False
|
|
241
|
-
- not allowed
|
|
200
|
+
- Not support :class:`mindspore.dataset.PKSampler` for `sampler` parameter yet.
|
|
201
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
202
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
203
|
+
|
|
204
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
242
205
|
|
|
243
206
|
Examples:
|
|
207
|
+
>>> import mindspore.dataset as ds
|
|
244
208
|
>>> gtzan_dataset_directory = "/path/to/gtzan_dataset_directory"
|
|
245
209
|
>>>
|
|
246
210
|
>>> # 1) Read 500 samples (audio files) in gtzan_dataset_directory
|
|
@@ -317,24 +281,25 @@ class LibriTTSDataset(MappableDataset, AudioBaseDataset):
|
|
|
317
281
|
|
|
318
282
|
Args:
|
|
319
283
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
320
|
-
usage (str, optional): Part of this dataset, can be 'dev-clean'
|
|
321
|
-
'train-clean-100'
|
|
284
|
+
usage (str, optional): Part of this dataset, can be ``'dev-clean'``, ``'dev-other'``, ``'test-clean'``,
|
|
285
|
+
``'test-other'``, ``'train-clean-100'``, ``'train-clean-360'``, ``'train-other-500'``, or ``'all'``.
|
|
286
|
+
Default: ``None`` , means ``'all'``.
|
|
322
287
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
323
|
-
Default: None, will read all audio.
|
|
288
|
+
Default: ``None`` , will read all audio.
|
|
324
289
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
325
|
-
Default: None, will use global default workers(8), it can be set
|
|
326
|
-
by
|
|
290
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
291
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
327
292
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
328
|
-
Default: None, expected order behavior shown in the table below.
|
|
293
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
329
294
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
330
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
331
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
295
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
296
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
332
297
|
When this argument is specified, `num_samples` reflects the max sample number of per shard.
|
|
333
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
298
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
334
299
|
argument can only be specified when `num_shards` is also specified.
|
|
335
300
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
336
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
337
|
-
Default: None, which means no cache is used.
|
|
301
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
302
|
+
Default: ``None`` , which means no cache is used.
|
|
338
303
|
|
|
339
304
|
Raises:
|
|
340
305
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -345,38 +310,19 @@ class LibriTTSDataset(MappableDataset, AudioBaseDataset):
|
|
|
345
310
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
346
311
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
347
312
|
|
|
313
|
+
Tutorial Examples:
|
|
314
|
+
- `Load & Process Data With Dataset Pipeline
|
|
315
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
316
|
+
|
|
348
317
|
Note:
|
|
349
|
-
- Not support
|
|
350
|
-
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
..
|
|
354
|
-
:widths: 25 25 50
|
|
355
|
-
:header-rows: 1
|
|
356
|
-
|
|
357
|
-
* - Parameter `sampler`
|
|
358
|
-
- Parameter `shuffle`
|
|
359
|
-
- Expected Order Behavior
|
|
360
|
-
* - None
|
|
361
|
-
- None
|
|
362
|
-
- random order
|
|
363
|
-
* - None
|
|
364
|
-
- True
|
|
365
|
-
- random order
|
|
366
|
-
* - None
|
|
367
|
-
- False
|
|
368
|
-
- sequential order
|
|
369
|
-
* - Sampler object
|
|
370
|
-
- None
|
|
371
|
-
- order defined by sampler
|
|
372
|
-
* - Sampler object
|
|
373
|
-
- True
|
|
374
|
-
- not allowed
|
|
375
|
-
* - Sampler object
|
|
376
|
-
- False
|
|
377
|
-
- not allowed
|
|
318
|
+
- Not support :class:`mindspore.dataset.PKSampler` for `sampler` parameter yet.
|
|
319
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
320
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
321
|
+
|
|
322
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
378
323
|
|
|
379
324
|
Examples:
|
|
325
|
+
>>> import mindspore.dataset as ds
|
|
380
326
|
>>> libri_tts_dataset_dir = "/path/to/libri_tts_dataset_directory"
|
|
381
327
|
>>>
|
|
382
328
|
>>> # 1) Read 500 samples (audio files) in libri_tts_dataset_directory
|
|
@@ -465,22 +411,22 @@ class LJSpeechDataset(MappableDataset, AudioBaseDataset):
|
|
|
465
411
|
Args:
|
|
466
412
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
467
413
|
num_samples (int, optional): The number of audios to be included in the dataset.
|
|
468
|
-
Default: None, all audios.
|
|
414
|
+
Default: ``None`` , all audios.
|
|
469
415
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
470
|
-
Default: None, will use global default workers(8), it can be set
|
|
471
|
-
by
|
|
472
|
-
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None, expected
|
|
416
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
417
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
418
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected
|
|
473
419
|
order behavior shown in the table below.
|
|
474
420
|
sampler (Sampler, optional): Object used to choose samples from the dataset.
|
|
475
|
-
Default: None, expected order behavior shown in the table below.
|
|
421
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
476
422
|
num_shards (int, optional): Number of shards that the dataset will be divided into.
|
|
477
|
-
Default: None. When this argument is specified, `num_samples` reflects
|
|
423
|
+
Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
478
424
|
the maximum sample number of per shard.
|
|
479
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
425
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
480
426
|
argument can only be specified when `num_shards` is also specified.
|
|
481
427
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
482
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
483
|
-
Default: None, which means no cache is used.
|
|
428
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
429
|
+
Default: ``None`` , which means no cache is used.
|
|
484
430
|
|
|
485
431
|
Raises:
|
|
486
432
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -491,37 +437,18 @@ class LJSpeechDataset(MappableDataset, AudioBaseDataset):
|
|
|
491
437
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
492
438
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
493
439
|
|
|
440
|
+
Tutorial Examples:
|
|
441
|
+
- `Load & Process Data With Dataset Pipeline
|
|
442
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
443
|
+
|
|
494
444
|
Note:
|
|
495
|
-
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
..
|
|
499
|
-
:widths: 25 25 50
|
|
500
|
-
:header-rows: 1
|
|
501
|
-
|
|
502
|
-
* - Parameter `sampler`
|
|
503
|
-
- Parameter `shuffle`
|
|
504
|
-
- Expected Order Behavior
|
|
505
|
-
* - None
|
|
506
|
-
- None
|
|
507
|
-
- random order
|
|
508
|
-
* - None
|
|
509
|
-
- True
|
|
510
|
-
- random order
|
|
511
|
-
* - None
|
|
512
|
-
- False
|
|
513
|
-
- sequential order
|
|
514
|
-
* - Sampler object
|
|
515
|
-
- None
|
|
516
|
-
- order defined by sampler
|
|
517
|
-
* - Sampler object
|
|
518
|
-
- True
|
|
519
|
-
- not allowed
|
|
520
|
-
* - Sampler object
|
|
521
|
-
- False
|
|
522
|
-
- not allowed
|
|
445
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
446
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
447
|
+
|
|
448
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
523
449
|
|
|
524
450
|
Examples:
|
|
451
|
+
>>> import mindspore.dataset as ds
|
|
525
452
|
>>> lj_speech_dataset_dir = "/path/to/lj_speech_dataset_directory"
|
|
526
453
|
>>>
|
|
527
454
|
>>> # 1) Get all samples from LJSPEECH dataset in sequence
|
|
@@ -603,25 +530,26 @@ class SpeechCommandsDataset(MappableDataset, AudioBaseDataset):
|
|
|
603
530
|
|
|
604
531
|
Args:
|
|
605
532
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
606
|
-
usage (str, optional): Usage of this dataset, can be 'train'
|
|
607
|
-
will read from 84,843 samples, 'test' will read from 11,005 samples, 'valid'
|
|
608
|
-
test samples and 'all' will read from all 105,829 samples.
|
|
533
|
+
usage (str, optional): Usage of this dataset, can be ``'train'``, ``'test'``, ``'valid'`` or ``'all'``.
|
|
534
|
+
``'train'`` will read from 84,843 samples, ``'test'`` will read from 11,005 samples, ``'valid'``
|
|
535
|
+
will read from 9,981 test samples and ``'all'`` will read from all 105,829 samples.
|
|
536
|
+
Default: ``None`` , will read all samples.
|
|
609
537
|
num_samples (int, optional): The number of samples to be included in the dataset.
|
|
610
|
-
Default: None, will read all samples.
|
|
538
|
+
Default: ``None`` , will read all samples.
|
|
611
539
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
612
|
-
Default: None, will use global default workers(8), it can be set
|
|
613
|
-
by
|
|
540
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
541
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
614
542
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
615
|
-
Default: None, expected order behavior shown in the table below.
|
|
543
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
616
544
|
sampler (Sampler, optional): Object used to choose samples from the dataset.
|
|
617
|
-
Default: None, expected order behavior shown in the table below.
|
|
618
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
545
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
546
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
619
547
|
When this argument is specified, `num_samples` reflects the maximum sample number of per shard.
|
|
620
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None.
|
|
621
|
-
when `num_shards` is also specified.
|
|
548
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` .
|
|
549
|
+
This argument can only be specified when `num_shards` is also specified.
|
|
622
550
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
623
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
624
|
-
Default: None, which means no cache is used.
|
|
551
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
552
|
+
Default: ``None`` , which means no cache is used.
|
|
625
553
|
|
|
626
554
|
Raises:
|
|
627
555
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -632,37 +560,18 @@ class SpeechCommandsDataset(MappableDataset, AudioBaseDataset):
|
|
|
632
560
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
633
561
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
634
562
|
|
|
563
|
+
Tutorial Examples:
|
|
564
|
+
- `Load & Process Data With Dataset Pipeline
|
|
565
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
566
|
+
|
|
635
567
|
Note:
|
|
636
|
-
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
..
|
|
640
|
-
:widths: 25 25 50
|
|
641
|
-
:header-rows: 1
|
|
642
|
-
|
|
643
|
-
* - Parameter `sampler`
|
|
644
|
-
- Parameter `shuffle`
|
|
645
|
-
- Expected Order Behavior
|
|
646
|
-
* - None
|
|
647
|
-
- None
|
|
648
|
-
- random order
|
|
649
|
-
* - None
|
|
650
|
-
- True
|
|
651
|
-
- random order
|
|
652
|
-
* - None
|
|
653
|
-
- False
|
|
654
|
-
- sequential order
|
|
655
|
-
* - Sampler object
|
|
656
|
-
- None
|
|
657
|
-
- order defined by sampler
|
|
658
|
-
* - Sampler object
|
|
659
|
-
- True
|
|
660
|
-
- not allowed
|
|
661
|
-
* - Sampler object
|
|
662
|
-
- False
|
|
663
|
-
- not allowed
|
|
568
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
569
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
570
|
+
|
|
571
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
664
572
|
|
|
665
573
|
Examples:
|
|
574
|
+
>>> import mindspore.dataset as ds
|
|
666
575
|
>>> speech_commands_dataset_dir = "/path/to/speech_commands_dataset_directory"
|
|
667
576
|
>>>
|
|
668
577
|
>>> # Read 3 samples from SpeechCommands dataset
|
|
@@ -727,33 +636,33 @@ class TedliumDataset(MappableDataset, AudioBaseDataset):
|
|
|
727
636
|
|
|
728
637
|
Args:
|
|
729
638
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
730
|
-
release (str): Release of the dataset, can be 'release1'
|
|
639
|
+
release (str): Release of the dataset, can be ``'release1'``, ``'release2'``, ``'release3'``.
|
|
731
640
|
usage (str, optional): Usage of this dataset.
|
|
732
|
-
For release1 or release2, can be 'train'
|
|
733
|
-
'train' will read from train samples,
|
|
734
|
-
'test' will read from test samples,
|
|
735
|
-
'dev' will read from dev samples,
|
|
736
|
-
'all' will read from all samples.
|
|
737
|
-
For release3
|
|
738
|
-
extensions (str, optional): Extensions of the SPH files, only '.sph' is valid.
|
|
739
|
-
Default: None, ".sph"
|
|
641
|
+
For release1 or release2, can be ``'train'``, ``'test'``, ``'dev'`` or ``'all'``.
|
|
642
|
+
``'train'`` will read from train samples,
|
|
643
|
+
``'test'`` will read from test samples,
|
|
644
|
+
``'dev'`` will read from dev samples,
|
|
645
|
+
``'all'`` will read from all samples.
|
|
646
|
+
For `release3`, can only be ``'all'``, it will read from data samples. Default: ``None`` , all samples.
|
|
647
|
+
extensions (str, optional): Extensions of the SPH files, only ``'.sph'`` is valid.
|
|
648
|
+
Default: ``None`` , set to ``".sph"``.
|
|
740
649
|
num_samples (int, optional): The number of audio samples to be included in the dataset.
|
|
741
|
-
Default: None, all samples.
|
|
650
|
+
Default: ``None`` , all samples.
|
|
742
651
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
743
|
-
Default: None, will use global default workers(8), it can be set
|
|
744
|
-
by
|
|
745
|
-
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None, expected
|
|
652
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
653
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
654
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected
|
|
746
655
|
order behavior shown in the table below.
|
|
747
656
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
748
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
657
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
749
658
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
750
|
-
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
659
|
+
into. Default: ``None`` . When this argument is specified, `num_samples` reflects
|
|
751
660
|
the maximum sample number of per shard.
|
|
752
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
661
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This
|
|
753
662
|
argument can only be specified when `num_shards` is also specified.
|
|
754
663
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
755
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
756
|
-
Default: None, which means no cache is used.
|
|
664
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
665
|
+
Default: ``None`` , which means no cache is used.
|
|
757
666
|
|
|
758
667
|
Raises:
|
|
759
668
|
RuntimeError: If `dataset_dir` does not contain stm files.
|
|
@@ -764,37 +673,18 @@ class TedliumDataset(MappableDataset, AudioBaseDataset):
|
|
|
764
673
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
765
674
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
766
675
|
|
|
676
|
+
Tutorial Examples:
|
|
677
|
+
- `Load & Process Data With Dataset Pipeline
|
|
678
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
679
|
+
|
|
767
680
|
Note:
|
|
768
|
-
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
..
|
|
772
|
-
:widths: 25 25 50
|
|
773
|
-
:header-rows: 1
|
|
774
|
-
|
|
775
|
-
* - Parameter `sampler`
|
|
776
|
-
- Parameter `shuffle`
|
|
777
|
-
- Expected Order Behavior
|
|
778
|
-
* - None
|
|
779
|
-
- None
|
|
780
|
-
- random order
|
|
781
|
-
* - None
|
|
782
|
-
- True
|
|
783
|
-
- random order
|
|
784
|
-
* - None
|
|
785
|
-
- False
|
|
786
|
-
- sequential order
|
|
787
|
-
* - Sampler object
|
|
788
|
-
- None
|
|
789
|
-
- order defined by sampler
|
|
790
|
-
* - Sampler object
|
|
791
|
-
- True
|
|
792
|
-
- not allowed
|
|
793
|
-
* - Sampler object
|
|
794
|
-
- False
|
|
795
|
-
- not allowed
|
|
681
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
682
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
683
|
+
|
|
684
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
796
685
|
|
|
797
686
|
Examples:
|
|
687
|
+
>>> import mindspore.dataset as ds
|
|
798
688
|
>>> # 1) Get all train samples from TEDLIUM_release1 dataset in sequence.
|
|
799
689
|
>>> dataset = ds.TedliumDataset(dataset_dir="/path/to/tedlium1_dataset_directory",
|
|
800
690
|
... release="release1", shuffle=False)
|
|
@@ -938,21 +828,21 @@ class YesNoDataset(MappableDataset, AudioBaseDataset):
|
|
|
938
828
|
Args:
|
|
939
829
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
940
830
|
num_samples (int, optional): The number of images to be included in the dataset.
|
|
941
|
-
Default: None, will read all images.
|
|
831
|
+
Default: ``None`` , will read all images.
|
|
942
832
|
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
943
|
-
Default: None, will use global default workers(8), it can be set
|
|
944
|
-
by
|
|
833
|
+
Default: ``None`` , will use global default workers(8), it can be set
|
|
834
|
+
by :func:`mindspore.dataset.config.set_num_parallel_workers` .
|
|
945
835
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
946
|
-
Default: None, expected order behavior shown in the table below.
|
|
836
|
+
Default: ``None`` , expected order behavior shown in the table below.
|
|
947
837
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
948
|
-
dataset. Default: None, expected order behavior shown in the table below.
|
|
949
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
838
|
+
dataset. Default: ``None`` , expected order behavior shown in the table below.
|
|
839
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` .
|
|
950
840
|
When this argument is specified, `num_samples` reflects the maximum sample number of per shard.
|
|
951
|
-
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This argument can only
|
|
841
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This argument can only
|
|
952
842
|
be specified when `num_shards` is also specified.
|
|
953
843
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
954
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.
|
|
955
|
-
Default: None, which means no cache is used.
|
|
844
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
|
|
845
|
+
Default: ``None`` , which means no cache is used.
|
|
956
846
|
|
|
957
847
|
Raises:
|
|
958
848
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -963,37 +853,18 @@ class YesNoDataset(MappableDataset, AudioBaseDataset):
|
|
|
963
853
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
964
854
|
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
965
855
|
|
|
856
|
+
Tutorial Examples:
|
|
857
|
+
- `Load & Process Data With Dataset Pipeline
|
|
858
|
+
<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
|
|
859
|
+
|
|
966
860
|
Note:
|
|
967
|
-
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
..
|
|
971
|
-
:widths: 25 25 50
|
|
972
|
-
:header-rows: 1
|
|
973
|
-
|
|
974
|
-
* - Parameter `sampler`
|
|
975
|
-
- Parameter `shuffle`
|
|
976
|
-
- Expected Order Behavior
|
|
977
|
-
* - None
|
|
978
|
-
- None
|
|
979
|
-
- random order
|
|
980
|
-
* - None
|
|
981
|
-
- True
|
|
982
|
-
- random order
|
|
983
|
-
* - None
|
|
984
|
-
- False
|
|
985
|
-
- sequential order
|
|
986
|
-
* - Sampler object
|
|
987
|
-
- None
|
|
988
|
-
- order defined by sampler
|
|
989
|
-
* - Sampler object
|
|
990
|
-
- True
|
|
991
|
-
- not allowed
|
|
992
|
-
* - Sampler object
|
|
993
|
-
- False
|
|
994
|
-
- not allowed
|
|
861
|
+
- The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler
|
|
862
|
+
used in the dataset, and their effects when combined with parameter `sampler` are as follows.
|
|
863
|
+
|
|
864
|
+
.. include:: mindspore.dataset.sampler.txt
|
|
995
865
|
|
|
996
866
|
Examples:
|
|
867
|
+
>>> import mindspore.dataset as ds
|
|
997
868
|
>>> yes_no_dataset_dir = "/path/to/yes_no_dataset_directory"
|
|
998
869
|
>>>
|
|
999
870
|
>>> # Read 3 samples from YesNo dataset
|