mindspore 1.10.0__cp37-cp37m-win_amd64.whl → 2.0.0rc1__cp37-cp37m-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mindspore might be problematic. Click here for more details.
- mindspore/.commit_id +1 -1
- mindspore/ConcurrencyCheck.dll +0 -0
- mindspore/CppBuildInsights.dll +0 -0
- mindspore/CppCoreCheck.dll +0 -0
- mindspore/EnumIndex.dll +0 -0
- mindspore/EspXEngine.dll +0 -0
- mindspore/HResultCheck.dll +0 -0
- mindspore/KernelTraceControl.dll +0 -0
- mindspore/LocalESPC.dll +0 -0
- mindspore/Microsoft.Diagnostics.Tracing.EventSource.dll +0 -0
- mindspore/Microsoft.VisualStudio.RemoteControl.dll +0 -0
- mindspore/Microsoft.VisualStudio.Telemetry.dll +0 -0
- mindspore/Microsoft.VisualStudio.Utilities.Internal.dll +0 -0
- mindspore/Newtonsoft.Json.dll +0 -0
- mindspore/System.Runtime.CompilerServices.Unsafe.dll +0 -0
- mindspore/VariantClear.dll +0 -0
- mindspore/__init__.py +9 -4
- mindspore/_c_dataengine.cp37-win_amd64.pyd +0 -0
- mindspore/_c_expression.cp37-win_amd64.pyd +0 -0
- mindspore/_c_mindrecord.cp37-win_amd64.pyd +0 -0
- mindspore/_check_jit_forbidden_api.py +102 -0
- mindspore/_checkparam.py +1066 -1001
- mindspore/_extends/builtin_operations.py +32 -4
- mindspore/_extends/graph_kernel/model/graph_split.py +66 -222
- mindspore/_extends/parallel_compile/akg_compiler/akg_process.py +12 -9
- mindspore/_extends/parallel_compile/akg_compiler/build_tbe_kernel.py +119 -26
- mindspore/_extends/parallel_compile/akg_compiler/tbe_topi.py +50 -50
- mindspore/_extends/parallel_compile/akg_compiler/util.py +9 -6
- mindspore/_extends/parallel_compile/tbe_compiler/tbe_adapter.py +4 -25
- mindspore/_extends/parallel_compile/tbe_compiler/tbe_helper.py +9 -4
- mindspore/_extends/parallel_compile/tbe_compiler/tbe_job_manager.py +1 -27
- mindspore/_extends/parse/__init__.py +5 -3
- mindspore/_extends/parse/namespace.py +17 -2
- mindspore/_extends/parse/parser.py +193 -34
- mindspore/_extends/parse/resources.py +7 -8
- mindspore/_extends/parse/standard_method.py +1780 -435
- mindspore/_extends/parse/trope.py +3 -1
- mindspore/amp.py +53 -58
- mindspore/atlprov.dll +0 -0
- mindspore/boost/adasum.py +3 -2
- mindspore/boost/boost.py +2 -2
- mindspore/boost/boost_cell_wrapper.py +46 -26
- mindspore/boost/dim_reduce.py +6 -5
- mindspore/boost/grad_accumulation.py +2 -1
- mindspore/boost/group_loss_scale_manager.py +1 -1
- mindspore/c1.dll +0 -0
- mindspore/c1xx.dll +0 -0
- mindspore/c2.dll +0 -0
- mindspore/cfgpersist.dll +0 -0
- mindspore/clang_rt.asan_dbg_dynamic-x86_64.dll +0 -0
- mindspore/clang_rt.asan_dynamic-x86_64.dll +0 -0
- mindspore/common/__init__.py +11 -10
- mindspore/common/_decorator.py +2 -0
- mindspore/common/_register_for_adapter.py +55 -0
- mindspore/common/_stub_tensor.py +201 -0
- mindspore/common/_utils.py +57 -0
- mindspore/common/api.py +582 -297
- mindspore/common/dtype.py +66 -18
- mindspore/common/dump.py +2 -2
- mindspore/common/initializer.py +38 -1
- mindspore/common/jit_config.py +25 -13
- mindspore/common/mutable.py +53 -24
- mindspore/common/parameter.py +60 -37
- mindspore/common/seed.py +8 -24
- mindspore/common/sparse_tensor.py +927 -0
- mindspore/common/tensor.py +1627 -3900
- mindspore/communication/__init__.py +10 -5
- mindspore/communication/_comm_helper.py +78 -214
- mindspore/communication/_hccl_management.py +2 -1
- mindspore/communication/management.py +136 -47
- mindspore/config/op_info.config +501 -1008
- mindspore/context.py +291 -56
- mindspore/d3dcompiler_47.dll +0 -0
- mindspore/dataset/__init__.py +12 -8
- mindspore/dataset/audio/__init__.py +9 -9
- mindspore/dataset/audio/transforms.py +1090 -228
- mindspore/dataset/audio/utils.py +87 -39
- mindspore/dataset/audio/validators.py +223 -1
- mindspore/dataset/callback/ds_callback.py +17 -15
- mindspore/dataset/core/config.py +246 -17
- mindspore/dataset/core/py_util_helpers.py +4 -3
- mindspore/dataset/core/validator_helpers.py +10 -10
- mindspore/{parallel/nn/layers.py → dataset/debug/__init__.py} +7 -8
- mindspore/dataset/debug/debug_hook.py +65 -0
- mindspore/dataset/debug/pre_defined_hook.py +67 -0
- mindspore/dataset/engine/__init__.py +7 -3
- mindspore/dataset/engine/cache_client.py +9 -9
- mindspore/dataset/engine/datasets.py +648 -477
- mindspore/dataset/engine/datasets_audio.py +165 -167
- mindspore/dataset/engine/datasets_standard_format.py +93 -67
- mindspore/dataset/engine/datasets_text.py +492 -342
- mindspore/dataset/engine/datasets_user_defined.py +85 -50
- mindspore/dataset/engine/datasets_vision.py +1224 -699
- mindspore/dataset/engine/graphdata.py +134 -69
- mindspore/dataset/engine/iterators.py +50 -9
- mindspore/dataset/engine/offload.py +52 -31
- mindspore/dataset/engine/samplers.py +27 -24
- mindspore/dataset/engine/serializer_deserializer.py +14 -15
- mindspore/dataset/engine/validators.py +213 -52
- mindspore/dataset/text/__init__.py +10 -8
- mindspore/dataset/text/transforms.py +152 -57
- mindspore/dataset/text/utils.py +98 -49
- mindspore/dataset/text/validators.py +25 -0
- mindspore/dataset/transforms/__init__.py +4 -2
- mindspore/dataset/transforms/c_transforms.py +11 -13
- mindspore/dataset/transforms/py_transforms.py +2 -2
- mindspore/dataset/transforms/py_transforms_util.py +10 -0
- mindspore/dataset/transforms/transforms.py +13 -15
- mindspore/dataset/transforms/validators.py +7 -7
- mindspore/dataset/utils/__init__.py +2 -1
- mindspore/dataset/utils/browse_dataset.py +13 -13
- mindspore/dataset/utils/line_reader.py +121 -0
- mindspore/dataset/vision/__init__.py +8 -7
- mindspore/dataset/vision/c_transforms.py +125 -126
- mindspore/dataset/vision/py_transforms.py +37 -37
- mindspore/dataset/vision/py_transforms_util.py +23 -20
- mindspore/dataset/vision/transforms.py +316 -315
- mindspore/dataset/vision/utils.py +313 -17
- mindspore/dataset/vision/validators.py +6 -6
- mindspore/default_config.py +0 -1
- mindspore/dpcmi.dll +0 -0
- mindspore/{compression → experimental}/__init__.py +6 -5
- mindspore/experimental/map_parameter.py +275 -0
- mindspore/include/OWNERS +0 -1
- mindspore/include/api/callback/callback.h +9 -13
- mindspore/include/api/callback/ckpt_saver.h +2 -2
- mindspore/include/api/callback/loss_monitor.h +2 -2
- mindspore/include/api/callback/lr_scheduler.h +5 -5
- mindspore/include/api/callback/time_monitor.h +2 -2
- mindspore/include/api/callback/train_accuracy.h +4 -6
- mindspore/include/api/cfg.h +19 -6
- mindspore/include/api/context.h +70 -9
- mindspore/include/api/delegate.h +8 -1
- mindspore/include/api/dual_abi_helper.h +8 -24
- mindspore/include/api/metrics/accuracy.h +2 -2
- mindspore/include/api/metrics/metrics.h +4 -3
- mindspore/include/api/model.h +9 -4
- mindspore/include/api/model_group.h +68 -0
- mindspore/include/api/model_parallel_runner.h +17 -17
- mindspore/include/api/net.h +12 -11
- mindspore/include/api/serialization.h +20 -4
- mindspore/include/api/status.h +7 -1
- mindspore/include/api/types.h +25 -21
- mindspore/include/api/visible.h +4 -0
- mindspore/include/c_api/model_c.h +5 -0
- mindspore/include/c_api/status_c.h +1 -1
- mindspore/include/dataset/config.h +1 -1
- mindspore/include/dataset/constants.h +14 -0
- mindspore/include/dataset/text.h +59 -0
- mindspore/include/dataset/vision.h +56 -117
- mindspore/include/dataset/vision_lite.h +102 -0
- mindspore/jpeg62.dll +0 -0
- mindspore/log.py +28 -28
- mindspore/mindrecord/common/exceptions.py +2 -4
- mindspore/mindrecord/filereader.py +19 -1
- mindspore/mindrecord/filewriter.py +250 -88
- mindspore/mindrecord/mindpage.py +13 -13
- mindspore/mindrecord/shardheader.py +15 -15
- mindspore/mindrecord/shardreader.py +9 -0
- mindspore/mindrecord/shardwriter.py +29 -29
- mindspore/mindrecord/tools/cifar100_to_mr.py +9 -9
- mindspore/mindrecord/tools/cifar10_to_mr.py +9 -9
- mindspore/mindrecord/tools/csv_to_mr.py +4 -4
- mindspore/mindrecord/tools/imagenet_to_mr.py +70 -65
- mindspore/mindrecord/tools/mnist_to_mr.py +41 -41
- mindspore/mindrecord/tools/tfrecord_to_mr.py +6 -6
- mindspore/{libmindspore_backend.dll → mindspore_backend.dll} +0 -0
- mindspore/mindspore_common.dll +0 -0
- mindspore/mindspore_core.dll +0 -0
- mindspore/mindspore_glog.dll +0 -0
- mindspore/mindspore_shared_lib.dll +0 -0
- mindspore/msobj140.dll +0 -0
- mindspore/mspdb140.dll +0 -0
- mindspore/mspdbcore.dll +0 -0
- mindspore/mspdbst.dll +0 -0
- mindspore/mspft140.dll +0 -0
- mindspore/msvcdis140.dll +0 -0
- mindspore/msvcp140_1.dll +0 -0
- mindspore/msvcp140_2.dll +0 -0
- mindspore/msvcp140_atomic_wait.dll +0 -0
- mindspore/msvcp140_codecvt_ids.dll +0 -0
- mindspore/nn/__init__.py +1 -5
- mindspore/nn/cell.py +297 -234
- mindspore/nn/dynamic_lr.py +1 -1
- mindspore/nn/grad/cell_grad.py +17 -42
- mindspore/nn/layer/__init__.py +7 -4
- mindspore/nn/layer/activation.py +131 -88
- mindspore/nn/layer/basic.py +313 -613
- mindspore/nn/layer/channel_shuffle.py +103 -0
- mindspore/nn/layer/combined.py +1 -1
- mindspore/nn/layer/container.py +52 -6
- mindspore/nn/layer/conv.py +112 -43
- mindspore/nn/layer/dense.py +10 -9
- mindspore/nn/layer/embedding.py +36 -34
- mindspore/nn/layer/image.py +123 -27
- mindspore/nn/layer/math.py +108 -107
- mindspore/nn/layer/normalization.py +212 -366
- mindspore/nn/layer/padding.py +370 -42
- mindspore/nn/layer/pooling.py +1443 -219
- mindspore/nn/layer/rnn_cells.py +11 -16
- mindspore/nn/layer/rnns.py +38 -39
- mindspore/nn/layer/thor_layer.py +24 -25
- mindspore/nn/layer/timedistributed.py +5 -5
- mindspore/nn/layer/transformer.py +701 -0
- mindspore/nn/learning_rate_schedule.py +8 -8
- mindspore/nn/loss/__init__.py +9 -6
- mindspore/nn/loss/loss.py +678 -142
- mindspore/nn/metrics.py +53 -0
- mindspore/nn/optim/_dist_optimizer_registry.py +2 -2
- mindspore/nn/optim/ada_grad.py +8 -8
- mindspore/nn/optim/adadelta.py +2 -3
- mindspore/nn/optim/adafactor.py +18 -14
- mindspore/nn/optim/adam.py +429 -87
- mindspore/nn/optim/adamax.py +5 -6
- mindspore/nn/optim/adasum.py +10 -8
- mindspore/nn/optim/asgd.py +7 -7
- mindspore/nn/optim/ftrl.py +81 -11
- mindspore/nn/optim/lamb.py +7 -8
- mindspore/nn/optim/lars.py +4 -4
- mindspore/nn/optim/lazyadam.py +82 -7
- mindspore/nn/optim/momentum.py +8 -7
- mindspore/nn/optim/optimizer.py +19 -10
- mindspore/nn/optim/proximal_ada_grad.py +6 -5
- mindspore/nn/optim/rmsprop.py +3 -3
- mindspore/nn/optim/rprop.py +20 -16
- mindspore/nn/optim/sgd.py +21 -15
- mindspore/nn/optim/thor.py +23 -21
- mindspore/nn/probability/__init__.py +0 -2
- mindspore/nn/probability/bijector/bijector.py +7 -6
- mindspore/nn/probability/bijector/invert.py +4 -2
- mindspore/nn/probability/bijector/softplus.py +2 -2
- mindspore/nn/probability/bnn_layers/dense_variational.py +1 -1
- mindspore/nn/probability/bnn_layers/layer_distribution.py +2 -2
- mindspore/nn/probability/distribution/__init__.py +6 -0
- mindspore/nn/probability/distribution/_utils/custom_ops.py +3 -2
- mindspore/nn/probability/distribution/_utils/utils.py +11 -17
- mindspore/nn/probability/distribution/bernoulli.py +6 -6
- mindspore/nn/probability/distribution/beta.py +1 -1
- mindspore/nn/probability/distribution/categorical.py +9 -9
- mindspore/nn/probability/distribution/cauchy.py +8 -8
- mindspore/nn/probability/distribution/distribution.py +12 -6
- mindspore/nn/probability/distribution/exponential.py +5 -5
- mindspore/nn/probability/distribution/gamma.py +3 -3
- mindspore/nn/probability/distribution/geometric.py +6 -5
- mindspore/nn/probability/distribution/gumbel.py +5 -5
- mindspore/nn/probability/distribution/half_normal.py +133 -0
- mindspore/nn/probability/distribution/laplace.py +128 -0
- mindspore/nn/probability/distribution/log_normal.py +0 -1
- mindspore/nn/probability/distribution/logistic.py +4 -5
- mindspore/nn/probability/distribution/normal.py +11 -15
- mindspore/nn/probability/distribution/poisson.py +6 -2
- mindspore/nn/probability/distribution/student_t.py +150 -0
- mindspore/nn/probability/distribution/transformed_distribution.py +4 -4
- mindspore/nn/probability/distribution/uniform.py +5 -5
- mindspore/nn/reinforcement/_tensors_queue.py +3 -3
- mindspore/nn/reinforcement/tensor_array.py +2 -2
- mindspore/nn/sparse/sparse.py +8 -1
- mindspore/nn/wrap/cell_wrapper.py +55 -27
- mindspore/nn/wrap/grad_reducer.py +20 -11
- mindspore/nn/wrap/loss_scale.py +47 -30
- mindspore/numpy/array_creations.py +33 -22
- mindspore/numpy/array_ops.py +46 -42
- mindspore/numpy/logic_ops.py +6 -27
- mindspore/numpy/math_ops.py +26 -19
- mindspore/numpy/utils.py +1 -8
- mindspore/numpy/utils_const.py +112 -62
- mindspore/opencv_core452.dll +0 -0
- mindspore/opencv_imgcodecs452.dll +0 -0
- mindspore/opencv_imgproc452.dll +0 -0
- mindspore/ops/__init__.py +6 -3
- mindspore/ops/_constants.py +0 -6
- mindspore/ops/_grad/__init__.py +2 -1
- mindspore/ops/_grad/grad_array_ops.py +209 -152
- mindspore/ops/_grad/grad_base.py +55 -17
- mindspore/ops/_grad/grad_clip_ops.py +11 -3
- mindspore/ops/_grad/grad_comm_ops.py +58 -47
- mindspore/ops/_grad/grad_implementations.py +21 -61
- mindspore/ops/_grad/grad_inner_ops.py +48 -6
- mindspore/ops/_grad/grad_math_ops.py +306 -161
- mindspore/ops/_grad/grad_nn_ops.py +192 -181
- mindspore/ops/_grad/grad_other_ops.py +1 -1
- mindspore/ops/_grad/grad_quant_ops.py +5 -5
- mindspore/ops/_grad/grad_sequence_ops.py +296 -0
- mindspore/ops/_grad/grad_sparse.py +15 -9
- mindspore/ops/_grad_experimental/__init__.py +1 -0
- mindspore/ops/_grad_experimental/grad_array_ops.py +441 -55
- mindspore/ops/_grad_experimental/grad_image_ops.py +25 -7
- mindspore/ops/_grad_experimental/grad_inner_ops.py +3 -44
- mindspore/ops/_grad_experimental/grad_linalg_ops.py +16 -21
- mindspore/ops/_grad_experimental/grad_math_ops.py +979 -49
- mindspore/ops/_grad_experimental/grad_nn_ops.py +78 -8
- mindspore/ops/_grad_experimental/grad_scalar_ops.py +112 -0
- mindspore/ops/_grad_experimental/grad_sparse_ops.py +197 -13
- mindspore/ops/_op_impl/__init__.py +3 -3
- mindspore/ops/_op_impl/_custom_op/__init__.py +0 -1
- mindspore/ops/_op_impl/_custom_op/_basic.py +0 -1
- mindspore/ops/_op_impl/_custom_op/batch_matmul_impl.py +1 -1
- mindspore/ops/_op_impl/_custom_op/batchnorm_fold.py +4 -2
- mindspore/ops/_op_impl/_custom_op/batchnorm_fold2.py +2 -2
- mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad.py +2 -2
- mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad_reduce.py +5 -5
- mindspore/ops/_op_impl/_custom_op/batchnorm_fold_grad.py +3 -3
- mindspore/ops/_op_impl/_custom_op/cholesky_trsm_impl.py +1 -1
- mindspore/ops/_op_impl/_custom_op/correction_mul.py +3 -3
- mindspore/ops/_op_impl/_custom_op/correction_mul_grad.py +2 -2
- mindspore/ops/_op_impl/_custom_op/dsd_back_impl.py +4 -8
- mindspore/ops/_op_impl/_custom_op/dsd_impl.py +1 -1
- mindspore/ops/_op_impl/_custom_op/fake_learned_scale_quant_perchannel.py +2 -2
- mindspore/ops/_op_impl/_custom_op/fake_learned_scale_quant_perchannel_grad.py +2 -2
- mindspore/ops/_op_impl/_custom_op/fake_learned_scale_quant_perchannel_grad_reduce.py +2 -2
- mindspore/ops/_op_impl/_custom_op/fake_learned_scale_quant_perlayer.py +2 -2
- mindspore/ops/_op_impl/_custom_op/fake_learned_scale_quant_perlayer_grad.py +2 -2
- mindspore/ops/_op_impl/_custom_op/fake_learned_scale_quant_perlayer_grad_reduce.py +2 -2
- mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel.py +2 -2
- mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel_grad.py +2 -2
- mindspore/ops/_op_impl/_custom_op/fake_quant_perlayer.py +2 -2
- mindspore/ops/_op_impl/_custom_op/fake_quant_perlayer_grad.py +2 -2
- mindspore/ops/_op_impl/_custom_op/fused_abs_max1_impl.py +1 -1
- mindspore/ops/_op_impl/_custom_op/img2col_impl.py +1 -1
- mindspore/ops/_op_impl/_custom_op/matmul_cube_dense_left_impl.py +2 -2
- mindspore/ops/_op_impl/_custom_op/matmul_cube_dense_right_impl.py +1 -1
- mindspore/ops/_op_impl/_custom_op/matmul_cube_fracz_left_cast_impl.py +1 -1
- mindspore/ops/_op_impl/_custom_op/matmul_cube_fracz_right_mul_impl.py +1 -1
- mindspore/ops/_op_impl/_custom_op/matmul_cube_impl.py +2 -2
- mindspore/ops/_op_impl/_custom_op/matmul_dds_grad_impl.py +0 -1
- mindspore/ops/_op_impl/_custom_op/matmul_dds_impl.py +0 -1
- mindspore/ops/_op_impl/_custom_op/matrix_combine_impl.py +1 -1
- mindspore/ops/_op_impl/_custom_op/minmax_update_perchannel.py +2 -2
- mindspore/ops/_op_impl/_custom_op/minmax_update_perlayer.py +2 -2
- mindspore/ops/_op_impl/_custom_op/transpose02314_impl.py +1 -1
- mindspore/ops/_op_impl/aicpu/__init__.py +238 -3
- mindspore/ops/_op_impl/aicpu/abs.py +36 -0
- mindspore/ops/_op_impl/aicpu/adaptive_avg_pool_2d.py +34 -0
- mindspore/ops/_op_impl/aicpu/adaptive_avg_pool_2d_grad.py +34 -0
- mindspore/ops/_op_impl/aicpu/adaptive_avg_pool_3d.py +39 -0
- mindspore/ops/_op_impl/aicpu/adaptive_avg_pool_3d_grad.py +39 -0
- mindspore/ops/_op_impl/aicpu/adaptive_max_pool_2d_grad.py +37 -0
- mindspore/ops/_op_impl/aicpu/adaptive_max_pool_3d.py +42 -0
- mindspore/ops/_op_impl/aicpu/adaptive_max_pool_3d_grad.py +152 -0
- mindspore/ops/_op_impl/aicpu/add.py +43 -0
- mindspore/ops/_op_impl/aicpu/addcdiv.py +0 -32
- mindspore/ops/_op_impl/aicpu/addcmul.py +0 -84
- mindspore/ops/_op_impl/aicpu/affine_grid_grad.py +35 -0
- mindspore/ops/_op_impl/aicpu/arg_max.py +75 -0
- mindspore/ops/_op_impl/aicpu/arg_min.py +75 -0
- mindspore/ops/_op_impl/aicpu/argmin_with_value.py +43 -0
- mindspore/ops/_op_impl/aicpu/batch_matmul.py +43 -0
- mindspore/ops/_op_impl/aicpu/batch_norm_grad_grad.py +49 -0
- mindspore/ops/_op_impl/aicpu/bernoulli.py +48 -0
- mindspore/ops/_op_impl/aicpu/bessel_i0.py +31 -0
- mindspore/ops/_op_impl/aicpu/bias_add.py +44 -0
- mindspore/ops/_op_impl/aicpu/bias_add_grad.py +43 -0
- mindspore/ops/_op_impl/aicpu/bincount.py +33 -0
- mindspore/{nn/probability/infer/variational/__init__.py → ops/_op_impl/aicpu/cauchy.py} +17 -10
- mindspore/ops/_op_impl/aicpu/channel_shuffle.py +40 -0
- mindspore/ops/_op_impl/aicpu/cholesky.py +1 -1
- mindspore/ops/_op_impl/{cpu/bias_add.py → aicpu/choleskygrad.py} +9 -7
- mindspore/ops/_op_impl/aicpu/combined_non_max_suppression.py +42 -0
- mindspore/ops/_op_impl/aicpu/concat_offset.py +42 -0
- mindspore/ops/_op_impl/aicpu/concat_offset_v1.py +31 -0
- mindspore/ops/_op_impl/aicpu/conj.py +11 -0
- mindspore/ops/_op_impl/aicpu/crop_and_resize_grad_image.py +38 -0
- mindspore/ops/_op_impl/aicpu/cumulative_logsumexp.py +36 -0
- mindspore/ops/_op_impl/aicpu/deformable_offsets.py +38 -0
- mindspore/ops/_op_impl/aicpu/deformable_offsets_grad.py +2 -2
- mindspore/ops/_op_impl/aicpu/dense_to_sparse_set_operation.py +48 -0
- mindspore/ops/_op_impl/aicpu/diag.py +36 -0
- mindspore/ops/_op_impl/aicpu/diag_part.py +36 -0
- mindspore/ops/_op_impl/aicpu/diagonal.py +35 -0
- mindspore/ops/_op_impl/{cpu/bias_add_grad.py → aicpu/digamma.py} +9 -7
- mindspore/ops/_op_impl/aicpu/eig.py +35 -0
- mindspore/ops/_op_impl/aicpu/fft_with_size.py +41 -0
- mindspore/ops/_op_impl/aicpu/flatten.py +1 -0
- mindspore/ops/_op_impl/aicpu/fmax.py +36 -0
- mindspore/ops/_op_impl/aicpu/fmin.py +37 -0
- mindspore/ops/_op_impl/aicpu/fractional_max_pool3d_with_fixed_ksize.py +1 -1
- mindspore/ops/_op_impl/aicpu/fse_decode.py +43 -0
- mindspore/ops/_op_impl/aicpu/glu.py +33 -0
- mindspore/ops/_op_impl/aicpu/glu_grad.py +34 -0
- mindspore/ops/_op_impl/aicpu/greater.py +41 -0
- mindspore/ops/_op_impl/aicpu/greater_equal.py +41 -0
- mindspore/ops/_op_impl/aicpu/index_put.py +50 -0
- mindspore/ops/_op_impl/{tbe/scatter_add_ds.py → aicpu/inplace_index_add.py} +17 -21
- mindspore/ops/_op_impl/aicpu/instance_norm_v2.py +41 -0
- mindspore/ops/_op_impl/aicpu/instance_norm_v2_grad.py +44 -0
- mindspore/ops/_op_impl/aicpu/layer_norm_grad_grad.py +47 -0
- mindspore/ops/_op_impl/aicpu/less.py +41 -0
- mindspore/ops/_op_impl/aicpu/less_equal.py +41 -0
- mindspore/ops/_op_impl/aicpu/lgamma.py +32 -0
- mindspore/ops/_op_impl/aicpu/log_normal_reverse.py +33 -0
- mindspore/ops/_op_impl/aicpu/logit.py +33 -0
- mindspore/ops/_op_impl/aicpu/logit_grad.py +34 -0
- mindspore/ops/_op_impl/aicpu/masked_fill.py +42 -0
- mindspore/ops/_op_impl/aicpu/masked_scatter.py +39 -0
- mindspore/ops/_op_impl/aicpu/matmul.py +39 -0
- mindspore/ops/_op_impl/aicpu/matrix_logarithm.py +31 -0
- mindspore/ops/_op_impl/aicpu/matrix_power.py +32 -0
- mindspore/ops/_op_impl/aicpu/matrix_solve_ls.py +36 -0
- mindspore/ops/_op_impl/aicpu/matrix_triangular_solve.py +36 -0
- mindspore/ops/_op_impl/aicpu/mirror_pad.py +2 -0
- mindspore/ops/_op_impl/aicpu/mirror_pad_grad.py +0 -4
- mindspore/ops/_op_impl/aicpu/mul.py +3 -1
- mindspore/ops/_op_impl/aicpu/multinomial.py +14 -6
- mindspore/ops/_op_impl/aicpu/multinomial_with_replacement.py +35 -0
- mindspore/ops/_op_impl/aicpu/nan_to_num.py +34 -0
- mindspore/ops/_op_impl/aicpu/nllloss.py +38 -0
- mindspore/ops/_op_impl/aicpu/nllloss_grad.py +39 -0
- mindspore/ops/_op_impl/aicpu/ones_like.py +0 -2
- mindspore/ops/_op_impl/aicpu/polar.py +32 -0
- mindspore/ops/_op_impl/aicpu/polygamma.py +34 -0
- mindspore/ops/_op_impl/aicpu/qr.py +36 -0
- mindspore/ops/_op_impl/aicpu/quant_dtype_cast.py +40 -0
- mindspore/ops/_op_impl/aicpu/quantile.py +35 -0
- mindspore/ops/_op_impl/aicpu/ragged_tensor_to_sparse.py +73 -0
- mindspore/ops/_op_impl/aicpu/ragged_tensor_to_tensor.py +74 -0
- mindspore/ops/_op_impl/aicpu/random_shuffle.py +3 -0
- mindspore/ops/_op_impl/aicpu/randperm_v2.py +41 -0
- mindspore/ops/_op_impl/aicpu/range.py +36 -0
- mindspore/ops/_op_impl/aicpu/reciprocal.py +34 -0
- mindspore/ops/_op_impl/aicpu/reciprocal_grad.py +35 -0
- mindspore/ops/_op_impl/aicpu/reduce_sum.py +57 -0
- mindspore/ops/_op_impl/aicpu/resize_bicubic.py +2 -8
- mindspore/ops/_op_impl/aicpu/resize_bicubic_grad.py +1 -1
- mindspore/ops/_op_impl/aicpu/resize_v2.py +68 -0
- mindspore/ops/_op_impl/aicpu/resize_v2_grad.py +68 -0
- mindspore/ops/_op_impl/aicpu/scatter_elements.py +4 -0
- mindspore/ops/_op_impl/aicpu/scatter_nd_update.py +2 -0
- mindspore/ops/_op_impl/aicpu/search_sorted.py +12 -6
- mindspore/ops/_op_impl/aicpu/self_adjoint_eig.py +34 -0
- mindspore/ops/_op_impl/aicpu/sequence_add.py +34 -0
- mindspore/ops/_op_impl/aicpu/sequence_add_offset.py +34 -0
- mindspore/ops/_op_impl/aicpu/sequence_addn.py +38 -0
- mindspore/ops/_op_impl/aicpu/slice_grad.py +76 -0
- mindspore/ops/_op_impl/aicpu/smooth_l1_loss.py +35 -0
- mindspore/ops/_op_impl/aicpu/smooth_l1_loss_grad.py +37 -0
- mindspore/ops/_op_impl/aicpu/sort.py +39 -0
- mindspore/ops/_op_impl/aicpu/sparse_apply_adagrad_da.py +0 -24
- mindspore/ops/_op_impl/aicpu/sparse_cross.py +42 -0
- mindspore/ops/_op_impl/aicpu/sparse_fill_empty_rows.py +63 -0
- mindspore/ops/_op_impl/aicpu/sparse_fill_empty_rows_grad.py +45 -0
- mindspore/ops/_op_impl/aicpu/sparse_matrix_mat_mul.py +56 -0
- mindspore/ops/_op_impl/{tbe/slice_ds.py → aicpu/sparse_segment_sum.py} +16 -24
- mindspore/ops/_op_impl/aicpu/sparse_segment_sum_with_num_segments.py +68 -0
- mindspore/ops/_op_impl/aicpu/sparse_slice.py +63 -0
- mindspore/ops/_op_impl/aicpu/sparse_slice_grad.py +61 -0
- mindspore/ops/_op_impl/aicpu/squared_difference.py +2 -0
- mindspore/ops/_op_impl/aicpu/strided_slice_v2.py +93 -0
- mindspore/ops/_op_impl/aicpu/strided_slice_v2_grad.py +66 -0
- mindspore/ops/_op_impl/aicpu/tensor_scatter_update.py +59 -0
- mindspore/ops/_op_impl/{tbe/gather_v2.py → aicpu/tile.py} +24 -24
- mindspore/ops/_op_impl/aicpu/tridiagonal_solve.py +35 -0
- mindspore/ops/_op_impl/aicpu/tril_indices.py +34 -0
- mindspore/ops/_op_impl/aicpu/triu_indices.py +34 -0
- mindspore/ops/_op_impl/aicpu/uniform.py +34 -0
- mindspore/ops/_op_impl/aicpu/uniform_candidate_sampler.py +1 -0
- mindspore/ops/_op_impl/aicpu/unique_consecutive.py +10 -2
- mindspore/ops/_op_impl/cpu/__init__.py +1 -2
- mindspore/ops/_op_impl/cpu/dynamic_shape.py +5 -1
- mindspore/ops/_op_impl/cpu/maximum_grad.py +2 -0
- mindspore/{compression/common/__init__.py → ops/_op_impl/cpu/pyexecute.py} +13 -8
- mindspore/ops/_op_impl/cpu/reduce_sum.py +8 -0
- mindspore/ops/_op_impl/cpu/sparse_slice.py +62 -0
- mindspore/ops/_op_impl/cpu/sparse_slice_grad.py +60 -0
- mindspore/ops/_op_impl/cpu/tensor_shape.py +5 -1
- mindspore/ops/_op_impl/tbe/__init__.py +27 -608
- mindspore/ops/_op_impl/tbe/addcdiv_ds.py +42 -0
- mindspore/ops/_op_impl/tbe/addcmul_ds.py +44 -0
- mindspore/ops/_op_impl/tbe/assign_add_ds.py +1 -0
- mindspore/ops/_op_impl/tbe/atomic_addr_clean.py +1 -1
- mindspore/ops/_op_impl/tbe/avg_pool_3d_grad.py +1 -1
- mindspore/ops/_op_impl/tbe/basic_lstm_cell_c_state_grad_v2.py +0 -1
- mindspore/ops/_op_impl/tbe/batch_to_space.py +1 -1
- mindspore/ops/_op_impl/tbe/batch_to_space_nd.py +1 -1
- mindspore/ops/_op_impl/tbe/batch_to_space_nd_v2.py +41 -0
- mindspore/ops/_op_impl/tbe/bce_with_logits_loss.py +1 -0
- mindspore/ops/_op_impl/tbe/bias_add_grad.py +2 -0
- mindspore/ops/_op_impl/tbe/bn_infer_grad.py +4 -2
- mindspore/ops/_op_impl/tbe/bn_infer_grad_ds.py +40 -0
- mindspore/ops/_op_impl/tbe/bn_training_update.py +0 -1
- mindspore/ops/_op_impl/tbe/bn_training_update_ds.py +0 -1
- mindspore/ops/_op_impl/tbe/broadcast_to_ds.py +6 -4
- mindspore/ops/_op_impl/tbe/cast.py +0 -2
- mindspore/ops/_op_impl/tbe/cast_ds.py +3 -3
- mindspore/ops/_op_impl/tbe/ctc_loss_v2.py +0 -2
- mindspore/ops/_op_impl/tbe/ctc_loss_v2_grad.py +0 -2
- mindspore/ops/_op_impl/tbe/data_format_dim_map_ds.py +1 -0
- mindspore/ops/_op_impl/tbe/deformable_offsets.py +1 -0
- mindspore/ops/_op_impl/tbe/depthwise_conv2d.py +1 -1
- mindspore/ops/_op_impl/tbe/dynamic_atomic_addr_clean.py +1 -1
- mindspore/ops/_op_impl/tbe/gather_nd.py +1 -0
- mindspore/ops/_op_impl/tbe/greater.py +2 -0
- mindspore/ops/_op_impl/tbe/{index_add.py → inplace_index_add.py} +3 -6
- mindspore/ops/_op_impl/tbe/layer_norm_beta_gamma_backprop_v2.py +0 -1
- mindspore/ops/_op_impl/tbe/npu_clear_float_status_v2.py +35 -0
- mindspore/ops/_op_impl/tbe/npu_get_float_status_v2.py +35 -0
- mindspore/ops/_op_impl/tbe/one_hot_ds.py +0 -6
- mindspore/ops/_op_impl/tbe/{greater_ds.py → reduce_all_ds.py} +13 -16
- mindspore/ops/_op_impl/tbe/reduce_any_ds.py +39 -0
- mindspore/ops/_op_impl/tbe/roi_align_ds.py +44 -0
- mindspore/ops/_op_impl/tbe/roi_align_grad_ds.py +44 -0
- mindspore/ops/_op_impl/tbe/scatter_add.py +2 -0
- mindspore/ops/_op_impl/tbe/scatter_nd_add.py +2 -2
- mindspore/ops/_op_impl/tbe/slice.py +26 -15
- mindspore/ops/_op_impl/tbe/space_to_batch.py +1 -1
- mindspore/ops/_op_impl/tbe/space_to_batch_nd.py +1 -1
- mindspore/ops/_op_impl/tbe/strided_slice_grad_d.py +1 -0
- mindspore/ops/_op_impl/tbe/trans_data_ds.py +15 -5
- mindspore/ops/_op_impl/tbe/unsorted_segment_sum.py +1 -1
- mindspore/ops/_op_impl/tbe/unsorted_segment_sum_ds.py +2 -0
- mindspore/ops/_primitive_cache.py +3 -2
- mindspore/ops/_register_for_op.py +11 -0
- mindspore/ops/_utils/__init__.py +1 -1
- mindspore/ops/_utils/utils.py +20 -41
- mindspore/ops/_vmap/__init__.py +2 -2
- mindspore/ops/_vmap/vmap_array_ops.py +170 -78
- mindspore/ops/_vmap/vmap_base.py +24 -10
- mindspore/ops/_vmap/vmap_convolution_ops.py +7 -10
- mindspore/ops/_vmap/vmap_grad_math_ops.py +4 -4
- mindspore/ops/_vmap/vmap_grad_nn_ops.py +41 -9
- mindspore/ops/_vmap/vmap_image_ops.py +52 -0
- mindspore/ops/_vmap/vmap_math_ops.py +77 -6
- mindspore/ops/_vmap/vmap_nn_ops.py +78 -29
- mindspore/ops/_vmap/vmap_other_ops.py +3 -1
- mindspore/ops/_vmap/vmap_random_ops.py +55 -3
- mindspore/ops/_vmap/vmap_sparse_ops.py +1 -0
- mindspore/ops/bprop_mindir/AdaptiveAvgPool2D_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/AdaptiveMaxPool2D_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/ApproximateEqual_bprop.mindir +18 -19
- mindspore/ops/bprop_mindir/Argmax_bprop.mindir +13 -12
- mindspore/ops/bprop_mindir/Argmin_bprop.mindir +14 -13
- mindspore/ops/bprop_mindir/AssignSub_bprop.mindir +17 -18
- mindspore/ops/bprop_mindir/Assign_bprop.mindir +16 -16
- mindspore/ops/bprop_mindir/AvgPool3D_bprop.mindir +150 -0
- mindspore/ops/bprop_mindir/AvgPool_bprop.mindir +66 -0
- mindspore/ops/bprop_mindir/BCEWithLogitsLoss_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/BNTrainingReduce_bprop.mindir +13 -12
- mindspore/ops/bprop_mindir/BatchNormGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/BatchToSpaceND_bprop.mindir +28 -0
- mindspore/ops/bprop_mindir/BiasAddGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/BinaryCrossEntropy_bprop.mindir +33 -0
- mindspore/ops/bprop_mindir/BroadcastTo_bprop.mindir +306 -0
- mindspore/ops/bprop_mindir/Broadcast_bprop.mindir +12 -8
- mindspore/ops/bprop_mindir/CTCLoss_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Concat_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Conv2DBackpropFilter_bprop.mindir +240 -0
- mindspore/ops/bprop_mindir/Conv2DBackpropInput_bprop.mindir +247 -0
- mindspore/ops/bprop_mindir/Conv2DTranspose_bprop.mindir +247 -0
- mindspore/ops/bprop_mindir/Conv3DTranspose_bprop.mindir +315 -0
- mindspore/ops/bprop_mindir/Conv3D_bprop.mindir +278 -0
- mindspore/ops/bprop_mindir/DType_bprop.mindir +12 -12
- mindspore/ops/bprop_mindir/DeformableOffsets_bprop.mindir +58 -0
- mindspore/ops/bprop_mindir/Depend_bprop.mindir +12 -13
- mindspore/ops/bprop_mindir/DepthToSpace_bprop.mindir +23 -0
- mindspore/ops/bprop_mindir/DepthwiseConv2dNative_bprop.mindir +138 -0
- mindspore/ops/bprop_mindir/DiagPart_bprop.mindir +15 -0
- mindspore/ops/bprop_mindir/Dropout2D_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Dropout3D_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/DropoutDoMask_bprop.mindir +22 -24
- mindspore/ops/bprop_mindir/DropoutGenMask_bprop.mindir +16 -14
- mindspore/ops/bprop_mindir/DropoutGrad_bprop.mindir +27 -0
- mindspore/ops/bprop_mindir/Dropout_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/DynamicGRUV2_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/DynamicRNN_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/DynamicShape_bprop.mindir +12 -12
- mindspore/ops/bprop_mindir/Elu_bprop.mindir +16 -0
- mindspore/ops/bprop_mindir/EmbeddingLookup_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Equal_bprop.mindir +18 -19
- mindspore/ops/bprop_mindir/ExpandDims_bprop.mindir +58 -0
- mindspore/ops/bprop_mindir/FastGeLU_bprop.mindir +16 -0
- mindspore/ops/bprop_mindir/Flatten_bprop.mindir +54 -0
- mindspore/ops/bprop_mindir/FloorDiv_bprop.mindir +18 -15
- mindspore/ops/bprop_mindir/GatherD_bprop.mindir +26 -0
- mindspore/ops/bprop_mindir/GatherNd_bprop.mindir +57 -0
- mindspore/ops/bprop_mindir/Gather_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/GreaterEqual_bprop.mindir +17 -18
- mindspore/ops/bprop_mindir/Greater_bprop.mindir +18 -19
- mindspore/ops/bprop_mindir/HSigmoid_bprop.mindir +16 -0
- mindspore/ops/bprop_mindir/HSwish_bprop.mindir +16 -0
- mindspore/ops/bprop_mindir/IOU_bprop.mindir +18 -19
- mindspore/ops/bprop_mindir/InstanceNorm_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/IsFinite_bprop.mindir +13 -12
- mindspore/ops/bprop_mindir/IsInf_bprop.mindir +13 -10
- mindspore/ops/bprop_mindir/IsNan_bprop.mindir +14 -11
- mindspore/ops/bprop_mindir/KLDivLoss_bprop.mindir +126 -0
- mindspore/ops/bprop_mindir/L2Loss_bprop.mindir +15 -0
- mindspore/ops/bprop_mindir/L2Normalize_bprop.mindir +30 -0
- mindspore/ops/bprop_mindir/LRN_bprop.mindir +43 -0
- mindspore/ops/bprop_mindir/LayerNormGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/LessEqual_bprop.mindir +18 -19
- mindspore/ops/bprop_mindir/Less_bprop.mindir +17 -18
- mindspore/ops/bprop_mindir/LinSpace_bprop.mindir +22 -19
- mindspore/ops/bprop_mindir/Load_bprop.mindir +12 -13
- mindspore/ops/bprop_mindir/LogSoftmax_bprop.mindir +23 -0
- mindspore/ops/bprop_mindir/LogicalAnd_bprop.mindir +17 -18
- mindspore/ops/bprop_mindir/LogicalNot_bprop.mindir +14 -13
- mindspore/ops/bprop_mindir/MaskedSelect_bprop.mindir +21 -0
- mindspore/ops/bprop_mindir/MaxPool3DGradGrad_bprop.mindir +74 -0
- mindspore/ops/bprop_mindir/MaxPool3DGrad_bprop.mindir +74 -0
- mindspore/ops/bprop_mindir/MaxPool3D_bprop.mindir +75 -0
- mindspore/ops/bprop_mindir/MaxPoolGradGrad_bprop.mindir +65 -0
- mindspore/ops/bprop_mindir/MaxPoolWithArgmax_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Maximum_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Minimum_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/MirrorPad_bprop.mindir +27 -0
- mindspore/ops/bprop_mindir/Mish_bprop.mindir +35 -0
- mindspore/ops/bprop_mindir/MulNoNan_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/NLLLoss_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/NonZero_bprop.mindir +14 -0
- mindspore/ops/bprop_mindir/NotEqual_bprop.mindir +18 -19
- mindspore/ops/bprop_mindir/OneHot_bprop.mindir +25 -23
- mindspore/ops/bprop_mindir/OnesLike_bprop.mindir +13 -13
- mindspore/ops/bprop_mindir/PReLU_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Pad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Padding_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/RNNTLoss_bprop.mindir +29 -0
- mindspore/ops/bprop_mindir/ROIAlign_bprop.mindir +82 -0
- mindspore/ops/bprop_mindir/Range_bprop.mindir +21 -19
- mindspore/ops/bprop_mindir/Rank_bprop.mindir +11 -11
- mindspore/ops/bprop_mindir/ReLU6_bprop.mindir +16 -0
- mindspore/ops/bprop_mindir/ReLUV2_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/ReduceAll_bprop.mindir +18 -17
- mindspore/ops/bprop_mindir/ReduceAny_bprop.mindir +18 -17
- mindspore/ops/bprop_mindir/ReluGrad_bprop.mindir +19 -23
- mindspore/ops/bprop_mindir/Reshape_bprop.mindir +60 -0
- mindspore/ops/bprop_mindir/ResizeBilinear_bprop.mindir +29 -0
- mindspore/ops/bprop_mindir/ResizeNearestNeighbor_bprop.mindir +89 -0
- mindspore/ops/bprop_mindir/ReverseSequence_bprop.mindir +52 -0
- mindspore/ops/bprop_mindir/ReverseV2_bprop.mindir +22 -0
- mindspore/ops/bprop_mindir/Round_bprop.mindir +14 -13
- mindspore/ops/bprop_mindir/ScatterMax_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/ScatterMin_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/ScatterNdUpdate_bprop.mindir +22 -0
- mindspore/ops/bprop_mindir/ScatterNd_bprop.mindir +24 -0
- mindspore/ops/bprop_mindir/ScatterNonAliasingAdd_bprop.mindir +22 -0
- mindspore/ops/bprop_mindir/ScatterUpdate_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/SeLU_bprop.mindir +21 -0
- mindspore/ops/bprop_mindir/Select_bprop.mindir +30 -34
- mindspore/ops/bprop_mindir/Shape_bprop.mindir +12 -12
- mindspore/ops/bprop_mindir/SigmoidCrossEntropyWithLogits_bprop.mindir +21 -0
- mindspore/ops/bprop_mindir/SigmoidGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Sigmoid_bprop.mindir +16 -0
- mindspore/ops/bprop_mindir/Sign_bprop.mindir +13 -12
- mindspore/ops/bprop_mindir/Slice_bprop.mindir +26 -0
- mindspore/ops/bprop_mindir/SmoothL1Loss_bprop.mindir +36 -0
- mindspore/ops/bprop_mindir/SoftmaxCrossEntropyWithLogits_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Softplus_bprop.mindir +16 -0
- mindspore/ops/bprop_mindir/Softsign_bprop.mindir +33 -0
- mindspore/ops/bprop_mindir/Sort_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/SpaceToBatchND_bprop.mindir +28 -0
- mindspore/ops/bprop_mindir/SpaceToDepth_bprop.mindir +23 -0
- mindspore/ops/bprop_mindir/SparseGatherV2_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/SparseSoftmaxCrossEntropyWithLogits_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Split_bprop.mindir +22 -0
- mindspore/ops/bprop_mindir/Squeeze_bprop.mindir +54 -0
- mindspore/ops/bprop_mindir/StridedSliceGrad_bprop.mindir +95 -0
- mindspore/ops/bprop_mindir/StridedSlice_bprop.mindir +98 -0
- mindspore/ops/bprop_mindir/Switch_bprop.mindir +28 -32
- mindspore/ops/bprop_mindir/TanhGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Tanh_bprop.mindir +66 -0
- mindspore/ops/bprop_mindir/TensorScatterAdd_bprop.mindir +22 -0
- mindspore/ops/bprop_mindir/TensorScatterUpdate_bprop.mindir +29 -0
- mindspore/ops/bprop_mindir/TensorShape_bprop.mindir +14 -0
- mindspore/ops/bprop_mindir/Tile_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/TopK_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/TransShape_bprop.mindir +23 -0
- mindspore/ops/bprop_mindir/TruncateDiv_bprop.mindir +18 -15
- mindspore/ops/bprop_mindir/TupleGetItem_bprop.mindir +11 -13
- mindspore/ops/bprop_mindir/Unique_bprop.mindir +16 -0
- mindspore/ops/bprop_mindir/Unstack_bprop.mindir +22 -0
- mindspore/ops/bprop_mindir/UpsampleNearest3D_bprop.mindir +32 -0
- mindspore/ops/bprop_mindir/UpsampleTrilinear3D_bprop.mindir +38 -0
- mindspore/ops/bprop_mindir/ZerosLike_bprop.mindir +13 -12
- mindspore/ops/bprop_mindir/__init__.py +1 -4
- mindspore/ops/bprop_mindir/generate_mindir.py +32 -20
- mindspore/ops/composite/__init__.py +12 -13
- mindspore/ops/composite/base.py +261 -254
- mindspore/ops/composite/env_ops.py +41 -0
- mindspore/ops/composite/math_ops.py +197 -156
- mindspore/ops/composite/multitype_ops/_compile_utils.py +428 -176
- mindspore/ops/composite/multitype_ops/_constexpr_utils.py +188 -87
- mindspore/ops/composite/multitype_ops/add_impl.py +23 -1
- mindspore/ops/composite/multitype_ops/div_impl.py +3 -3
- mindspore/ops/composite/multitype_ops/equal_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/floordiv_impl.py +1 -1
- mindspore/ops/composite/multitype_ops/getitem_impl.py +52 -5
- mindspore/ops/composite/multitype_ops/greater_equal_impl.py +31 -0
- mindspore/ops/composite/multitype_ops/greater_impl.py +31 -0
- mindspore/ops/composite/multitype_ops/in_impl.py +15 -3
- mindspore/ops/composite/multitype_ops/less_equal_impl.py +33 -2
- mindspore/ops/composite/multitype_ops/less_impl.py +33 -0
- mindspore/ops/composite/multitype_ops/logical_and_impl.py +2 -2
- mindspore/ops/composite/multitype_ops/logical_or_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/mod_impl.py +1 -1
- mindspore/ops/composite/multitype_ops/mul_impl.py +21 -7
- mindspore/ops/composite/multitype_ops/not_in_impl.py +15 -3
- mindspore/ops/composite/multitype_ops/ones_like_impl.py +2 -4
- mindspore/ops/composite/multitype_ops/pow_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/setitem_impl.py +62 -70
- mindspore/ops/composite/multitype_ops/sub_impl.py +3 -3
- mindspore/ops/composite/multitype_ops/zeros_like_impl.py +41 -4
- mindspore/ops/function/__init__.py +323 -8
- mindspore/ops/function/array_func.py +3511 -780
- mindspore/ops/function/clip_func.py +329 -0
- mindspore/ops/function/debug_func.py +6 -6
- mindspore/ops/function/grad/__init__.py +5 -1
- mindspore/ops/function/grad/grad_func.py +736 -65
- mindspore/ops/function/image_func.py +270 -0
- mindspore/ops/function/linalg_func.py +268 -8
- mindspore/ops/function/math_func.py +8032 -3164
- mindspore/ops/function/nn_func.py +5619 -1855
- mindspore/ops/function/other_func.py +115 -0
- mindspore/ops/function/parameter_func.py +11 -10
- mindspore/ops/function/random_func.py +939 -77
- mindspore/ops/function/sparse_func.py +249 -84
- mindspore/ops/function/sparse_unary_func.py +2303 -0
- mindspore/ops/function/spectral_func.py +146 -0
- mindspore/ops/function/vmap_func.py +114 -0
- mindspore/ops/functional.py +182 -254
- mindspore/ops/op_info_register.py +79 -34
- mindspore/ops/operations/__init__.py +210 -118
- mindspore/ops/operations/_csr_ops.py +7 -7
- mindspore/ops/operations/_embedding_cache_ops.py +25 -15
- mindspore/ops/operations/_grad_ops.py +447 -322
- mindspore/ops/operations/_inner_ops.py +547 -176
- mindspore/ops/operations/_map_tensor_ops.py +112 -0
- mindspore/ops/operations/_ms_kernel.py +29 -27
- mindspore/ops/operations/_ocr_ops.py +11 -11
- mindspore/ops/operations/_opaque_predicate_registry.py +41 -0
- mindspore/ops/operations/_quant_ops.py +186 -101
- mindspore/ops/operations/_rl_inner_ops.py +122 -61
- mindspore/ops/operations/_scalar_ops.py +466 -0
- mindspore/ops/operations/_sequence_ops.py +1047 -0
- mindspore/ops/operations/_tensor_array.py +10 -11
- mindspore/ops/operations/_thor_ops.py +4 -4
- mindspore/ops/operations/array_ops.py +1428 -1226
- mindspore/ops/operations/comm_ops.py +180 -117
- mindspore/ops/operations/control_ops.py +4 -2
- mindspore/ops/operations/custom_ops.py +185 -98
- mindspore/ops/operations/debug_ops.py +92 -54
- mindspore/ops/operations/image_ops.py +406 -211
- mindspore/ops/operations/inner_ops.py +42 -53
- mindspore/ops/operations/linalg_ops.py +32 -29
- mindspore/ops/operations/math_ops.py +2076 -897
- mindspore/ops/operations/nn_ops.py +1282 -1252
- mindspore/ops/operations/other_ops.py +124 -278
- mindspore/ops/operations/random_ops.py +345 -178
- mindspore/ops/operations/rl_ops.py +8 -9
- mindspore/ops/operations/sparse_ops.py +502 -157
- mindspore/ops/operations/spectral_ops.py +107 -0
- mindspore/ops/primitive.py +192 -15
- mindspore/ops/vm_impl_registry.py +23 -2
- mindspore/parallel/__init__.py +6 -1
- mindspore/parallel/_auto_parallel_context.py +199 -92
- mindspore/parallel/_cell_wrapper.py +4 -2
- mindspore/parallel/_cost_model_context.py +3 -0
- mindspore/parallel/_dp_allreduce_fusion.py +2 -1
- mindspore/parallel/_offload_context.py +185 -0
- mindspore/parallel/_parallel_serialization.py +167 -28
- mindspore/parallel/_ps_context.py +9 -5
- mindspore/parallel/_recovery_context.py +1 -1
- mindspore/parallel/_tensor.py +9 -1
- mindspore/{nn/transformer → parallel/_transformer}/__init__.py +6 -6
- mindspore/{nn/transformer → parallel/_transformer}/layers.py +59 -37
- mindspore/{nn/transformer → parallel/_transformer}/loss.py +4 -7
- mindspore/{nn/transformer → parallel/_transformer}/moe.py +160 -35
- mindspore/{nn/transformer → parallel/_transformer}/op_parallel_config.py +3 -3
- mindspore/{nn/transformer → parallel/_transformer}/transformer.py +235 -196
- mindspore/parallel/_utils.py +47 -7
- mindspore/parallel/algo_parameter_config.py +5 -1
- mindspore/parallel/checkpoint_transform.py +329 -0
- mindspore/parallel/shard.py +229 -0
- mindspore/perf_msvcbuildinsights.dll +0 -0
- mindspore/pgodb140.dll +0 -0
- mindspore/pgort140.dll +0 -0
- mindspore/profiler/__init__.py +2 -1
- mindspore/profiler/common/util.py +4 -3
- mindspore/profiler/common/validator/validate_path.py +2 -2
- mindspore/profiler/envprofiling.py +249 -0
- mindspore/profiler/parser/aicpu_data_parser.py +38 -39
- mindspore/profiler/parser/ascend_timeline_generator.py +497 -0
- mindspore/profiler/parser/base_timeline_generator.py +471 -0
- mindspore/profiler/parser/cpu_gpu_timeline_generator.py +684 -0
- mindspore/profiler/parser/framework_parser.py +42 -16
- mindspore/profiler/parser/hccl_parser.py +158 -158
- mindspore/profiler/parser/hwts_log_parser.py +7 -6
- mindspore/profiler/parser/integrator.py +18 -1579
- mindspore/profiler/parser/minddata_analyzer.py +8 -8
- mindspore/profiler/parser/msadvisor_analyzer.py +14 -27
- mindspore/profiler/parser/msadvisor_parser.py +2 -4
- mindspore/profiler/parser/optime_parser.py +17 -18
- mindspore/profiler/parser/profiler_info.py +108 -0
- mindspore/profiler/parser/step_trace_parser.py +1 -1
- mindspore/profiler/profiling.py +396 -194
- mindspore/rewrite/__init__.py +6 -2
- mindspore/rewrite/api/node.py +51 -110
- mindspore/rewrite/api/node_type.py +10 -6
- mindspore/rewrite/api/pattern_engine.py +51 -7
- mindspore/rewrite/api/scoped_value.py +64 -53
- mindspore/rewrite/api/symbol_tree.py +108 -61
- mindspore/rewrite/api/tree_node_helper.py +2 -3
- mindspore/{compression/quant/__init__.py → rewrite/ast_creator_register.py} +20 -11
- mindspore/rewrite/ast_helpers/__init__.py +6 -3
- mindspore/rewrite/ast_helpers/ast_creator.py +115 -0
- mindspore/rewrite/ast_helpers/ast_finder.py +99 -1
- mindspore/rewrite/ast_helpers/ast_modifier.py +17 -4
- mindspore/rewrite/ast_helpers/ast_replacer.py +1 -1
- mindspore/rewrite/ast_transformers/__init__.py +0 -1
- mindspore/rewrite/ast_transformers/flatten_recursive_stmt.py +46 -5
- mindspore/rewrite/ast_transformers/remove_return_out_of_if.py +6 -3
- mindspore/rewrite/common/__init__.py +2 -0
- mindspore/rewrite/common/event.py +1 -1
- mindspore/rewrite/common/observable.py +1 -1
- mindspore/rewrite/common/observer.py +1 -1
- mindspore/rewrite/common/rewrite_elog.py +35 -0
- mindspore/rewrite/namer.py +2 -2
- mindspore/rewrite/namespace.py +14 -4
- mindspore/rewrite/node.py +161 -13
- mindspore/rewrite/parser.py +0 -1
- mindspore/rewrite/parser_register.py +0 -1
- mindspore/rewrite/parsers/arguments_parser.py +3 -2
- mindspore/rewrite/parsers/assign_parser.py +267 -67
- mindspore/rewrite/parsers/attribute_parser.py +56 -0
- mindspore/rewrite/parsers/class_def_parser.py +191 -108
- mindspore/rewrite/parsers/constant_parser.py +101 -0
- mindspore/rewrite/parsers/container_parser.py +88 -0
- mindspore/rewrite/parsers/for_parser.py +28 -15
- mindspore/rewrite/parsers/function_def_parser.py +21 -5
- mindspore/rewrite/parsers/if_parser.py +11 -28
- mindspore/rewrite/parsers/module_parser.py +9 -6
- mindspore/rewrite/parsers/return_parser.py +3 -2
- mindspore/rewrite/sparsify/__init__.py +0 -0
- mindspore/rewrite/sparsify/sparse_transformer.py +448 -0
- mindspore/rewrite/sparsify/sparsify.py +109 -0
- mindspore/rewrite/sparsify/utils.py +173 -0
- mindspore/rewrite/symbol_tree.py +322 -109
- mindspore/rewrite/symbol_tree_builder.py +45 -8
- mindspore/rewrite/symbol_tree_dumper.py +0 -1
- mindspore/rewrite/topological_manager.py +1 -2
- mindspore/run_check/_check_version.py +209 -112
- mindspore/run_check/run_check.py +2 -1
- mindspore/tbbmalloc.dll +0 -0
- mindspore/tinyxml2.dll +0 -0
- mindspore/train/__init__.py +6 -4
- mindspore/train/_utils.py +28 -5
- mindspore/train/amp.py +321 -50
- mindspore/train/callback/__init__.py +3 -1
- mindspore/train/callback/_backup_and_restore.py +120 -0
- mindspore/train/callback/_callback.py +8 -8
- mindspore/train/callback/_checkpoint.py +12 -9
- mindspore/train/callback/_early_stop.py +13 -7
- mindspore/train/callback/_history.py +8 -8
- mindspore/train/callback/_lambda_callback.py +6 -6
- mindspore/train/callback/_landscape.py +36 -38
- mindspore/train/callback/_loss_monitor.py +12 -6
- mindspore/train/callback/_lr_scheduler_callback.py +2 -4
- mindspore/train/callback/_on_request_exit.py +212 -0
- mindspore/train/callback/_reduce_lr_on_plateau.py +13 -7
- mindspore/train/callback/_summary_collector.py +27 -19
- mindspore/train/callback/_time_monitor.py +13 -7
- mindspore/train/checkpoint_pb2.py +68 -8
- mindspore/train/data_sink.py +122 -33
- mindspore/train/dataset_helper.py +28 -87
- mindspore/train/loss_scale_manager.py +4 -7
- mindspore/{nn → train}/metrics/__init__.py +20 -20
- mindspore/{nn → train}/metrics/accuracy.py +12 -10
- mindspore/{nn → train}/metrics/auc.py +4 -4
- mindspore/{nn → train}/metrics/bleu_score.py +4 -4
- mindspore/{nn → train}/metrics/confusion_matrix.py +10 -8
- mindspore/{nn → train}/metrics/cosine_similarity.py +4 -4
- mindspore/{nn → train}/metrics/dice.py +6 -5
- mindspore/{nn → train}/metrics/error.py +7 -5
- mindspore/{nn → train}/metrics/fbeta.py +9 -7
- mindspore/{nn → train}/metrics/hausdorff_distance.py +8 -6
- mindspore/{nn → train}/metrics/loss.py +4 -3
- mindspore/{nn → train}/metrics/mean_surface_distance.py +6 -5
- mindspore/{nn → train}/metrics/metric.py +6 -5
- mindspore/{nn → train}/metrics/occlusion_sensitivity.py +4 -3
- mindspore/{nn → train}/metrics/perplexity.py +5 -4
- mindspore/{nn → train}/metrics/precision.py +5 -4
- mindspore/{nn → train}/metrics/recall.py +5 -4
- mindspore/{nn → train}/metrics/roc.py +7 -6
- mindspore/{nn → train}/metrics/root_mean_square_surface_distance.py +6 -5
- mindspore/{nn → train}/metrics/topk.py +7 -5
- mindspore/train/mind_ir_pb2.py +339 -32
- mindspore/train/model.py +113 -84
- mindspore/train/serialization.py +547 -167
- mindspore/train/summary/_summary_adapter.py +1 -1
- mindspore/train/summary/summary_record.py +43 -12
- mindspore/train/train_thor/convert_utils.py +7 -1
- mindspore/train/train_thor/dataset_helper.py +3 -3
- mindspore/train/train_thor/model_thor.py +0 -4
- mindspore/turbojpeg.dll +0 -0
- mindspore/vcmeta.dll +0 -0
- mindspore/vcruntime140.dll +0 -0
- mindspore/vcruntime140_1.dll +0 -0
- mindspore/version.py +1 -1
- {mindspore-1.10.0.dist-info → mindspore-2.0.0rc1.dist-info}/METADATA +4 -3
- {mindspore-1.10.0.dist-info → mindspore-2.0.0rc1.dist-info}/RECORD +901 -660
- mindspore/compression/common/constant.py +0 -124
- mindspore/compression/export/__init__.py +0 -19
- mindspore/compression/export/quant_export.py +0 -514
- mindspore/compression/quant/qat.py +0 -636
- mindspore/compression/quant/quant_utils.py +0 -462
- mindspore/compression/quant/quantizer.py +0 -68
- mindspore/libatomic-1.dll +0 -0
- mindspore/libgcc_s_seh-1.dll +0 -0
- mindspore/libgfortran-4.dll +0 -0
- mindspore/libgomp-1.dll +0 -0
- mindspore/libjpeg-62.dll +0 -0
- mindspore/libmindspore.dll +0 -0
- mindspore/libmindspore_common.dll +0 -0
- mindspore/libmindspore_core.dll +0 -0
- mindspore/libmindspore_glog.dll +0 -0
- mindspore/libnnacl.dll +0 -0
- mindspore/libopencv_core452.dll +0 -0
- mindspore/libopencv_imgcodecs452.dll +0 -0
- mindspore/libopencv_imgproc452.dll +0 -0
- mindspore/libquadmath-0.dll +0 -0
- mindspore/libsqlite3.dll +0 -0
- mindspore/libssp-0.dll +0 -0
- mindspore/libstdc++-6.dll +0 -0
- mindspore/libtinyxml2.dll +0 -0
- mindspore/libturbojpeg.dll +0 -0
- mindspore/libwinpthread-1.dll +0 -0
- mindspore/nn/layer/quant.py +0 -1868
- mindspore/nn/layer/rnn_utils.py +0 -90
- mindspore/nn/probability/dpn/__init__.py +0 -22
- mindspore/nn/probability/dpn/vae/__init__.py +0 -25
- mindspore/nn/probability/dpn/vae/cvae.py +0 -138
- mindspore/nn/probability/dpn/vae/vae.py +0 -122
- mindspore/nn/probability/infer/__init__.py +0 -22
- mindspore/nn/probability/infer/variational/elbo.py +0 -70
- mindspore/nn/probability/infer/variational/svi.py +0 -84
- mindspore/nn/probability/toolbox/__init__.py +0 -22
- mindspore/nn/probability/toolbox/anomaly_detection.py +0 -99
- mindspore/nn/probability/toolbox/uncertainty_evaluation.py +0 -363
- mindspore/nn/probability/transforms/__init__.py +0 -22
- mindspore/nn/probability/transforms/transform_bnn.py +0 -262
- mindspore/nn/probability/zhusuan/__init__.py +0 -18
- mindspore/nn/probability/zhusuan/framework/__init__.py +0 -18
- mindspore/nn/probability/zhusuan/framework/bn.py +0 -95
- mindspore/nn/probability/zhusuan/variational/__init__.py +0 -18
- mindspore/nn/probability/zhusuan/variational/elbo.py +0 -46
- mindspore/ops/_op_impl/tbe/bias_add_grad_ds.py +0 -52
- mindspore/ops/_op_impl/tbe/scatter_nd_add_ds.py +0 -43
- mindspore/ops/bprop_mindir/AssignAdd_bprop.mindir +0 -20
- mindspore/ops/bprop_mindir/Identity_bprop.mindir +0 -9
- mindspore/ops/bprop_mindir/LogicalOr_bprop.mindir +0 -20
- mindspore/ops/bprop_mindir/ReLU_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/UpdateState_bprop.mindir +0 -17
- mindspore/ops/bprop_mindir/stop_gradient_bprop.mindir +0 -12
- mindspore/ops/composite/array_ops.py +0 -210
- mindspore/ops/composite/clip_ops.py +0 -238
- mindspore/ops/composite/random_ops.py +0 -426
- mindspore/ops/composite/vmap_ops.py +0 -38
- mindspore/ops/operations/sponge_ops.py +0 -3531
- mindspore/ops/operations/sponge_update_ops.py +0 -2546
- mindspore/parallel/nn/__init__.py +0 -42
- mindspore/parallel/nn/loss.py +0 -22
- mindspore/parallel/nn/moe.py +0 -21
- mindspore/parallel/nn/op_parallel_config.py +0 -22
- mindspore/parallel/nn/transformer.py +0 -31
- mindspore/run_check/_check_deps_version.py +0 -84
- {mindspore-1.10.0.dist-info → mindspore-2.0.0rc1.dist-info}/WHEEL +0 -0
- {mindspore-1.10.0.dist-info → mindspore-2.0.0rc1.dist-info}/entry_points.txt +0 -0
- {mindspore-1.10.0.dist-info → mindspore-2.0.0rc1.dist-info}/top_level.txt +0 -0
|
@@ -34,9 +34,9 @@ from ..core.validator_helpers import replace_none
|
|
|
34
34
|
|
|
35
35
|
class CMUArcticDataset(MappableDataset, AudioBaseDataset):
|
|
36
36
|
"""
|
|
37
|
-
|
|
37
|
+
CMU Arctic dataset.
|
|
38
38
|
|
|
39
|
-
The generated dataset has four columns: :py:obj:`[
|
|
39
|
+
The generated dataset has four columns: :py:obj:`[waveform, sample_rate, transcript, utterance_id]` .
|
|
40
40
|
The tensor of column :py:obj:`waveform` is of the float32 type.
|
|
41
41
|
The tensor of column :py:obj:`sample_rate` is of a scalar of uint32 type.
|
|
42
42
|
The tensor of column :py:obj:`transcript` is of a scalar of string type.
|
|
@@ -45,64 +45,64 @@ class CMUArcticDataset(MappableDataset, AudioBaseDataset):
|
|
|
45
45
|
Args:
|
|
46
46
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
47
47
|
name (str, optional): Part of this dataset, can be 'aew', 'ahw', 'aup', 'awb', 'axb', 'bdl',
|
|
48
|
-
'clb', 'eey', 'fem', 'gka', 'jmk', 'ksp', 'ljm', 'lnh', 'rms', 'rxr', 'slp' or 'slt'
|
|
49
|
-
|
|
50
|
-
num_samples (int, optional): The number of audio to be included in the dataset
|
|
51
|
-
|
|
52
|
-
num_parallel_workers (int, optional): Number of
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
48
|
+
'clb', 'eey', 'fem', 'gka', 'jmk', 'ksp', 'ljm', 'lnh', 'rms', 'rxr', 'slp' or 'slt'.
|
|
49
|
+
Default: None, means 'aew'.
|
|
50
|
+
num_samples (int, optional): The number of audio to be included in the dataset.
|
|
51
|
+
Default: None, will read all audio.
|
|
52
|
+
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
53
|
+
Default: None, will use global default workers(8), it can be set
|
|
54
|
+
by `mindspore.dataset.config.set_num_parallel_workers` .
|
|
55
|
+
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
56
|
+
Default: None, expected order behavior shown in the table below.
|
|
56
57
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
57
|
-
dataset
|
|
58
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into
|
|
58
|
+
dataset. Default: None, expected order behavior shown in the table below.
|
|
59
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None, no dividing.
|
|
59
60
|
When this argument is specified, `num_samples` reflects the max sample number of per shard.
|
|
60
|
-
shard_id (int, optional): The shard ID within `num_shards`
|
|
61
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: None, will use 0. This
|
|
61
62
|
argument can only be specified when `num_shards` is also specified.
|
|
62
63
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
63
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/
|
|
64
|
-
|
|
64
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.0/dataset/cache.html>`_ .
|
|
65
|
+
Default: None, which means no cache is used.
|
|
65
66
|
|
|
66
67
|
Raises:
|
|
67
|
-
RuntimeError: If source raises an exception during execution.
|
|
68
68
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
69
69
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
70
70
|
RuntimeError: If `sampler` and `shuffle` are specified at the same time.
|
|
71
71
|
RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time.
|
|
72
72
|
RuntimeError: If `num_shards` is specified but `shard_id` is None.
|
|
73
73
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
74
|
-
ValueError: If `shard_id` is
|
|
74
|
+
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
75
75
|
|
|
76
76
|
Note:
|
|
77
|
-
-
|
|
78
|
-
- This dataset can take in a `sampler
|
|
77
|
+
- Not support `mindspore.dataset.PKSampler` for `sampler` parameter yet.
|
|
78
|
+
- This dataset can take in a `sampler` . `sampler` and `shuffle` are mutually exclusive.
|
|
79
79
|
The table below shows what input arguments are allowed and their expected behavior.
|
|
80
80
|
|
|
81
|
-
.. list-table:: Expected Order Behavior of Using
|
|
81
|
+
.. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`
|
|
82
82
|
:widths: 25 25 50
|
|
83
83
|
:header-rows: 1
|
|
84
84
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
85
|
+
* - Parameter `sampler`
|
|
86
|
+
- Parameter `shuffle`
|
|
87
|
+
- Expected Order Behavior
|
|
88
|
+
* - None
|
|
89
|
+
- None
|
|
90
|
+
- random order
|
|
91
|
+
* - None
|
|
92
|
+
- True
|
|
93
|
+
- random order
|
|
94
|
+
* - None
|
|
95
|
+
- False
|
|
96
|
+
- sequential order
|
|
97
|
+
* - Sampler object
|
|
98
|
+
- None
|
|
99
|
+
- order defined by sampler
|
|
100
|
+
* - Sampler object
|
|
101
|
+
- True
|
|
102
|
+
- not allowed
|
|
103
|
+
* - Sampler object
|
|
104
|
+
- False
|
|
105
|
+
- not allowed
|
|
106
106
|
|
|
107
107
|
Examples:
|
|
108
108
|
>>> cmu_arctic_dataset_directory = "/path/to/cmu_arctic_dataset_directory"
|
|
@@ -115,7 +115,7 @@ class CMUArcticDataset(MappableDataset, AudioBaseDataset):
|
|
|
115
115
|
|
|
116
116
|
About CMUArctic dataset:
|
|
117
117
|
|
|
118
|
-
The CMU
|
|
118
|
+
The CMU Arctic databases are designed for the purpose of speech synthesis research.
|
|
119
119
|
These single speaker speech databases have been carefully recorded under studio conditions
|
|
120
120
|
and consist of approximately 1200 phonetically balanced English utterances. In addition to wavefiles,
|
|
121
121
|
the databases provide complete support for the Festival Speech Synthesis System, including pre-built
|
|
@@ -172,49 +172,49 @@ class CMUArcticDataset(MappableDataset, AudioBaseDataset):
|
|
|
172
172
|
|
|
173
173
|
class GTZANDataset(MappableDataset, AudioBaseDataset):
|
|
174
174
|
"""
|
|
175
|
-
|
|
175
|
+
GTZAN dataset.
|
|
176
176
|
|
|
177
|
-
The generated dataset has three columns: :py:obj:`[
|
|
177
|
+
The generated dataset has three columns: :py:obj:`[waveform, sample_rate, label]` .
|
|
178
178
|
The tensor of column :py:obj:`waveform` is of the float32 type.
|
|
179
179
|
The tensor of column :py:obj:`sample_rate` is of a scalar of uint32 type.
|
|
180
180
|
The tensor of column :py:obj:`label` is of a scalar of string type.
|
|
181
181
|
|
|
182
182
|
Args:
|
|
183
183
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
184
|
-
usage (str, optional): Usage of this dataset, can be 'train', 'valid', 'test' or 'all'
|
|
185
|
-
|
|
186
|
-
num_samples (int, optional): The number of audio to be included in the dataset
|
|
187
|
-
|
|
188
|
-
num_parallel_workers (int, optional): Number of
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
184
|
+
usage (str, optional): Usage of this dataset, can be 'train', 'valid', 'test' or 'all'.
|
|
185
|
+
Default: None, will read all samples.
|
|
186
|
+
num_samples (int, optional): The number of audio to be included in the dataset.
|
|
187
|
+
Default: None, will read all audio.
|
|
188
|
+
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
189
|
+
Default: None, will use global default workers(8), it can be set
|
|
190
|
+
by `mindspore.dataset.config.set_num_parallel_workers` .
|
|
191
|
+
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
192
|
+
Default: None, expected order behavior shown in the table below.
|
|
192
193
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
193
|
-
dataset
|
|
194
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into
|
|
194
|
+
dataset. Default: None, expected order behavior shown in the table below.
|
|
195
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
195
196
|
When this argument is specified, `num_samples` reflects the max sample number of per shard.
|
|
196
|
-
shard_id (int, optional): The shard ID within `num_shards`
|
|
197
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
197
198
|
argument can only be specified when `num_shards` is also specified.
|
|
198
199
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
199
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/
|
|
200
|
-
|
|
200
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.0/dataset/cache.html>`_ .
|
|
201
|
+
Default: None, which means no cache is used.
|
|
201
202
|
|
|
202
203
|
Raises:
|
|
203
|
-
RuntimeError: If source raises an exception during execution.
|
|
204
204
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
205
205
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
206
206
|
RuntimeError: If `sampler` and `shuffle` are specified at the same time.
|
|
207
207
|
RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time.
|
|
208
208
|
RuntimeError: If `num_shards` is specified but `shard_id` is None.
|
|
209
209
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
210
|
-
ValueError: If `shard_id` is
|
|
210
|
+
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
211
211
|
|
|
212
212
|
Note:
|
|
213
|
-
-
|
|
214
|
-
- This dataset can take in a `sampler
|
|
213
|
+
- Not support `mindspore.dataset.PKSampler` for `sampler` parameter yet.
|
|
214
|
+
- This dataset can take in a `sampler` . `sampler` and `shuffle` are mutually exclusive.
|
|
215
215
|
The table below shows what input arguments are allowed and their expected behavior.
|
|
216
216
|
|
|
217
|
-
.. list-table:: Expected Order Behavior of Using
|
|
217
|
+
.. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`
|
|
218
218
|
:widths: 25 25 50
|
|
219
219
|
:header-rows: 1
|
|
220
220
|
|
|
@@ -254,7 +254,7 @@ class GTZANDataset(MappableDataset, AudioBaseDataset):
|
|
|
254
254
|
The GTZAN dataset appears in at least 100 published works and is the most commonly used
|
|
255
255
|
public dataset for evaluation in machine listening research for music genre recognition.
|
|
256
256
|
It consists of 1000 audio tracks, each of which is 30 seconds long. It contains 10 genres (blues,
|
|
257
|
-
classical, country, disco, hiphop, jazz, metal, pop, reggae and
|
|
257
|
+
classical, country, disco, hiphop, jazz, metal, pop, reggae and rock), each of which is
|
|
258
258
|
represented by 100 tracks. The tracks are all 22050Hz Mono 16-bit audio files in .wav format.
|
|
259
259
|
|
|
260
260
|
You can construct the following directory structure from GTZAN dataset and read by MindSpore's API.
|
|
@@ -303,10 +303,10 @@ class GTZANDataset(MappableDataset, AudioBaseDataset):
|
|
|
303
303
|
|
|
304
304
|
class LibriTTSDataset(MappableDataset, AudioBaseDataset):
|
|
305
305
|
"""
|
|
306
|
-
|
|
306
|
+
LibriTTS dataset.
|
|
307
307
|
|
|
308
|
-
The generated dataset has seven columns :py:obj:`[
|
|
309
|
-
|
|
308
|
+
The generated dataset has seven columns :py:obj:`[waveform, sample_rate, original_text, normalized_text,
|
|
309
|
+
speaker_id, chapter_id, utterance_id]` .
|
|
310
310
|
The tensor of column :py:obj:`waveform` is of the float32 type.
|
|
311
311
|
The tensor of column :py:obj:`sample_rate` is of a scalar of uint32 type.
|
|
312
312
|
The tensor of column :py:obj:`original_text` is of a scalar of string type.
|
|
@@ -318,39 +318,39 @@ class LibriTTSDataset(MappableDataset, AudioBaseDataset):
|
|
|
318
318
|
Args:
|
|
319
319
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
320
320
|
usage (str, optional): Part of this dataset, can be 'dev-clean', 'dev-other', 'test-clean', 'test-other',
|
|
321
|
-
'train-clean-100', 'train-clean-360', 'train-other-500', or 'all'
|
|
322
|
-
num_samples (int, optional): The number of images to be included in the dataset
|
|
323
|
-
|
|
324
|
-
num_parallel_workers (int, optional): Number of
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
321
|
+
'train-clean-100', 'train-clean-360', 'train-other-500', or 'all'. Default: None, means 'all'.
|
|
322
|
+
num_samples (int, optional): The number of images to be included in the dataset.
|
|
323
|
+
Default: None, will read all audio.
|
|
324
|
+
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
325
|
+
Default: None, will use global default workers(8), it can be set
|
|
326
|
+
by `mindspore.dataset.config.set_num_parallel_workers` .
|
|
327
|
+
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
328
|
+
Default: None, expected order behavior shown in the table below.
|
|
328
329
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
329
|
-
dataset
|
|
330
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into
|
|
330
|
+
dataset. Default: None, expected order behavior shown in the table below.
|
|
331
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
331
332
|
When this argument is specified, `num_samples` reflects the max sample number of per shard.
|
|
332
|
-
shard_id (int, optional): The shard ID within `num_shards`
|
|
333
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
333
334
|
argument can only be specified when `num_shards` is also specified.
|
|
334
335
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
335
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/
|
|
336
|
-
|
|
336
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.0/dataset/cache.html>`_ .
|
|
337
|
+
Default: None, which means no cache is used.
|
|
337
338
|
|
|
338
339
|
Raises:
|
|
339
|
-
RuntimeError: If source raises an exception during execution.
|
|
340
340
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
341
341
|
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
342
342
|
RuntimeError: If `sampler` and `shuffle` are specified at the same time.
|
|
343
343
|
RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time.
|
|
344
344
|
RuntimeError: If `num_shards` is specified but `shard_id` is None.
|
|
345
345
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
346
|
-
ValueError: If `shard_id` is
|
|
346
|
+
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
347
347
|
|
|
348
348
|
Note:
|
|
349
|
-
-
|
|
350
|
-
- This dataset can take in a `sampler
|
|
349
|
+
- Not support `mindspore.dataset.PKSampler` for `sampler` parameter yet.
|
|
350
|
+
- This dataset can take in a `sampler` . `sampler` and `shuffle` are mutually exclusive.
|
|
351
351
|
The table below shows what input arguments are allowed and their expected behavior.
|
|
352
352
|
|
|
353
|
-
.. list-table:: Expected Order Behavior of Using
|
|
353
|
+
.. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`
|
|
354
354
|
:widths: 25 25 50
|
|
355
355
|
:header-rows: 1
|
|
356
356
|
|
|
@@ -454,44 +454,45 @@ class LibriTTSDataset(MappableDataset, AudioBaseDataset):
|
|
|
454
454
|
|
|
455
455
|
class LJSpeechDataset(MappableDataset, AudioBaseDataset):
|
|
456
456
|
"""
|
|
457
|
-
|
|
457
|
+
LJSpeech dataset.
|
|
458
458
|
|
|
459
|
-
The generated dataset has four columns :py:obj:`[waveform, sample_rate, transcription, normalized_transcript]
|
|
460
|
-
The
|
|
461
|
-
The
|
|
462
|
-
The
|
|
463
|
-
The
|
|
459
|
+
The generated dataset has four columns :py:obj:`[waveform, sample_rate, transcription, normalized_transcript]` .
|
|
460
|
+
The column :py:obj:`waveform` is a tensor of the float32 type.
|
|
461
|
+
The column :py:obj:`sample_rate` is a scalar of the int32 type.
|
|
462
|
+
The column :py:obj:`transcription` is a scalar of the string type.
|
|
463
|
+
The column :py:obj:`normalized_transcript` is a scalar of the string type.
|
|
464
464
|
|
|
465
465
|
Args:
|
|
466
466
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
467
|
-
num_samples (int, optional): The number of audios to be included in the dataset
|
|
468
|
-
|
|
469
|
-
num_parallel_workers (int, optional): Number of
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
467
|
+
num_samples (int, optional): The number of audios to be included in the dataset.
|
|
468
|
+
Default: None, all audios.
|
|
469
|
+
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
470
|
+
Default: None, will use global default workers(8), it can be set
|
|
471
|
+
by `mindspore.dataset.config.set_num_parallel_workers` .
|
|
472
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None, expected
|
|
473
|
+
order behavior shown in the table below.
|
|
474
|
+
sampler (Sampler, optional): Object used to choose samples from the dataset.
|
|
475
|
+
Default: None, expected order behavior shown in the table below.
|
|
476
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into.
|
|
477
|
+
Default: None. When this argument is specified, `num_samples` reflects
|
|
477
478
|
the maximum sample number of per shard.
|
|
478
|
-
shard_id (int, optional): The shard ID within `num_shards`
|
|
479
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
479
480
|
argument can only be specified when `num_shards` is also specified.
|
|
480
481
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
481
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/
|
|
482
|
-
|
|
482
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.0/dataset/cache.html>`_ .
|
|
483
|
+
Default: None, which means no cache is used.
|
|
483
484
|
|
|
484
485
|
Raises:
|
|
485
486
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
486
|
-
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
487
487
|
RuntimeError: If `sampler` and `shuffle` are specified at the same time.
|
|
488
488
|
RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time.
|
|
489
489
|
RuntimeError: If `num_shards` is specified but `shard_id` is None.
|
|
490
490
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
491
|
-
ValueError: If `
|
|
491
|
+
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
492
|
+
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
492
493
|
|
|
493
494
|
Note:
|
|
494
|
-
- This dataset can take in a `sampler
|
|
495
|
+
- This dataset can take in a `sampler` . `sampler` and `shuffle` are mutually exclusive.
|
|
495
496
|
The table below shows what input arguments are allowed and their expected behavior.
|
|
496
497
|
|
|
497
498
|
.. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`
|
|
@@ -591,9 +592,9 @@ class LJSpeechDataset(MappableDataset, AudioBaseDataset):
|
|
|
591
592
|
|
|
592
593
|
class SpeechCommandsDataset(MappableDataset, AudioBaseDataset):
|
|
593
594
|
"""
|
|
594
|
-
|
|
595
|
+
Speech Commands dataset.
|
|
595
596
|
|
|
596
|
-
The generated dataset has five columns :py:obj:`[waveform, sample_rate, label, speaker_id, utterance_number]
|
|
597
|
+
The generated dataset has five columns :py:obj:`[waveform, sample_rate, label, speaker_id, utterance_number]` .
|
|
597
598
|
The tensor of column :py:obj:`waveform` is a vector of the float32 type.
|
|
598
599
|
The tensor of column :py:obj:`sample_rate` is a scalar of the int32 type.
|
|
599
600
|
The tensor of column :py:obj:`label` is a scalar of the string type.
|
|
@@ -604,34 +605,35 @@ class SpeechCommandsDataset(MappableDataset, AudioBaseDataset):
|
|
|
604
605
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
605
606
|
usage (str, optional): Usage of this dataset, can be 'train', 'test', 'valid' or 'all'. 'train'
|
|
606
607
|
will read from 84,843 samples, 'test' will read from 11,005 samples, 'valid' will read from 9,981
|
|
607
|
-
test samples and 'all' will read from all 105,829 samples
|
|
608
|
-
num_samples (int, optional): The number of samples to be included in the dataset
|
|
609
|
-
|
|
610
|
-
num_parallel_workers (int, optional): Number of
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
608
|
+
test samples and 'all' will read from all 105,829 samples. Default: None, will read all samples.
|
|
609
|
+
num_samples (int, optional): The number of samples to be included in the dataset.
|
|
610
|
+
Default: None, will read all samples.
|
|
611
|
+
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
612
|
+
Default: None, will use global default workers(8), it can be set
|
|
613
|
+
by `mindspore.dataset.config.set_num_parallel_workers` .
|
|
614
|
+
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
615
|
+
Default: None, expected order behavior shown in the table below.
|
|
616
|
+
sampler (Sampler, optional): Object used to choose samples from the dataset.
|
|
617
|
+
Default: None, expected order behavior shown in the table below.
|
|
618
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
617
619
|
When this argument is specified, `num_samples` reflects the maximum sample number of per shard.
|
|
618
|
-
shard_id (int, optional): The shard ID within `num_shards`
|
|
620
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This argument can only be specified
|
|
619
621
|
when `num_shards` is also specified.
|
|
620
622
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
621
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/
|
|
622
|
-
|
|
623
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.0/dataset/cache.html>`_ .
|
|
624
|
+
Default: None, which means no cache is used.
|
|
623
625
|
|
|
624
626
|
Raises:
|
|
625
627
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
626
|
-
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
627
628
|
RuntimeError: If `sampler` and `shuffle` are specified at the same time.
|
|
628
629
|
RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time.
|
|
629
630
|
RuntimeError: If `num_shards` is specified but `shard_id` is None.
|
|
630
631
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
631
|
-
ValueError: If `
|
|
632
|
+
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
633
|
+
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
632
634
|
|
|
633
635
|
Note:
|
|
634
|
-
- This dataset can take in a `sampler
|
|
636
|
+
- This dataset can take in a `sampler` . `sampler` and `shuffle` are mutually exclusive.
|
|
635
637
|
The table below shows what input arguments are allowed and their expected behavior.
|
|
636
638
|
|
|
637
639
|
.. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`
|
|
@@ -665,9 +667,6 @@ class SpeechCommandsDataset(MappableDataset, AudioBaseDataset):
|
|
|
665
667
|
>>>
|
|
666
668
|
>>> # Read 3 samples from SpeechCommands dataset
|
|
667
669
|
>>> dataset = ds.SpeechCommandsDataset(dataset_dir=speech_commands_dataset_dir, num_samples=3)
|
|
668
|
-
>>>
|
|
669
|
-
>>> # Note: In SpeechCommands dataset, each dictionary has keys "waveform", "sample_rate", "label",
|
|
670
|
-
>>> # "speaker_id" and "utterance_number".
|
|
671
670
|
|
|
672
671
|
About SpeechCommands dataset:
|
|
673
672
|
|
|
@@ -717,18 +716,14 @@ class SpeechCommandsDataset(MappableDataset, AudioBaseDataset):
|
|
|
717
716
|
|
|
718
717
|
class TedliumDataset(MappableDataset, AudioBaseDataset):
|
|
719
718
|
"""
|
|
720
|
-
|
|
719
|
+
Tedlium dataset.
|
|
721
720
|
The columns of generated dataset depend on the source SPH files and the corresponding STM files.
|
|
722
721
|
|
|
723
722
|
The generated dataset has six columns :py:obj:`[waveform, sample_rate, transcript, talk_id, speaker_id,
|
|
724
|
-
identifier]
|
|
723
|
+
identifier]` .
|
|
725
724
|
|
|
726
|
-
The
|
|
727
|
-
|
|
728
|
-
The tensor of column :py:obj:`transcript` is a scalar of the string type.
|
|
729
|
-
The tensor of column :py:obj:`talk_id` is a scalar of the string type.
|
|
730
|
-
The tensor of column :py:obj:`speaker_id` is a scalar of the string type.
|
|
731
|
-
The tensor of column :py:obj:`identifier` is a scalar of the string type.
|
|
725
|
+
The data type of column `waveform` is float32, the data type of column `sample_rate` is int32,
|
|
726
|
+
and the data type of columns `transcript` , `talk_id` , `speaker_id` and `identifier` is string.
|
|
732
727
|
|
|
733
728
|
Args:
|
|
734
729
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
@@ -739,37 +734,38 @@ class TedliumDataset(MappableDataset, AudioBaseDataset):
|
|
|
739
734
|
'test' will read from test samples,
|
|
740
735
|
'dev' will read from dev samples,
|
|
741
736
|
'all' will read from all samples.
|
|
742
|
-
For release3, can only be 'all', it will read from data samples
|
|
737
|
+
For release3, can only be 'all', it will read from data samples. Default: None, all samples.
|
|
743
738
|
extensions (str, optional): Extensions of the SPH files, only '.sph' is valid.
|
|
744
|
-
|
|
745
|
-
num_samples (int, optional): The number of audio samples to be included in the dataset
|
|
746
|
-
|
|
747
|
-
num_parallel_workers (int, optional): Number of
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
739
|
+
Default: None, ".sph".
|
|
740
|
+
num_samples (int, optional): The number of audio samples to be included in the dataset.
|
|
741
|
+
Default: None, all samples.
|
|
742
|
+
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
743
|
+
Default: None, will use global default workers(8), it can be set
|
|
744
|
+
by `mindspore.dataset.config.set_num_parallel_workers` .
|
|
745
|
+
shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: None, expected
|
|
746
|
+
order behavior shown in the table below.
|
|
751
747
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
752
|
-
dataset
|
|
748
|
+
dataset. Default: None, expected order behavior shown in the table below.
|
|
753
749
|
num_shards (int, optional): Number of shards that the dataset will be divided
|
|
754
|
-
into
|
|
750
|
+
into. Default: None. When this argument is specified, `num_samples` reflects
|
|
755
751
|
the maximum sample number of per shard.
|
|
756
|
-
shard_id (int, optional): The shard ID within `num_shards`
|
|
752
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This
|
|
757
753
|
argument can only be specified when `num_shards` is also specified.
|
|
758
754
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
759
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/
|
|
760
|
-
|
|
755
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.0/dataset/cache.html>`_ .
|
|
756
|
+
Default: None, which means no cache is used.
|
|
761
757
|
|
|
762
758
|
Raises:
|
|
763
759
|
RuntimeError: If `dataset_dir` does not contain stm files.
|
|
764
|
-
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
765
760
|
RuntimeError: If `sampler` and `shuffle` are specified at the same time.
|
|
766
761
|
RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time.
|
|
767
762
|
RuntimeError: If `num_shards` is specified but `shard_id` is None.
|
|
768
763
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
769
|
-
ValueError: If `
|
|
764
|
+
ValueError: If `num_parallel_workers` exceeds the max thread numbers.
|
|
765
|
+
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
770
766
|
|
|
771
767
|
Note:
|
|
772
|
-
- This dataset can take in a `sampler
|
|
768
|
+
- This dataset can take in a `sampler` . `sampler` and `shuffle` are mutually exclusive.
|
|
773
769
|
The table below shows what input arguments are allowed and their expected behavior.
|
|
774
770
|
|
|
775
771
|
.. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`
|
|
@@ -865,6 +861,8 @@ class TedliumDataset(MappableDataset, AudioBaseDataset):
|
|
|
865
861
|
└── readme
|
|
866
862
|
└── TEDLIUM.150k.dic
|
|
867
863
|
|
|
864
|
+
The directory structure of TEDLIUM release3 is slightly different.
|
|
865
|
+
|
|
868
866
|
.. code-block::
|
|
869
867
|
|
|
870
868
|
.
|
|
@@ -930,30 +928,31 @@ class TedliumDataset(MappableDataset, AudioBaseDataset):
|
|
|
930
928
|
|
|
931
929
|
class YesNoDataset(MappableDataset, AudioBaseDataset):
|
|
932
930
|
"""
|
|
933
|
-
|
|
931
|
+
YesNo dataset.
|
|
934
932
|
|
|
935
|
-
The generated dataset has three columns :py:obj:`[waveform, sample_rate, labels]
|
|
933
|
+
The generated dataset has three columns :py:obj:`[waveform, sample_rate, labels]` .
|
|
936
934
|
The tensor of column :py:obj:`waveform` is a vector of the float32 type.
|
|
937
935
|
The tensor of column :py:obj:`sample_rate` is a scalar of the int32 type.
|
|
938
936
|
The tensor of column :py:obj:`labels` is a scalar of the int32 type.
|
|
939
937
|
|
|
940
938
|
Args:
|
|
941
939
|
dataset_dir (str): Path to the root directory that contains the dataset.
|
|
942
|
-
num_samples (int, optional): The number of images to be included in the dataset
|
|
943
|
-
|
|
944
|
-
num_parallel_workers (int, optional): Number of
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
940
|
+
num_samples (int, optional): The number of images to be included in the dataset.
|
|
941
|
+
Default: None, will read all images.
|
|
942
|
+
num_parallel_workers (int, optional): Number of worker threads to read the data.
|
|
943
|
+
Default: None, will use global default workers(8), it can be set
|
|
944
|
+
by `mindspore.dataset.config.set_num_parallel_workers` .
|
|
945
|
+
shuffle (bool, optional): Whether or not to perform shuffle on the dataset.
|
|
946
|
+
Default: None, expected order behavior shown in the table below.
|
|
948
947
|
sampler (Sampler, optional): Object used to choose samples from the
|
|
949
|
-
dataset
|
|
950
|
-
num_shards (int, optional): Number of shards that the dataset will be divided into
|
|
948
|
+
dataset. Default: None, expected order behavior shown in the table below.
|
|
949
|
+
num_shards (int, optional): Number of shards that the dataset will be divided into. Default: None.
|
|
951
950
|
When this argument is specified, `num_samples` reflects the maximum sample number of per shard.
|
|
952
|
-
shard_id (int, optional): The shard ID within `num_shards`
|
|
951
|
+
shard_id (int, optional): The shard ID within `num_shards` . Default: None. This argument can only
|
|
953
952
|
be specified when `num_shards` is also specified.
|
|
954
953
|
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details:
|
|
955
|
-
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/
|
|
956
|
-
|
|
954
|
+
`Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/r2.0/dataset/cache.html>`_ .
|
|
955
|
+
Default: None, which means no cache is used.
|
|
957
956
|
|
|
958
957
|
Raises:
|
|
959
958
|
RuntimeError: If `dataset_dir` does not contain data files.
|
|
@@ -962,10 +961,10 @@ class YesNoDataset(MappableDataset, AudioBaseDataset):
|
|
|
962
961
|
RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time.
|
|
963
962
|
RuntimeError: If `num_shards` is specified but `shard_id` is None.
|
|
964
963
|
RuntimeError: If `shard_id` is specified but `num_shards` is None.
|
|
965
|
-
ValueError: If `shard_id` is
|
|
964
|
+
ValueError: If `shard_id` is not in range of [0, `num_shards` ).
|
|
966
965
|
|
|
967
966
|
Note:
|
|
968
|
-
- This dataset can take in a `sampler
|
|
967
|
+
- This dataset can take in a `sampler` . `sampler` and `shuffle` are mutually exclusive.
|
|
969
968
|
The table below shows what input arguments are allowed and their expected behavior.
|
|
970
969
|
|
|
971
970
|
.. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`
|
|
@@ -1005,8 +1004,7 @@ class YesNoDataset(MappableDataset, AudioBaseDataset):
|
|
|
1005
1004
|
About YesNo dataset:
|
|
1006
1005
|
|
|
1007
1006
|
Yesno is an audio dataset consisting of 60 recordings of one individual saying yes or no in Hebrew; each
|
|
1008
|
-
recording is eight words long.
|
|
1009
|
-
remain anonymous.
|
|
1007
|
+
recording is eight words long.
|
|
1010
1008
|
|
|
1011
1009
|
Here is the original YesNo dataset structure.
|
|
1012
1010
|
You can unzip the dataset files into this directory structure and read by MindSpore's API.
|