mindspore 2.0.0rc1__cp38-none-any.whl → 2.2.0__cp38-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mindspore might be problematic. Click here for more details.
- mindspore/.commit_id +1 -1
- mindspore/Third_Party_Open_Source_Software_Notice +2 -2
- mindspore/__init__.py +5 -2
- mindspore/_akg/akg/build_module.py +5 -6
- mindspore/_akg/akg/composite/build_module.py +49 -16
- mindspore/_akg/akg/composite/split_stitch.py +10 -11
- mindspore/_akg/akg/config/repository.json +195 -0
- mindspore/_akg/akg/global_configs.py +5 -1
- mindspore/_akg/akg/ms/info_version_adapt.py +67 -1
- mindspore/_akg/akg/tvm/api.py +4 -3
- mindspore/_akg/akg/tvm/autotvm/__init__.py +1 -2
- mindspore/_akg/akg/tvm/autotvm/graph_tuner/base_graph_tuner.py +1 -5
- mindspore/_akg/akg/tvm/autotvm/measure/__init__.py +1 -1
- mindspore/_akg/akg/tvm/autotvm/measure/measure.py +1 -10
- mindspore/_akg/akg/tvm/autotvm/measure/measure_methods.py +1 -372
- mindspore/_akg/akg/tvm/build_module.py +16 -1
- mindspore/_akg/akg/tvm/contrib/graph_runtime.py +0 -53
- mindspore/_akg/akg/tvm/hybrid/parser.py +7 -6
- mindspore/_akg/akg/tvm/ir_builder.py +1 -1
- mindspore/_akg/akg/tvm/module.py +1 -2
- mindspore/_akg/akg/tvm/stmt.py +2 -2
- mindspore/_akg/akg/utils/composite_op_helper.py +9 -10
- mindspore/_akg/akg/utils/kernel_exec.py +58 -260
- mindspore/_akg/akg/utils/op_dsl.py +17 -1
- mindspore/_akg/akg/utils/result_analysis.py +4 -24
- mindspore/_akg/akg/utils/tbe_codegen_utils.py +198 -0
- mindspore/_c_dataengine.cpython-38-aarch64-linux-gnu.so +0 -0
- mindspore/_c_expression.cpython-38-aarch64-linux-gnu.so +0 -0
- mindspore/_c_mindrecord.cpython-38-aarch64-linux-gnu.so +0 -0
- mindspore/_check_jit_forbidden_api.py +5 -1
- mindspore/_checkparam.py +79 -62
- mindspore/_extends/graph_kernel/__init__.py +0 -1
- mindspore/_extends/graph_kernel/model/graph_split.py +2 -0
- mindspore/_extends/graph_kernel/model/model_builder.py +9 -50
- mindspore/_extends/graph_kernel/splitter.py +1 -9
- mindspore/_extends/parallel_compile/akg_compiler/akg_process.py +128 -21
- mindspore/_extends/parallel_compile/akg_compiler/build_tbe_kernel.py +2 -2
- mindspore/_extends/parallel_compile/akg_compiler/tbe_topi.py +4 -2
- mindspore/_extends/parallel_compile/tbe_compiler/tbe_adapter.py +18 -13
- mindspore/_extends/parallel_compile/tbe_compiler/tbe_helper.py +13 -9
- mindspore/_extends/parallel_compile/tbe_compiler/tbe_job.py +1 -1
- mindspore/_extends/parallel_compile/tbe_compiler/tbe_job_manager.py +1 -1
- mindspore/_extends/parse/__init__.py +19 -17
- mindspore/_extends/parse/namespace.py +7 -36
- mindspore/_extends/parse/parser.py +375 -189
- mindspore/_extends/parse/resources.py +36 -41
- mindspore/_extends/parse/standard_method.py +350 -245
- mindspore/_extends/parse/trope.py +2 -12
- mindspore/_extends/remote/kernel_build_server.py +24 -7
- mindspore/_extends/remote/kernel_build_server_akg_v2.py +55 -0
- mindspore/_install_custom.py +43 -0
- mindspore/_mindspore_offline_debug.cpython-38-aarch64-linux-gnu.so +0 -0
- mindspore/amp.py +85 -19
- mindspore/bin/cache_admin +0 -0
- mindspore/bin/cache_server +0 -0
- mindspore/boost/base.py +2 -2
- mindspore/boost/boost.py +27 -32
- mindspore/boost/boost_cell_wrapper.py +37 -13
- mindspore/boost/grad_accumulation.py +1 -1
- mindspore/boost/grad_freeze.py +34 -6
- mindspore/boost/group_loss_scale_manager.py +15 -14
- mindspore/boost/less_batch_normalization.py +28 -3
- mindspore/common/__init__.py +15 -11
- mindspore/common/_auto_dynamic.py +68 -0
- mindspore/common/_jit_fallback_utils.py +111 -0
- mindspore/common/_register_for_adapter.py +17 -5
- mindspore/common/_register_for_tensor.py +2 -2
- mindspore/common/_stub_tensor.py +18 -15
- mindspore/common/_utils.py +31 -7
- mindspore/common/api.py +269 -101
- mindspore/common/auto_dynamic_shape.py +498 -0
- mindspore/common/dtype.py +61 -21
- mindspore/common/dump.py +9 -7
- mindspore/common/initializer.py +106 -76
- mindspore/common/jit_config.py +35 -14
- mindspore/common/lazy_inline.py +187 -0
- mindspore/common/mindir_util.py +101 -0
- mindspore/common/mutable.py +10 -13
- mindspore/common/parameter.py +246 -55
- mindspore/common/seed.py +13 -7
- mindspore/common/sparse_tensor.py +29 -33
- mindspore/common/tensor.py +907 -251
- mindspore/communication/__init__.py +7 -4
- mindspore/communication/_comm_helper.py +84 -4
- mindspore/communication/management.py +160 -88
- mindspore/config/op_info.config +99 -75
- mindspore/config/super_bar_config.json +36 -4
- mindspore/context.py +526 -219
- mindspore/dataset/__init__.py +9 -46
- mindspore/dataset/audio/__init__.py +4 -19
- mindspore/dataset/audio/transforms.py +545 -233
- mindspore/dataset/audio/utils.py +21 -18
- mindspore/dataset/callback/ds_callback.py +42 -13
- mindspore/dataset/core/config.py +158 -100
- mindspore/dataset/core/validator_helpers.py +1 -63
- mindspore/dataset/debug/debug_hook.py +45 -13
- mindspore/dataset/debug/pre_defined_hook.py +5 -5
- mindspore/dataset/engine/__init__.py +0 -5
- mindspore/dataset/engine/cache_client.py +38 -15
- mindspore/dataset/engine/datasets.py +615 -278
- mindspore/dataset/engine/datasets_audio.py +154 -283
- mindspore/dataset/engine/datasets_standard_format.py +104 -116
- mindspore/dataset/engine/datasets_text.py +443 -326
- mindspore/dataset/engine/datasets_user_defined.py +251 -164
- mindspore/dataset/engine/datasets_vision.py +839 -1443
- mindspore/dataset/engine/iterators.py +11 -4
- mindspore/dataset/engine/obs/obs_mindrecord_dataset.py +7 -3
- mindspore/dataset/engine/obs/util.py +3 -0
- mindspore/dataset/engine/offload.py +6 -6
- mindspore/dataset/engine/queue.py +15 -14
- mindspore/dataset/engine/samplers.py +39 -23
- mindspore/dataset/engine/serializer_deserializer.py +22 -6
- mindspore/dataset/engine/validators.py +21 -331
- mindspore/dataset/text/__init__.py +5 -33
- mindspore/dataset/text/transforms.py +334 -165
- mindspore/dataset/text/utils.py +215 -145
- mindspore/dataset/transforms/__init__.py +1 -1
- mindspore/dataset/transforms/c_transforms.py +3 -2
- mindspore/dataset/transforms/py_transforms_util.py +40 -12
- mindspore/dataset/transforms/transforms.py +174 -71
- mindspore/dataset/utils/browse_dataset.py +25 -17
- mindspore/dataset/utils/line_reader.py +24 -21
- mindspore/dataset/vision/__init__.py +5 -26
- mindspore/dataset/vision/c_transforms.py +177 -165
- mindspore/dataset/vision/py_transforms.py +114 -119
- mindspore/dataset/vision/py_transforms_util.py +54 -51
- mindspore/dataset/vision/transforms.py +1127 -381
- mindspore/dataset/vision/utils.py +54 -38
- mindspore/dataset/vision/validators.py +12 -2
- mindspore/experimental/map_parameter.py +38 -4
- mindspore/{dataset/datapreprocess → experimental/optim}/__init__.py +14 -4
- mindspore/experimental/optim/adam.py +192 -0
- mindspore/experimental/optim/adamw.py +181 -0
- mindspore/experimental/optim/lr_scheduler.py +1427 -0
- mindspore/experimental/optim/optimizer.py +252 -0
- mindspore/experimental/optim/sgd.py +147 -0
- mindspore/gen_ops.py +273 -0
- mindspore/include/OWNERS +1 -2
- mindspore/include/api/context.h +21 -1
- mindspore/include/api/data_type.h +2 -1
- mindspore/include/api/graph.h +0 -15
- mindspore/include/api/kernel.h +2 -0
- mindspore/include/api/kernel_api.h +37 -12
- mindspore/include/api/model.h +29 -42
- mindspore/include/api/model_group.h +14 -3
- mindspore/include/api/model_parallel_runner.h +18 -2
- mindspore/include/api/serialization.h +26 -0
- mindspore/include/api/status.h +1 -0
- mindspore/include/api/types.h +38 -4
- mindspore/include/c_api/ms/abstract.h +67 -0
- mindspore/include/c_api/ms/attribute.h +197 -0
- mindspore/include/c_api/ms/base/handle_types.h +43 -0
- mindspore/include/c_api/ms/base/macros.h +32 -0
- mindspore/include/c_api/ms/base/status.h +33 -0
- mindspore/include/c_api/ms/base/types.h +282 -0
- mindspore/include/c_api/ms/context.h +102 -0
- mindspore/include/c_api/ms/graph.h +160 -0
- mindspore/include/c_api/ms/node.h +606 -0
- mindspore/include/c_api/ms/tensor.h +161 -0
- mindspore/include/c_api/ms/value.h +84 -0
- mindspore/include/c_api/status_c.h +3 -0
- mindspore/include/dataset/constants.h +6 -12
- mindspore/include/dataset/execute.h +23 -13
- mindspore/include/dataset/text.h +26 -26
- mindspore/include/dataset/transforms.h +25 -31
- mindspore/include/dataset/vision.h +60 -60
- mindspore/include/dataset/vision_ascend.h +5 -6
- mindspore/include/dataset/vision_lite.h +17 -17
- mindspore/include/mindapi/base/format.h +0 -1
- mindspore/include/mindapi/base/type_id.h +2 -1
- mindspore/include/mindapi/base/types.h +5 -1
- mindspore/lib/libdnnl.so.2 +0 -0
- mindspore/lib/libjemalloc.so.2 +0 -0
- mindspore/lib/libmindspore.so +0 -0
- mindspore/lib/libmindspore_backend.so +0 -0
- mindspore/lib/libmindspore_common.so +0 -0
- mindspore/lib/libmindspore_core.so +0 -0
- mindspore/lib/libmindspore_glog.so.0 +0 -0
- mindspore/lib/libmindspore_gpr.so.15 +0 -0
- mindspore/lib/libmindspore_grpc++.so.1 +0 -0
- mindspore/lib/libmindspore_grpc.so.15 +0 -0
- mindspore/lib/libmindspore_shared_lib.so +0 -0
- mindspore/lib/libmpi_adapter.so +0 -0
- mindspore/lib/libnnacl.so +0 -0
- mindspore/lib/libopencv_core.so.4.5 +0 -0
- mindspore/lib/libopencv_imgcodecs.so.4.5 +0 -0
- mindspore/lib/libopencv_imgproc.so.4.5 +0 -0
- mindspore/lib/libps_cache.so +0 -0
- mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/aicpu_kernel/impl/libcust_aicpu_kernels.so +0 -0
- mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/aicpu_kernel/impl/libcust_cpu_kernels.so +0 -0
- mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/config/cust_aicpu_kernel.json +9000 -0
- mindspore/lib/plugin/ascend/custom_aicpu_ops/op_proto/libcust_op_proto.so +0 -0
- mindspore/lib/plugin/ascend/libakg.so +0 -0
- mindspore/lib/plugin/ascend/libascend_collective.so +0 -0
- mindspore/lib/plugin/ascend/libdvpp_utils.so +0 -0
- mindspore/lib/plugin/ascend/libhccl_plugin.so +0 -0
- mindspore/lib/plugin/ascend/libmindspore_aicpu_kernels.so +0 -0
- mindspore/lib/plugin/ascend/libmindspore_cpu_kernels.so +0 -0
- mindspore/lib/plugin/cpu/libakg.so +0 -0
- mindspore/lib/plugin/libmindspore_ascend.so.1 +0 -0
- mindspore/lib/plugin/libmindspore_ascend.so.2 +0 -0
- mindspore/log.py +9 -6
- mindspore/mindrecord/filereader.py +33 -4
- mindspore/mindrecord/filewriter.py +70 -35
- mindspore/mindrecord/mindpage.py +40 -34
- mindspore/mindrecord/shardreader.py +1 -1
- mindspore/mindrecord/shardsegment.py +1 -1
- mindspore/mindrecord/tools/cifar100_to_mr.py +25 -18
- mindspore/mindrecord/tools/cifar10_to_mr.py +25 -18
- mindspore/mindrecord/tools/csv_to_mr.py +29 -13
- mindspore/mindrecord/tools/imagenet_to_mr.py +24 -10
- mindspore/mindrecord/tools/mnist_to_mr.py +24 -11
- mindspore/mindrecord/tools/tfrecord_to_mr.py +31 -26
- mindspore/nn/cell.py +463 -169
- mindspore/nn/dynamic_lr.py +47 -43
- mindspore/nn/layer/activation.py +225 -82
- mindspore/nn/layer/basic.py +121 -79
- mindspore/nn/layer/channel_shuffle.py +21 -21
- mindspore/nn/layer/combined.py +33 -26
- mindspore/nn/layer/container.py +277 -22
- mindspore/nn/layer/conv.py +441 -304
- mindspore/nn/layer/dense.py +19 -13
- mindspore/nn/layer/embedding.py +62 -49
- mindspore/nn/layer/flash_attention.py +264 -0
- mindspore/nn/layer/image.py +50 -39
- mindspore/nn/layer/math.py +62 -51
- mindspore/nn/layer/normalization.py +219 -167
- mindspore/nn/layer/padding.py +58 -70
- mindspore/nn/layer/pooling.py +334 -287
- mindspore/nn/layer/rnn_cells.py +53 -38
- mindspore/nn/layer/rnns.py +59 -56
- mindspore/nn/layer/thor_layer.py +52 -44
- mindspore/nn/layer/timedistributed.py +6 -4
- mindspore/nn/layer/transformer.py +284 -164
- mindspore/nn/learning_rate_schedule.py +34 -25
- mindspore/nn/loss/__init__.py +3 -2
- mindspore/nn/loss/loss.py +554 -311
- mindspore/nn/optim/ada_grad.py +12 -9
- mindspore/nn/optim/adadelta.py +14 -11
- mindspore/nn/optim/adafactor.py +19 -16
- mindspore/nn/optim/adam.py +62 -47
- mindspore/nn/optim/adamax.py +13 -10
- mindspore/nn/optim/adasum.py +12 -8
- mindspore/nn/optim/asgd.py +10 -9
- mindspore/nn/optim/ftrl.py +20 -17
- mindspore/nn/optim/lamb.py +16 -12
- mindspore/nn/optim/lars.py +8 -6
- mindspore/nn/optim/lazyadam.py +25 -20
- mindspore/nn/optim/momentum.py +10 -7
- mindspore/nn/optim/optimizer.py +61 -9
- mindspore/nn/optim/proximal_ada_grad.py +14 -13
- mindspore/nn/optim/rmsprop.py +17 -13
- mindspore/nn/optim/rprop.py +30 -17
- mindspore/nn/optim/sgd.py +40 -23
- mindspore/nn/optim/thor.py +24 -26
- mindspore/nn/probability/bijector/bijector.py +11 -11
- mindspore/nn/probability/bijector/exp.py +1 -1
- mindspore/nn/probability/bijector/gumbel_cdf.py +3 -3
- mindspore/nn/probability/bijector/invert.py +1 -1
- mindspore/nn/probability/bijector/power_transform.py +29 -29
- mindspore/nn/probability/bijector/scalar_affine.py +3 -3
- mindspore/nn/probability/bijector/softplus.py +5 -5
- mindspore/nn/probability/bnn_layers/bnn_cell_wrapper.py +4 -2
- mindspore/nn/probability/bnn_layers/conv_variational.py +13 -13
- mindspore/nn/probability/bnn_layers/dense_variational.py +12 -12
- mindspore/nn/probability/bnn_layers/layer_distribution.py +9 -8
- mindspore/nn/probability/distribution/_utils/custom_ops.py +19 -3
- mindspore/nn/probability/distribution/_utils/utils.py +1 -1
- mindspore/nn/probability/distribution/bernoulli.py +9 -9
- mindspore/nn/probability/distribution/beta.py +8 -8
- mindspore/nn/probability/distribution/categorical.py +23 -15
- mindspore/nn/probability/distribution/cauchy.py +5 -6
- mindspore/nn/probability/distribution/distribution.py +3 -3
- mindspore/nn/probability/distribution/exponential.py +4 -4
- mindspore/nn/probability/distribution/gamma.py +10 -10
- mindspore/nn/probability/distribution/geometric.py +8 -8
- mindspore/nn/probability/distribution/gumbel.py +8 -9
- mindspore/nn/probability/distribution/half_normal.py +5 -5
- mindspore/nn/probability/distribution/laplace.py +5 -5
- mindspore/nn/probability/distribution/log_normal.py +12 -11
- mindspore/nn/probability/distribution/logistic.py +8 -8
- mindspore/nn/probability/distribution/normal.py +6 -5
- mindspore/nn/probability/distribution/poisson.py +10 -11
- mindspore/nn/probability/distribution/student_t.py +8 -9
- mindspore/nn/probability/distribution/transformed_distribution.py +5 -5
- mindspore/nn/probability/distribution/uniform.py +11 -11
- mindspore/nn/reinforcement/tensor_array.py +2 -2
- mindspore/nn/sparse/sparse.py +9 -9
- mindspore/nn/wrap/cell_wrapper.py +188 -63
- mindspore/nn/wrap/grad_reducer.py +21 -12
- mindspore/nn/wrap/loss_scale.py +136 -49
- mindspore/numpy/__init__.py +4 -4
- mindspore/numpy/array_creations.py +55 -56
- mindspore/numpy/array_ops.py +134 -35
- mindspore/numpy/logic_ops.py +66 -20
- mindspore/numpy/math_ops.py +142 -139
- mindspore/numpy/utils_const.py +2 -2
- mindspore/offline_debug/convert_async.py +2 -2
- mindspore/ops/_grad_experimental/__init__.py +7 -5
- mindspore/ops/_grad_experimental/grad_array_ops.py +231 -348
- mindspore/ops/{_grad → _grad_experimental}/grad_base.py +1 -33
- mindspore/ops/{_grad → _grad_experimental}/grad_comm_ops.py +25 -13
- mindspore/ops/{_grad/__init__.py → _grad_experimental/grad_debug_ops.py} +15 -7
- mindspore/ops/{_grad → _grad_experimental}/grad_implementations.py +17 -11
- mindspore/ops/_grad_experimental/grad_inner_ops.py +33 -52
- mindspore/ops/_grad_experimental/grad_math_ops.py +151 -1224
- mindspore/ops/_grad_experimental/grad_nn_ops.py +141 -414
- mindspore/ops/{_grad → _grad_experimental}/grad_quant_ops.py +10 -6
- mindspore/ops/_grad_experimental/grad_sparse.py +317 -2
- mindspore/ops/_grad_experimental/grad_sparse_ops.py +3 -13
- mindspore/ops/{_grad → _grad_experimental}/taylor_rule.py +1 -1
- mindspore/ops/_op_impl/_custom_op/dsd_back_impl.py +1 -1
- mindspore/ops/_op_impl/_custom_op/flash_attention/__init__.py +0 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/attention.py +406 -0
- mindspore/{_extends/graph_kernel/expanders/complex/__init__.py → ops/_op_impl/_custom_op/flash_attention/constants.py} +27 -8
- mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_bwd.py +467 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_fwd.py +563 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_impl.py +193 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/tik_ops_utils.py +435 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/__init__.py +0 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/sparse_tiling.py +45 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/strategy.py +67 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/wukong_tiling.py +62 -0
- mindspore/ops/_op_impl/_custom_op/matmul_cube_dense_left_impl.py +2 -2
- mindspore/ops/_op_impl/aicpu/__init__.py +41 -1
- mindspore/ops/_op_impl/aicpu/adaptive_max_pool_2d.py +37 -0
- mindspore/ops/_op_impl/aicpu/bias_add_grad.py +0 -1
- mindspore/ops/_op_impl/aicpu/cast.py +52 -0
- mindspore/ops/_op_impl/aicpu/coalesce.py +2 -0
- mindspore/ops/_op_impl/aicpu/col2im.py +3 -1
- mindspore/ops/_op_impl/aicpu/count_nonzero.py +43 -0
- mindspore/ops/_op_impl/aicpu/dropout_genmask.py +6 -0
- mindspore/ops/_op_impl/aicpu/eps.py +32 -0
- mindspore/ops/_op_impl/aicpu/eye.py +4 -4
- mindspore/ops/_op_impl/aicpu/fft_with_size.py +6 -0
- mindspore/ops/_op_impl/aicpu/fill_diagonal.py +5 -0
- mindspore/ops/_op_impl/aicpu/gamma.py +2 -2
- mindspore/ops/_op_impl/aicpu/im2col.py +3 -5
- mindspore/ops/_op_impl/aicpu/lgamma.py +1 -0
- mindspore/ops/_op_impl/aicpu/log_uniform_candidate_sampler.py +6 -3
- mindspore/ops/_op_impl/aicpu/lu.py +39 -0
- mindspore/ops/_op_impl/aicpu/lu_unpack_grad.py +0 -1
- mindspore/ops/_op_impl/aicpu/masked_scatter.py +1 -0
- mindspore/ops/_op_impl/aicpu/masked_select_grad.py +3 -0
- mindspore/ops/_op_impl/aicpu/matrix_band_part.py +59 -0
- mindspore/ops/_op_impl/aicpu/matrix_power.py +6 -1
- mindspore/ops/_op_impl/aicpu/median.py +1 -0
- mindspore/ops/_op_impl/aicpu/multinomial.py +9 -9
- mindspore/ops/_op_impl/aicpu/not_equal.py +0 -5
- mindspore/ops/_op_impl/aicpu/pad_v3.py +3 -1
- mindspore/ops/_op_impl/aicpu/pad_v3_grad.py +2 -0
- mindspore/ops/_op_impl/aicpu/parameterized_truncated_normal.py +15 -7
- mindspore/ops/_op_impl/aicpu/random_categorical.py +39 -19
- mindspore/ops/_op_impl/aicpu/random_choice_with_mask.py +5 -2
- mindspore/ops/_op_impl/aicpu/random_poisson.py +103 -52
- mindspore/ops/_op_impl/aicpu/random_shuffle.py +17 -15
- mindspore/ops/_op_impl/aicpu/resize_bilinear_grad.py +0 -1
- mindspore/ops/_op_impl/aicpu/resize_nearest_neighbor_v2.py +0 -6
- mindspore/ops/_op_impl/aicpu/resize_nearest_neighbor_v2_grad.py +0 -7
- mindspore/ops/_op_impl/aicpu/scatter_nd.py +2 -0
- mindspore/ops/_op_impl/aicpu/sequence_concat.py +40 -0
- mindspore/ops/_op_impl/aicpu/sequence_stack.py +40 -0
- mindspore/ops/_op_impl/aicpu/{sparseaddmm.py → sparse_addmm.py} +2 -2
- mindspore/ops/_op_impl/aicpu/{sparsesparsemaximum.py → sparse_sparse_maximum.py} +4 -4
- mindspore/ops/_op_impl/aicpu/standard_laplace.py +5 -4
- mindspore/ops/_op_impl/aicpu/standard_normal.py +5 -4
- mindspore/ops/_op_impl/aicpu/truncated_normal.py +9 -7
- mindspore/ops/_op_impl/aicpu/uniform.py +5 -3
- mindspore/ops/_op_impl/aicpu/uniform_candidate_sampler.py +8 -4
- mindspore/ops/_op_impl/aicpu/uniform_int.py +5 -5
- mindspore/ops/_op_impl/aicpu/uniform_real.py +4 -4
- mindspore/ops/_op_impl/aicpu/upsample_nearest_3d.py +14 -6
- mindspore/ops/_op_impl/aicpu/upsample_nearest_3d_grad.py +22 -8
- mindspore/ops/_op_impl/aicpu/upsample_trilinear_3d.py +11 -6
- mindspore/ops/_op_impl/aicpu/upsample_trilinear_3d_grad.py +21 -10
- mindspore/ops/_op_impl/tbe/__init__.py +6 -4
- mindspore/ops/_op_impl/tbe/atomic_addr_clean.py +1 -1
- mindspore/ops/_op_impl/tbe/avg_pool.py +2 -2
- mindspore/ops/_op_impl/tbe/avg_pool_3d.py +3 -3
- mindspore/ops/_op_impl/tbe/avg_pool_3d_grad.py +4 -4
- mindspore/ops/_op_impl/tbe/avg_pool_ds.py +2 -2
- mindspore/ops/_op_impl/tbe/avg_pool_grad.py +3 -3
- mindspore/ops/_op_impl/tbe/avg_pool_grad_vm.py +3 -3
- mindspore/ops/_op_impl/tbe/batch_to_space.py +1 -1
- mindspore/ops/_op_impl/tbe/batch_to_space_nd.py +2 -2
- mindspore/ops/_op_impl/tbe/bn_infer.py +2 -2
- mindspore/ops/_op_impl/tbe/bn_infer_ds.py +3 -2
- mindspore/ops/_op_impl/tbe/broadcast_to.py +1 -1
- mindspore/ops/_op_impl/tbe/depthwise_conv2d.py +3 -3
- mindspore/ops/_op_impl/tbe/expand_dims.py +1 -1
- mindspore/ops/_op_impl/tbe/gather_v2.py +56 -0
- mindspore/ops/_op_impl/tbe/im2col.py +4 -4
- mindspore/ops/_op_impl/tbe/inplace_index_add.py +7 -3
- mindspore/ops/_op_impl/tbe/mem_set.py +38 -0
- mindspore/ops/_op_impl/tbe/scatter_nd_add.py +3 -0
- mindspore/ops/_op_impl/tbe/scatter_nd_d.py +1 -1
- mindspore/ops/_op_impl/tbe/space_to_batch.py +1 -1
- mindspore/ops/_op_impl/tbe/space_to_batch_nd.py +2 -2
- mindspore/ops/_op_impl/tbe/trans_data_ds.py +2 -0
- mindspore/ops/_primitive_cache.py +1 -1
- mindspore/ops/_tracefunc.py +241 -0
- mindspore/ops/_utils/utils.py +10 -2
- mindspore/ops/_vmap/vmap_array_ops.py +5 -3
- mindspore/ops/_vmap/vmap_base.py +5 -4
- mindspore/ops/_vmap/vmap_convolution_ops.py +1 -1
- mindspore/ops/_vmap/vmap_grad_math_ops.py +6 -4
- mindspore/ops/_vmap/vmap_grad_nn_ops.py +11 -6
- mindspore/ops/_vmap/vmap_math_ops.py +5 -2
- mindspore/ops/_vmap/vmap_nn_ops.py +135 -11
- mindspore/ops/arg_dtype_cast.py +54 -0
- mindspore/ops/composite/__init__.py +7 -5
- mindspore/ops/composite/base.py +78 -34
- mindspore/ops/composite/math_ops.py +5 -695
- mindspore/ops/composite/multitype_ops/_compile_utils.py +403 -97
- mindspore/ops/composite/multitype_ops/_constexpr_utils.py +28 -22
- mindspore/ops/composite/multitype_ops/add_impl.py +69 -7
- mindspore/ops/composite/multitype_ops/bitwise_and_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/bitwise_or_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/bitwise_xor_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/div_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/floordiv_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/getitem_impl.py +48 -10
- mindspore/ops/composite/multitype_ops/greater_equal_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/greater_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/left_shift_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/less_equal_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/less_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/logic_not_impl.py +2 -2
- mindspore/ops/composite/multitype_ops/mod_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/mul_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/negative_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/not_in_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/ones_like_impl.py +6 -0
- mindspore/ops/composite/multitype_ops/pow_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/right_shift_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/setitem_impl.py +10 -7
- mindspore/ops/composite/multitype_ops/sub_impl.py +1 -0
- mindspore/ops/composite/multitype_ops/uadd_impl.py +2 -0
- mindspore/ops/composite/multitype_ops/zeros_like_impl.py +9 -0
- mindspore/ops/deprecated.py +304 -0
- mindspore/ops/function/__init__.py +41 -4
- mindspore/ops/function/array_func.py +1108 -467
- mindspore/ops/function/clip_func.py +94 -27
- mindspore/ops/function/debug_func.py +3 -1
- mindspore/ops/function/grad/grad_func.py +82 -73
- mindspore/ops/function/image_func.py +28 -12
- mindspore/ops/function/linalg_func.py +135 -39
- mindspore/ops/function/math_func.py +3779 -894
- mindspore/ops/function/nn_func.py +1584 -657
- mindspore/ops/function/parameter_func.py +13 -3
- mindspore/ops/function/random_func.py +247 -153
- mindspore/ops/function/sparse_func.py +14 -11
- mindspore/ops/function/sparse_unary_func.py +173 -47
- mindspore/ops/function/spectral_func.py +8 -4
- mindspore/ops/function/vmap_func.py +8 -7
- mindspore/ops/functional.py +47 -16
- mindspore/ops/op_info_register.py +346 -86
- mindspore/ops/operations/__init__.py +38 -22
- mindspore/ops/operations/_grad_ops.py +145 -149
- mindspore/ops/operations/_inner_ops.py +298 -56
- mindspore/ops/operations/_ms_kernel.py +3 -3
- mindspore/ops/operations/_quant_ops.py +24 -28
- mindspore/ops/operations/_rl_inner_ops.py +9 -7
- mindspore/ops/operations/_scalar_ops.py +115 -0
- mindspore/ops/operations/_sequence_ops.py +148 -10
- mindspore/ops/operations/_tensor_array.py +1 -1
- mindspore/ops/operations/_thor_ops.py +2 -2
- mindspore/ops/operations/array_ops.py +1239 -561
- mindspore/ops/operations/comm_ops.py +166 -90
- mindspore/ops/operations/control_ops.py +3 -3
- mindspore/ops/operations/custom_ops.py +124 -102
- mindspore/ops/operations/debug_ops.py +24 -11
- mindspore/ops/operations/image_ops.py +86 -71
- mindspore/ops/operations/inner_ops.py +18 -13
- mindspore/ops/operations/linalg_ops.py +30 -11
- mindspore/ops/operations/math_ops.py +1730 -435
- mindspore/ops/operations/nn_ops.py +1953 -943
- mindspore/ops/operations/other_ops.py +65 -43
- mindspore/ops/operations/random_ops.py +258 -98
- mindspore/ops/operations/rl_ops.py +4 -36
- mindspore/ops/operations/sparse_ops.py +38 -33
- mindspore/ops/operations/spectral_ops.py +8 -4
- mindspore/ops/primitive.py +66 -44
- mindspore/ops/signature.py +5 -5
- mindspore/parallel/_auto_parallel_context.py +80 -19
- mindspore/parallel/_cost_model_context.py +42 -0
- mindspore/parallel/_offload_context.py +162 -72
- mindspore/parallel/_parallel_serialization.py +2 -2
- mindspore/parallel/_ps_context.py +16 -4
- mindspore/parallel/_recovery_context.py +2 -1
- mindspore/parallel/_tensor.py +15 -13
- mindspore/parallel/_transformer/layers.py +8 -6
- mindspore/parallel/_transformer/loss.py +1 -0
- mindspore/parallel/_transformer/moe.py +7 -7
- mindspore/parallel/_transformer/op_parallel_config.py +12 -1
- mindspore/parallel/_transformer/transformer.py +34 -14
- mindspore/parallel/_utils.py +36 -14
- mindspore/parallel/algo_parameter_config.py +114 -20
- mindspore/parallel/checkpoint_transform.py +16 -18
- mindspore/parallel/shard.py +16 -13
- mindspore/profiler/__init__.py +1 -1
- mindspore/profiler/common/struct_type.py +3 -3
- mindspore/profiler/common/util.py +3 -2
- mindspore/profiler/envprofiling.py +11 -4
- mindspore/profiler/parser/aicpu_data_parser.py +5 -3
- mindspore/profiler/parser/ascend_flops_generator.py +94 -0
- mindspore/profiler/parser/ascend_fpbp_generator.py +76 -0
- mindspore/profiler/parser/ascend_hccl_generator.py +288 -0
- mindspore/profiler/parser/ascend_msprof_exporter.py +213 -0
- mindspore/profiler/parser/ascend_msprof_generator.py +199 -0
- mindspore/profiler/parser/ascend_op_generator.py +276 -0
- mindspore/profiler/parser/ascend_steptrace_generator.py +94 -0
- mindspore/profiler/parser/ascend_timeline_generator.py +110 -54
- mindspore/profiler/parser/base_timeline_generator.py +11 -7
- mindspore/profiler/parser/cpu_gpu_timeline_generator.py +45 -46
- mindspore/profiler/parser/flops_parser.py +15 -11
- mindspore/profiler/parser/framework_parser.py +92 -73
- mindspore/profiler/parser/hccl_parser.py +16 -12
- mindspore/profiler/parser/integrator.py +22 -11
- mindspore/profiler/parser/memory_usage_parser.py +36 -11
- mindspore/profiler/parser/minddata_analyzer.py +12 -14
- mindspore/profiler/parser/minddata_pipeline_parser.py +1 -1
- mindspore/profiler/parser/msadvisor_parser.py +8 -4
- mindspore/profiler/parser/op_intermediate_parser.py +5 -2
- mindspore/profiler/parser/optime_parser.py +1 -1
- mindspore/profiler/parser/profiler_info.py +4 -5
- mindspore/profiler/parser/step_trace_parser.py +11 -14
- mindspore/profiler/profiling.py +678 -377
- mindspore/rewrite/api/node.py +211 -54
- mindspore/rewrite/api/node_type.py +5 -0
- mindspore/rewrite/api/pattern_engine.py +22 -23
- mindspore/rewrite/api/scoped_value.py +20 -17
- mindspore/rewrite/api/symbol_tree.py +252 -106
- mindspore/rewrite/api/tree_node_helper.py +3 -0
- mindspore/rewrite/ast_helpers/__init__.py +2 -1
- mindspore/rewrite/ast_helpers/ast_finder.py +129 -0
- mindspore/rewrite/ast_helpers/ast_modifier.py +116 -104
- mindspore/rewrite/ast_transformers/flatten_recursive_stmt.py +97 -46
- mindspore/rewrite/common/rewrite_elog.py +5 -1
- mindspore/rewrite/namer.py +51 -51
- mindspore/rewrite/namespace.py +14 -5
- mindspore/{ops/bprop_mindir → rewrite/node}/__init__.py +9 -4
- mindspore/rewrite/node/call_function.py +79 -0
- mindspore/rewrite/node/cell_container.py +135 -0
- mindspore/rewrite/node/control_flow.py +88 -0
- mindspore/rewrite/{node.py → node/node.py} +313 -247
- mindspore/rewrite/node/node_manager.py +254 -0
- mindspore/rewrite/node/node_topological_manager.py +243 -0
- mindspore/rewrite/parsers/arguments_parser.py +22 -21
- mindspore/rewrite/parsers/assign_parser.py +225 -239
- mindspore/rewrite/parsers/attribute_parser.py +9 -7
- mindspore/rewrite/parsers/class_def_parser.py +179 -218
- mindspore/rewrite/parsers/constant_parser.py +9 -6
- mindspore/rewrite/parsers/container_parser.py +9 -7
- mindspore/rewrite/parsers/for_parser.py +36 -15
- mindspore/rewrite/parsers/function_def_parser.py +23 -20
- mindspore/rewrite/parsers/if_parser.py +28 -24
- mindspore/rewrite/parsers/module_parser.py +202 -25
- mindspore/rewrite/{parser.py → parsers/parser.py} +4 -2
- mindspore/rewrite/{parser_register.py → parsers/parser_register.py} +1 -1
- mindspore/rewrite/parsers/return_parser.py +6 -6
- mindspore/rewrite/sparsify/sparse_transformer.py +12 -3
- mindspore/rewrite/sparsify/sparsify.py +4 -1
- mindspore/rewrite/sparsify/utils.py +11 -5
- mindspore/rewrite/symbol_tree.py +577 -732
- mindspore/rewrite/symbol_tree_builder.py +9 -175
- mindspore/rewrite/symbol_tree_dumper.py +2 -2
- mindspore/run_check/_check_version.py +46 -39
- mindspore/run_check/run_check.py +3 -2
- mindspore/{scipy/sparse → safeguard}/__init__.py +4 -5
- mindspore/safeguard/rewrite_obfuscation.py +517 -0
- mindspore/scipy/__init__.py +1 -1
- mindspore/scipy/linalg.py +67 -61
- mindspore/scipy/ops.py +5 -41
- mindspore/scipy/ops_grad.py +3 -2
- mindspore/scipy/ops_wrapper.py +5 -5
- mindspore/scipy/optimize/line_search.py +8 -8
- mindspore/scipy/optimize/linear_sum_assignment.py +4 -4
- mindspore/scipy/optimize/minimize.py +16 -12
- mindspore/scipy/utils.py +1 -52
- mindspore/scipy/utils_const.py +4 -4
- mindspore/train/__init__.py +4 -4
- mindspore/train/_utils.py +13 -5
- mindspore/train/amp.py +410 -148
- mindspore/train/anf_ir_pb2.py +16 -4
- mindspore/train/callback/_backup_and_restore.py +8 -11
- mindspore/train/callback/_callback.py +80 -3
- mindspore/train/callback/_checkpoint.py +82 -51
- mindspore/train/callback/_early_stop.py +12 -15
- mindspore/train/callback/_history.py +1 -1
- mindspore/train/callback/_lambda_callback.py +13 -13
- mindspore/train/callback/_landscape.py +21 -17
- mindspore/train/callback/_loss_monitor.py +9 -10
- mindspore/train/callback/_on_request_exit.py +16 -33
- mindspore/train/callback/_reduce_lr_on_plateau.py +21 -24
- mindspore/train/callback/_summary_collector.py +44 -30
- mindspore/train/callback/_time_monitor.py +62 -12
- mindspore/train/data_sink.py +10 -16
- mindspore/train/dataset_helper.py +154 -86
- mindspore/train/loss_scale_manager.py +14 -9
- mindspore/train/metrics/__init__.py +10 -2
- mindspore/train/metrics/accuracy.py +1 -1
- mindspore/train/metrics/auc.py +1 -1
- mindspore/train/metrics/bleu_score.py +2 -2
- mindspore/train/metrics/confusion_matrix.py +14 -14
- mindspore/train/metrics/cosine_similarity.py +3 -3
- mindspore/train/metrics/dice.py +1 -1
- mindspore/train/metrics/fbeta.py +1 -1
- mindspore/train/metrics/hausdorff_distance.py +8 -6
- mindspore/train/metrics/mean_surface_distance.py +5 -4
- mindspore/train/metrics/metric.py +49 -17
- mindspore/train/metrics/occlusion_sensitivity.py +4 -4
- mindspore/train/metrics/perplexity.py +1 -1
- mindspore/train/metrics/precision.py +2 -2
- mindspore/train/metrics/recall.py +2 -3
- mindspore/train/metrics/roc.py +7 -7
- mindspore/train/metrics/root_mean_square_surface_distance.py +5 -4
- mindspore/train/metrics/topk.py +7 -4
- mindspore/train/mind_ir_pb2.py +193 -48
- mindspore/train/model.py +377 -133
- mindspore/train/serialization.py +697 -245
- mindspore/train/summary/_summary_adapter.py +5 -2
- mindspore/train/summary/_writer_pool.py +4 -3
- mindspore/train/summary/summary_record.py +25 -23
- mindspore/train/train_thor/convert_utils.py +39 -23
- mindspore/train/train_thor/dataset_helper.py +4 -3
- mindspore/train/train_thor/model_thor.py +8 -8
- mindspore/version.py +1 -1
- {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/METADATA +7 -8
- {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/RECORD +633 -804
- {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/entry_points.txt +0 -1
- mindspore/_akg/akg/tvm/contrib/debugger/__init__.py +0 -16
- mindspore/_akg/akg/tvm/contrib/debugger/debug_result.py +0 -274
- mindspore/_akg/akg/tvm/contrib/debugger/debug_runtime.py +0 -259
- mindspore/_akg/akg/tvm/contrib/peak.py +0 -341
- mindspore/_akg/akg/tvm/contrib/rpc.py +0 -25
- mindspore/_akg/akg/tvm/contrib/xcode.py +0 -257
- mindspore/_akg/akg/tvm/exec/__init__.py +0 -17
- mindspore/_akg/akg/tvm/exec/autotvm_log_editor.py +0 -60
- mindspore/_akg/akg/tvm/exec/measure_peak.py +0 -48
- mindspore/_akg/akg/tvm/exec/query_rpc_tracker.py +0 -48
- mindspore/_akg/akg/tvm/exec/rpc_proxy.py +0 -98
- mindspore/_akg/akg/tvm/exec/rpc_server.py +0 -88
- mindspore/_akg/akg/tvm/exec/rpc_tracker.py +0 -62
- mindspore/_akg/akg/tvm/rpc/__init__.py +0 -29
- mindspore/_akg/akg/tvm/rpc/base.py +0 -182
- mindspore/_akg/akg/tvm/rpc/client.py +0 -436
- mindspore/_akg/akg/tvm/rpc/proxy.py +0 -595
- mindspore/_akg/akg/tvm/rpc/server.py +0 -413
- mindspore/_akg/akg/tvm/rpc/tornado_util.py +0 -121
- mindspore/_akg/akg/tvm/rpc/tracker.py +0 -431
- mindspore/_extends/graph_kernel/expander.py +0 -80
- mindspore/_extends/graph_kernel/expanders/__init__.py +0 -57
- mindspore/_extends/graph_kernel/expanders/_utils.py +0 -269
- mindspore/_extends/graph_kernel/expanders/addn.py +0 -33
- mindspore/_extends/graph_kernel/expanders/batchnorm.py +0 -152
- mindspore/_extends/graph_kernel/expanders/batchnorm_grad.py +0 -105
- mindspore/_extends/graph_kernel/expanders/bias_add_grad.py +0 -49
- mindspore/_extends/graph_kernel/expanders/clip_by_norm_no_div_sum.py +0 -33
- mindspore/_extends/graph_kernel/expanders/complex/abs.py +0 -30
- mindspore/_extends/graph_kernel/expanders/complex/add.py +0 -44
- mindspore/_extends/graph_kernel/expanders/complex/div.py +0 -62
- mindspore/_extends/graph_kernel/expanders/complex/mul.py +0 -52
- mindspore/_extends/graph_kernel/expanders/complex/real_div.py +0 -62
- mindspore/_extends/graph_kernel/expanders/complex/sub.py +0 -45
- mindspore/_extends/graph_kernel/expanders/conv2d.py +0 -200
- mindspore/_extends/graph_kernel/expanders/dropout_grad.py +0 -30
- mindspore/_extends/graph_kernel/expanders/equal_count.py +0 -50
- mindspore/_extends/graph_kernel/expanders/erfc.py +0 -35
- mindspore/_extends/graph_kernel/expanders/expand_dims.py +0 -50
- mindspore/_extends/graph_kernel/expanders/fused_adam.py +0 -44
- mindspore/_extends/graph_kernel/expanders/fused_adam_weight_decay.py +0 -47
- mindspore/_extends/graph_kernel/expanders/fused_mul_add.py +0 -28
- mindspore/_extends/graph_kernel/expanders/gather.py +0 -43
- mindspore/_extends/graph_kernel/expanders/gelu_grad.py +0 -70
- mindspore/_extends/graph_kernel/expanders/gkdropout.py +0 -40
- mindspore/_extends/graph_kernel/expanders/identity.py +0 -25
- mindspore/_extends/graph_kernel/expanders/layernorm.py +0 -93
- mindspore/_extends/graph_kernel/expanders/layernorm_grad.py +0 -113
- mindspore/_extends/graph_kernel/expanders/logsoftmax.py +0 -46
- mindspore/_extends/graph_kernel/expanders/logsoftmax_grad.py +0 -36
- mindspore/_extends/graph_kernel/expanders/matmul.py +0 -80
- mindspore/_extends/graph_kernel/expanders/maximum_grad.py +0 -59
- mindspore/_extends/graph_kernel/expanders/minimum_grad.py +0 -80
- mindspore/_extends/graph_kernel/expanders/oneslike.py +0 -26
- mindspore/_extends/graph_kernel/expanders/reduce_mean.py +0 -43
- mindspore/_extends/graph_kernel/expanders/relu_grad.py +0 -32
- mindspore/_extends/graph_kernel/expanders/sigmoid_cross_entropy_with_logits.py +0 -41
- mindspore/_extends/graph_kernel/expanders/sigmoid_cross_entropy_with_logits_grad.py +0 -35
- mindspore/_extends/graph_kernel/expanders/sigmoid_grad.py +0 -31
- mindspore/_extends/graph_kernel/expanders/slice.py +0 -35
- mindspore/_extends/graph_kernel/expanders/softmax_cross_entropy_with_logits.py +0 -42
- mindspore/_extends/graph_kernel/expanders/softmax_grad_ext.py +0 -41
- mindspore/_extends/graph_kernel/expanders/softsign.py +0 -28
- mindspore/_extends/graph_kernel/expanders/sqrt_grad.py +0 -29
- mindspore/_extends/graph_kernel/expanders/square_sum_all.py +0 -44
- mindspore/_extends/graph_kernel/expanders/square_sum_v1.py +0 -37
- mindspore/_extends/graph_kernel/expanders/squared_difference.py +0 -43
- mindspore/_extends/graph_kernel/expanders/tanh_grad.py +0 -31
- mindspore/_extends/graph_kernel/expanders/tile.py +0 -54
- mindspore/_extends/graph_kernel/model/op_infer.py +0 -506
- mindspore/_extends/parse/jit_fallback_modules.py +0 -51
- mindspore/dataset/datapreprocess/preprocess_imagenet_validate_dataset.py +0 -54
- mindspore/dataset/engine/graphdata.py +0 -1586
- mindspore/include/api/net.h +0 -142
- mindspore/ops/_grad/grad_array_ops.py +0 -1347
- mindspore/ops/_grad/grad_clip_ops.py +0 -84
- mindspore/ops/_grad/grad_debug_ops.py +0 -68
- mindspore/ops/_grad/grad_inner_ops.py +0 -235
- mindspore/ops/_grad/grad_math_ops.py +0 -1684
- mindspore/ops/_grad/grad_nn_ops.py +0 -1529
- mindspore/ops/_grad/grad_other_ops.py +0 -89
- mindspore/ops/_grad/grad_sequence_ops.py +0 -296
- mindspore/ops/_grad/grad_sparse.py +0 -323
- mindspore/ops/_grad_experimental/grad_image_ops.py +0 -249
- mindspore/ops/_grad_experimental/grad_linalg_ops.py +0 -195
- mindspore/ops/_grad_experimental/grad_scalar_ops.py +0 -112
- mindspore/ops/bprop_mindir/AdaptiveAvgPool2D_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/AdaptiveMaxPool2D_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/ApproximateEqual_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/Argmax_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/Argmin_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/AssignSub_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/Assign_bprop.mindir +0 -17
- mindspore/ops/bprop_mindir/AvgPool3D_bprop.mindir +0 -150
- mindspore/ops/bprop_mindir/AvgPool_bprop.mindir +0 -66
- mindspore/ops/bprop_mindir/BCEWithLogitsLoss_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/BNTrainingReduce_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/BatchNormGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/BatchToSpaceND_bprop.mindir +0 -28
- mindspore/ops/bprop_mindir/BiasAddGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/BinaryCrossEntropy_bprop.mindir +0 -33
- mindspore/ops/bprop_mindir/BroadcastTo_bprop.mindir +0 -306
- mindspore/ops/bprop_mindir/Broadcast_bprop.mindir +0 -13
- mindspore/ops/bprop_mindir/CTCLoss_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Concat_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Conv2DBackpropFilter_bprop.mindir +0 -240
- mindspore/ops/bprop_mindir/Conv2DBackpropInput_bprop.mindir +0 -247
- mindspore/ops/bprop_mindir/Conv2DTranspose_bprop.mindir +0 -247
- mindspore/ops/bprop_mindir/Conv3DTranspose_bprop.mindir +0 -315
- mindspore/ops/bprop_mindir/Conv3D_bprop.mindir +0 -278
- mindspore/ops/bprop_mindir/DType_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/DeformableOffsets_bprop.mindir +0 -58
- mindspore/ops/bprop_mindir/Depend_bprop.mindir +0 -13
- mindspore/ops/bprop_mindir/DepthToSpace_bprop.mindir +0 -23
- mindspore/ops/bprop_mindir/DepthwiseConv2dNative_bprop.mindir +0 -138
- mindspore/ops/bprop_mindir/DiagPart_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/Dropout2D_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Dropout3D_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/DropoutDoMask_bprop.mindir +0 -25
- mindspore/ops/bprop_mindir/DropoutGenMask_bprop.mindir +0 -18
- mindspore/ops/bprop_mindir/DropoutGrad_bprop.mindir +0 -27
- mindspore/ops/bprop_mindir/Dropout_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/DynamicGRUV2_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/DynamicRNN_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/DynamicShape_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/Elu_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/EmbeddingLookup_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Equal_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/ExpandDims_bprop.mindir +0 -58
- mindspore/ops/bprop_mindir/FastGeLU_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/Flatten_bprop.mindir +0 -54
- mindspore/ops/bprop_mindir/FloorDiv_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/GatherD_bprop.mindir +0 -26
- mindspore/ops/bprop_mindir/GatherNd_bprop.mindir +0 -57
- mindspore/ops/bprop_mindir/Gather_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/GreaterEqual_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/Greater_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/HSigmoid_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/HSwish_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/IOU_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/InstanceNorm_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/IsFinite_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/IsInf_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/IsNan_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/KLDivLoss_bprop.mindir +0 -126
- mindspore/ops/bprop_mindir/L2Loss_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/L2Normalize_bprop.mindir +0 -30
- mindspore/ops/bprop_mindir/LRN_bprop.mindir +0 -43
- mindspore/ops/bprop_mindir/LayerNormGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/LessEqual_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/Less_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/LinSpace_bprop.mindir +0 -23
- mindspore/ops/bprop_mindir/Load_bprop.mindir +0 -13
- mindspore/ops/bprop_mindir/LogSoftmax_bprop.mindir +0 -23
- mindspore/ops/bprop_mindir/LogicalAnd_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/LogicalNot_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/MaskedSelect_bprop.mindir +0 -21
- mindspore/ops/bprop_mindir/MaxPool3DGradGrad_bprop.mindir +0 -74
- mindspore/ops/bprop_mindir/MaxPool3DGrad_bprop.mindir +0 -74
- mindspore/ops/bprop_mindir/MaxPool3D_bprop.mindir +0 -75
- mindspore/ops/bprop_mindir/MaxPoolGradGrad_bprop.mindir +0 -65
- mindspore/ops/bprop_mindir/MaxPoolWithArgmax_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Maximum_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Minimum_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/MirrorPad_bprop.mindir +0 -27
- mindspore/ops/bprop_mindir/Mish_bprop.mindir +0 -35
- mindspore/ops/bprop_mindir/MulNoNan_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/NLLLoss_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/NonZero_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/NotEqual_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/OneHot_bprop.mindir +0 -26
- mindspore/ops/bprop_mindir/OnesLike_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/PReLU_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Pad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Padding_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/RNNTLoss_bprop.mindir +0 -29
- mindspore/ops/bprop_mindir/ROIAlign_bprop.mindir +0 -82
- mindspore/ops/bprop_mindir/Range_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/Rank_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/ReLU6_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/ReLUV2_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/ReduceAll_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/ReduceAny_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/ReluGrad_bprop.mindir +0 -20
- mindspore/ops/bprop_mindir/Reshape_bprop.mindir +0 -60
- mindspore/ops/bprop_mindir/ResizeBilinear_bprop.mindir +0 -29
- mindspore/ops/bprop_mindir/ResizeNearestNeighbor_bprop.mindir +0 -89
- mindspore/ops/bprop_mindir/ReverseSequence_bprop.mindir +0 -52
- mindspore/ops/bprop_mindir/ReverseV2_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/Round_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/ScatterMax_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/ScatterMin_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/ScatterNdUpdate_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/ScatterNd_bprop.mindir +0 -24
- mindspore/ops/bprop_mindir/ScatterNonAliasingAdd_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/ScatterUpdate_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/SeLU_bprop.mindir +0 -21
- mindspore/ops/bprop_mindir/Select_bprop.mindir +0 -31
- mindspore/ops/bprop_mindir/Shape_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/SigmoidCrossEntropyWithLogits_bprop.mindir +0 -21
- mindspore/ops/bprop_mindir/SigmoidGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Sigmoid_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/Sign_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/Slice_bprop.mindir +0 -26
- mindspore/ops/bprop_mindir/SmoothL1Loss_bprop.mindir +0 -36
- mindspore/ops/bprop_mindir/SoftmaxCrossEntropyWithLogits_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Softplus_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/Softsign_bprop.mindir +0 -33
- mindspore/ops/bprop_mindir/Sort_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/SpaceToBatchND_bprop.mindir +0 -28
- mindspore/ops/bprop_mindir/SpaceToDepth_bprop.mindir +0 -23
- mindspore/ops/bprop_mindir/SparseGatherV2_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/SparseSoftmaxCrossEntropyWithLogits_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Split_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/Squeeze_bprop.mindir +0 -54
- mindspore/ops/bprop_mindir/StridedSliceGrad_bprop.mindir +0 -95
- mindspore/ops/bprop_mindir/StridedSlice_bprop.mindir +0 -98
- mindspore/ops/bprop_mindir/Switch_bprop.mindir +0 -29
- mindspore/ops/bprop_mindir/TanhGrad_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/Tanh_bprop.mindir +0 -66
- mindspore/ops/bprop_mindir/TensorScatterAdd_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/TensorScatterUpdate_bprop.mindir +0 -29
- mindspore/ops/bprop_mindir/TensorShape_bprop.mindir +0 -14
- mindspore/ops/bprop_mindir/Tile_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/TopK_bprop.mindir +0 -0
- mindspore/ops/bprop_mindir/TransShape_bprop.mindir +0 -23
- mindspore/ops/bprop_mindir/TruncateDiv_bprop.mindir +0 -19
- mindspore/ops/bprop_mindir/TupleGetItem_bprop.mindir +0 -20
- mindspore/ops/bprop_mindir/Unique_bprop.mindir +0 -16
- mindspore/ops/bprop_mindir/Unstack_bprop.mindir +0 -22
- mindspore/ops/bprop_mindir/UpsampleNearest3D_bprop.mindir +0 -32
- mindspore/ops/bprop_mindir/UpsampleTrilinear3D_bprop.mindir +0 -38
- mindspore/ops/bprop_mindir/ZerosLike_bprop.mindir +0 -15
- mindspore/ops/bprop_mindir/generate_mindir.py +0 -114
- mindspore/rewrite/node_visitor.py +0 -44
- mindspore/rewrite/topological_manager.py +0 -203
- mindspore/scipy/sparse/linalg.py +0 -192
- {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/WHEEL +0 -0
- {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/top_level.txt +0 -0
mindspore/dataset/core/config.py
CHANGED
|
@@ -104,23 +104,26 @@ def _init_device_info():
|
|
|
104
104
|
|
|
105
105
|
def set_seed(seed):
|
|
106
106
|
"""
|
|
107
|
-
Set the seed
|
|
107
|
+
Set the seed for the random number generator in data pipeline.
|
|
108
|
+
|
|
109
|
+
The seed can be set to control the initial state of the random generator
|
|
110
|
+
for the purpose of fixing the result of random number generation.
|
|
108
111
|
|
|
109
112
|
Note:
|
|
110
|
-
This
|
|
111
|
-
|
|
112
|
-
be called when iterator is created to reset the random seed.
|
|
113
|
+
This interface will set the random seed of the `random`, `numpy.random`
|
|
114
|
+
and `mindspore.dataset` modules to the specified value at the same time.
|
|
113
115
|
|
|
114
116
|
Args:
|
|
115
|
-
seed(int):
|
|
117
|
+
seed (int): The desired seed. Must be non-negative.
|
|
116
118
|
|
|
117
119
|
Raises:
|
|
118
|
-
TypeError: If `seed`
|
|
119
|
-
ValueError: If `seed`
|
|
120
|
+
TypeError: If `seed` is not of type int.
|
|
121
|
+
ValueError: If `seed` is a negative value.
|
|
120
122
|
|
|
121
123
|
Examples:
|
|
122
124
|
>>> # Set a new global configuration value for the seed value.
|
|
123
125
|
>>> # Operations with randomness will use the seed value to generate random values.
|
|
126
|
+
>>> import mindspore.dataset as ds
|
|
124
127
|
>>> ds.config.set_seed(1000)
|
|
125
128
|
"""
|
|
126
129
|
if not isinstance(seed, int) or isinstance(seed, bool):
|
|
@@ -146,6 +149,7 @@ def get_seed():
|
|
|
146
149
|
Examples:
|
|
147
150
|
>>> # Get the global configuration of seed.
|
|
148
151
|
>>> # If set_seed() is never called before, the default value(std::mt19937::default_seed) will be returned.
|
|
152
|
+
>>> import mindspore.dataset as ds
|
|
149
153
|
>>> seed = ds.config.get_seed()
|
|
150
154
|
"""
|
|
151
155
|
return _config.get_seed()
|
|
@@ -153,23 +157,32 @@ def get_seed():
|
|
|
153
157
|
|
|
154
158
|
def set_prefetch_size(size):
|
|
155
159
|
"""
|
|
156
|
-
Set the queue
|
|
160
|
+
Set the buffer queue size between dataset operations in the pipeline.
|
|
161
|
+
|
|
162
|
+
The presence of a buffer queue allows the current operation to start
|
|
163
|
+
processing subsequent data before the next operation fetches it, so the
|
|
164
|
+
operations can execute asynchronously and concurrently.
|
|
165
|
+
|
|
166
|
+
A larger buffer queue size reduces the overall processing latency when
|
|
167
|
+
neighboring operations have unbalanced throughput rates, but also consumes
|
|
168
|
+
more system memory.
|
|
157
169
|
|
|
158
170
|
Args:
|
|
159
|
-
size (int): The
|
|
160
|
-
the thread is invalid.
|
|
171
|
+
size (int): The size of the buffer queue, must be greater than 0.
|
|
161
172
|
|
|
162
173
|
Raises:
|
|
163
174
|
TypeError: If `size` is not of type int.
|
|
164
175
|
ValueError: If `size` is not a positive number.
|
|
165
176
|
|
|
166
177
|
Note:
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
178
|
+
The total memory consumed by the buffer queue is proportional to the number
|
|
179
|
+
of worker threads. To avoid overuse of memory, when the number of worker
|
|
180
|
+
threads is greater than 4, the actual buffer queue size used will be adjusted
|
|
181
|
+
to the greater of (`size` * 4 / number of worker threads) and 1.
|
|
170
182
|
|
|
171
183
|
Examples:
|
|
172
184
|
>>> # Set a new global configuration value for the prefetch size.
|
|
185
|
+
>>> import mindspore.dataset as ds
|
|
173
186
|
>>> ds.config.set_prefetch_size(1000)
|
|
174
187
|
"""
|
|
175
188
|
if not isinstance(size, int) or isinstance(size, bool):
|
|
@@ -191,6 +204,7 @@ def get_prefetch_size():
|
|
|
191
204
|
Examples:
|
|
192
205
|
>>> # Get the global configuration of prefetch size.
|
|
193
206
|
>>> # If set_prefetch_size() is never called before, the default value(16) will be returned.
|
|
207
|
+
>>> import mindspore.dataset as ds
|
|
194
208
|
>>> prefetch_size = ds.config.get_prefetch_size()
|
|
195
209
|
"""
|
|
196
210
|
return _config.get_op_connector_size()
|
|
@@ -211,6 +225,7 @@ def set_num_parallel_workers(num):
|
|
|
211
225
|
Examples:
|
|
212
226
|
>>> # Set a new global configuration value for the number of parallel workers.
|
|
213
227
|
>>> # Now parallel dataset operations will run with 8 workers.
|
|
228
|
+
>>> import mindspore.dataset as ds
|
|
214
229
|
>>> ds.config.set_num_parallel_workers(8)
|
|
215
230
|
"""
|
|
216
231
|
if not isinstance(num, int) or isinstance(num, bool):
|
|
@@ -225,6 +240,7 @@ def get_num_parallel_workers():
|
|
|
225
240
|
"""
|
|
226
241
|
Get the global configuration of number of parallel workers.
|
|
227
242
|
This is the DEFAULT num_parallel_workers value used for each operation.
|
|
243
|
+
If `set_num_parallel_workers` is never called before, the default value(8) will be returned.
|
|
228
244
|
|
|
229
245
|
Returns:
|
|
230
246
|
int, number of parallel workers to be used as a default for each operation.
|
|
@@ -232,6 +248,7 @@ def get_num_parallel_workers():
|
|
|
232
248
|
Examples:
|
|
233
249
|
>>> # Get the global configuration of parallel workers.
|
|
234
250
|
>>> # If set_num_parallel_workers() is never called before, the default value(8) will be returned.
|
|
251
|
+
>>> import mindspore.dataset as ds
|
|
235
252
|
>>> num_parallel_workers = ds.config.get_num_parallel_workers()
|
|
236
253
|
"""
|
|
237
254
|
return _config.get_num_parallel_workers()
|
|
@@ -239,7 +256,7 @@ def get_num_parallel_workers():
|
|
|
239
256
|
|
|
240
257
|
def set_numa_enable(numa_enable):
|
|
241
258
|
"""
|
|
242
|
-
Set the default state of numa enabled. If `numa_enable` is True
|
|
259
|
+
Set the default state of numa enabled. If `numa_enable` is ``True``, need to
|
|
243
260
|
ensure `numa library <http://rpmfind.net/linux/rpm2html/search.php?query=libnuma-devel>`_ is installed.
|
|
244
261
|
|
|
245
262
|
Args:
|
|
@@ -251,6 +268,7 @@ def set_numa_enable(numa_enable):
|
|
|
251
268
|
Examples:
|
|
252
269
|
>>> # Set a new global configuration value for the state of numa enabled.
|
|
253
270
|
>>> # Now parallel dataset operations will run with numa bind function
|
|
271
|
+
>>> import mindspore.dataset as ds
|
|
254
272
|
>>> ds.config.set_numa_enable(True)
|
|
255
273
|
"""
|
|
256
274
|
if not isinstance(numa_enable, bool):
|
|
@@ -261,13 +279,14 @@ def set_numa_enable(numa_enable):
|
|
|
261
279
|
def get_numa_enable():
|
|
262
280
|
"""
|
|
263
281
|
Get the state of numa to indicate enabled/disabled.
|
|
264
|
-
This is the
|
|
282
|
+
This is the numa config used for all processes, it is disabled by default.
|
|
265
283
|
|
|
266
284
|
Returns:
|
|
267
285
|
bool, the default state of numa enabled.
|
|
268
286
|
|
|
269
287
|
Examples:
|
|
270
288
|
>>> # Get the global configuration of numa.
|
|
289
|
+
>>> import mindspore.dataset as ds
|
|
271
290
|
>>> numa_state = ds.config.get_numa_enable()
|
|
272
291
|
"""
|
|
273
292
|
return _config.get_numa_enable()
|
|
@@ -286,6 +305,7 @@ def set_monitor_sampling_interval(interval):
|
|
|
286
305
|
|
|
287
306
|
Examples:
|
|
288
307
|
>>> # Set a new global configuration value for the monitor sampling interval.
|
|
308
|
+
>>> import mindspore.dataset as ds
|
|
289
309
|
>>> ds.config.set_monitor_sampling_interval(100)
|
|
290
310
|
"""
|
|
291
311
|
if not isinstance(interval, int) or isinstance(interval, bool):
|
|
@@ -307,6 +327,7 @@ def get_monitor_sampling_interval():
|
|
|
307
327
|
Examples:
|
|
308
328
|
>>> # Get the global configuration of monitor sampling interval.
|
|
309
329
|
>>> # If set_monitor_sampling_interval() is never called before, the default value(1000) will be returned.
|
|
330
|
+
>>> import mindspore.dataset as ds
|
|
310
331
|
>>> sampling_interval = ds.config.get_monitor_sampling_interval()
|
|
311
332
|
"""
|
|
312
333
|
return _config.get_monitor_sampling_interval()
|
|
@@ -318,7 +339,7 @@ def set_auto_num_workers(enable):
|
|
|
318
339
|
|
|
319
340
|
If turned on, the num_parallel_workers in each op will be adjusted automatically, possibly overwriting the
|
|
320
341
|
num_parallel_workers passed in by user or the default value (if user doesn't pass anything) set by
|
|
321
|
-
|
|
342
|
+
:func:`mindspore.dataset.config.set_num_parallel_workers`.
|
|
322
343
|
|
|
323
344
|
For now, this function is only optimized for YoloV3 dataset with per_batch_map (running map in batch).
|
|
324
345
|
This feature aims to provide a baseline for optimized num_workers assignment for each operation.
|
|
@@ -332,6 +353,7 @@ def set_auto_num_workers(enable):
|
|
|
332
353
|
|
|
333
354
|
Examples:
|
|
334
355
|
>>> # Enable auto_num_worker feature, this might override the num_parallel_workers passed in by user
|
|
356
|
+
>>> import mindspore.dataset as ds
|
|
335
357
|
>>> ds.config.set_auto_num_workers(True)
|
|
336
358
|
"""
|
|
337
359
|
if not isinstance(enable, bool):
|
|
@@ -367,13 +389,14 @@ def _set_auto_workers_config(option):
|
|
|
367
389
|
|
|
368
390
|
def get_auto_num_workers():
|
|
369
391
|
"""
|
|
370
|
-
Get the setting (turned on or off) automatic number of workers.
|
|
392
|
+
Get the setting (turned on or off) automatic number of workers, it is disabled by default.
|
|
371
393
|
|
|
372
394
|
Returns:
|
|
373
395
|
bool, whether auto number worker feature is turned on.
|
|
374
396
|
|
|
375
397
|
Examples:
|
|
376
398
|
>>> # Get the global configuration of auto number worker feature.
|
|
399
|
+
>>> import mindspore.dataset as ds
|
|
377
400
|
>>> flag = ds.config.get_auto_num_workers()
|
|
378
401
|
"""
|
|
379
402
|
return _config.get_auto_num_workers()
|
|
@@ -393,6 +416,7 @@ def set_callback_timeout(timeout):
|
|
|
393
416
|
|
|
394
417
|
Examples:
|
|
395
418
|
>>> # Set a new global configuration value for the timeout value.
|
|
419
|
+
>>> import mindspore.dataset as ds
|
|
396
420
|
>>> ds.config.set_callback_timeout(100)
|
|
397
421
|
"""
|
|
398
422
|
if not isinstance(timeout, int) or isinstance(timeout, bool):
|
|
@@ -404,7 +428,8 @@ def set_callback_timeout(timeout):
|
|
|
404
428
|
|
|
405
429
|
def get_callback_timeout():
|
|
406
430
|
"""
|
|
407
|
-
Get the default timeout for :class:`mindspore.dataset.WaitedDSCallback` .
|
|
431
|
+
Get the default timeout (in seconds) for :class:`mindspore.dataset.WaitedDSCallback` .
|
|
432
|
+
If `set_callback_timeout` is never called before, the default value(60) will be returned.
|
|
408
433
|
|
|
409
434
|
Returns:
|
|
410
435
|
int, Timeout (in seconds) to be used to end the wait in :class:`mindspore.dataset.WaitedDSCallback` in case of
|
|
@@ -413,6 +438,7 @@ def get_callback_timeout():
|
|
|
413
438
|
Examples:
|
|
414
439
|
>>> # Get the global configuration of callback timeout.
|
|
415
440
|
>>> # If set_callback_timeout() is never called before, the default value(60) will be returned.
|
|
441
|
+
>>> import mindspore.dataset as ds
|
|
416
442
|
>>> callback_timeout = ds.config.get_callback_timeout()
|
|
417
443
|
"""
|
|
418
444
|
return _config.get_callback_timeout()
|
|
@@ -447,6 +473,7 @@ def load(file):
|
|
|
447
473
|
>>> # "seed": 5489,
|
|
448
474
|
>>> # "monitorSamplingInterval": 30
|
|
449
475
|
>>> # }
|
|
476
|
+
>>> import mindspore.dataset as ds
|
|
450
477
|
>>> config_file = "/path/to/config/file"
|
|
451
478
|
>>> ds.config.load(config_file)
|
|
452
479
|
"""
|
|
@@ -455,43 +482,40 @@ def load(file):
|
|
|
455
482
|
|
|
456
483
|
def set_enable_autotune(enable, filepath_prefix=None):
|
|
457
484
|
"""
|
|
458
|
-
Set whether to enable AutoTune
|
|
485
|
+
Set whether to enable AutoTune for data pipeline parameters.
|
|
459
486
|
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
487
|
+
It can be used to automatically adjust the parameter configurations
|
|
488
|
+
of each operation in the data processing pipeline, such as parallelism
|
|
489
|
+
and buffer queue size, according to the load of the environment resources
|
|
490
|
+
during training, so as to improve the overall processing speed.
|
|
463
491
|
|
|
464
|
-
|
|
465
|
-
for subsequent reuse.
|
|
492
|
+
AutoTune is not enabled by default.
|
|
466
493
|
|
|
467
494
|
Args:
|
|
468
495
|
enable (bool): Whether to enable AutoTune.
|
|
469
|
-
filepath_prefix (str, optional): The
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
If the file already exists, it will be automatically overwritten. Default: None,
|
|
475
|
-
means not to save the configuration file, but the tuned result still can be checked through INFO log.
|
|
496
|
+
filepath_prefix (str, optional): The path where the optimized parameter configuration will be saved.
|
|
497
|
+
Effective only if `enable` is `True`. The parameter configuration file on each Device will be
|
|
498
|
+
saved separately, and the final file name will be `filepath_prefix + RANK_ID + ".json"`,
|
|
499
|
+
where RANK_ID is the Device ID corresponding to the file. Default: ``None`` , no configuration
|
|
500
|
+
file is saved.
|
|
476
501
|
|
|
477
502
|
Raises:
|
|
478
503
|
TypeError: If `enable` is not of type boolean.
|
|
479
|
-
TypeError: If `
|
|
480
|
-
RuntimeError: If `
|
|
481
|
-
RuntimeError: If `
|
|
482
|
-
RuntimeError: If `
|
|
483
|
-
RuntimeError: If `
|
|
504
|
+
TypeError: If `filepath_prefix` is not of type str.
|
|
505
|
+
RuntimeError: If `filepath_prefix` is an empty string.
|
|
506
|
+
RuntimeError: If `filepath_prefix` is a directory.
|
|
507
|
+
RuntimeError: If `filepath_prefix` does not exist.
|
|
508
|
+
RuntimeError: If `filepath_prefix` does not have write permission.
|
|
484
509
|
|
|
485
510
|
Note:
|
|
486
|
-
-
|
|
487
|
-
|
|
488
|
-
-
|
|
489
|
-
initialized (mindspore.communication.management.init()), otherwise the AutoTune file will always suffix with
|
|
490
|
-
rank id 0.
|
|
511
|
+
- Saved parameter profiles can be loaded via the `mindspore.dataset.deserialize` interface to
|
|
512
|
+
directly obtain a data processing pipeline object configured with optimal parameters.
|
|
513
|
+
- The parameter tuning process can be viewed by turning on INFO level logging.
|
|
491
514
|
|
|
492
|
-
An example of the generated
|
|
493
|
-
|
|
494
|
-
|
|
515
|
+
An example of the generated configuration file is as follows, the "remark" field describes whether or not data
|
|
516
|
+
processing parameter tuning has been performed, the "summary" field briefly shows each operation in the data
|
|
517
|
+
processing pipeline and its corresponding optimal configuration, and the "tree" field provides complete
|
|
518
|
+
information about the structure of the data processing pipeline.
|
|
495
519
|
|
|
496
520
|
.. code-block::
|
|
497
521
|
|
|
@@ -509,6 +533,8 @@ def set_enable_autotune(enable, filepath_prefix=None):
|
|
|
509
533
|
}
|
|
510
534
|
|
|
511
535
|
Examples:
|
|
536
|
+
>>> import mindspore.dataset as ds
|
|
537
|
+
>>>
|
|
512
538
|
>>> # enable AutoTune and save optimized data pipeline configuration
|
|
513
539
|
>>> ds.config.set_enable_autotune(True, "/path/to/autotune_out.json")
|
|
514
540
|
>>>
|
|
@@ -542,13 +568,14 @@ def set_enable_autotune(enable, filepath_prefix=None):
|
|
|
542
568
|
|
|
543
569
|
def get_enable_autotune():
|
|
544
570
|
"""
|
|
545
|
-
Get whether AutoTune is currently enabled.
|
|
571
|
+
Get whether AutoTune is currently enabled, it is disabled by default.
|
|
546
572
|
|
|
547
573
|
Returns:
|
|
548
574
|
bool, whether AutoTune is currently enabled.
|
|
549
575
|
|
|
550
576
|
Examples:
|
|
551
577
|
>>> # get the state of AutoTune
|
|
578
|
+
>>> import mindspore.dataset as ds
|
|
552
579
|
>>> autotune_flag = ds.config.get_enable_autotune()
|
|
553
580
|
"""
|
|
554
581
|
return _config.get_enable_autotune()
|
|
@@ -558,7 +585,7 @@ def set_autotune_interval(interval):
|
|
|
558
585
|
"""
|
|
559
586
|
Set the configuration adjustment interval (in steps) for AutoTune.
|
|
560
587
|
|
|
561
|
-
The default setting is 0
|
|
588
|
+
The default setting is ``0``, which will adjust the configuration after each epoch.
|
|
562
589
|
Otherwise, the configuration will be adjusted every `interval` steps.
|
|
563
590
|
|
|
564
591
|
Args:
|
|
@@ -570,6 +597,7 @@ def set_autotune_interval(interval):
|
|
|
570
597
|
|
|
571
598
|
Examples:
|
|
572
599
|
>>> # set a new interval for AutoTune
|
|
600
|
+
>>> import mindspore.dataset as ds
|
|
573
601
|
>>> ds.config.set_autotune_interval(30)
|
|
574
602
|
"""
|
|
575
603
|
if not isinstance(interval, int) or isinstance(interval, bool):
|
|
@@ -583,12 +611,14 @@ def set_autotune_interval(interval):
|
|
|
583
611
|
def get_autotune_interval():
|
|
584
612
|
"""
|
|
585
613
|
Get the current configuration adjustment interval (in steps) for AutoTune.
|
|
614
|
+
If `set_autotune_interval` is never called before, the default value(0) will be returned.
|
|
586
615
|
|
|
587
616
|
Returns:
|
|
588
617
|
int, the configuration adjustment interval (in steps) for AutoTune.
|
|
589
618
|
|
|
590
619
|
Examples:
|
|
591
620
|
>>> # get the global configuration of the autotuning interval
|
|
621
|
+
>>> import mindspore.dataset as ds
|
|
592
622
|
>>> autotune_interval = ds.config.get_autotune_interval()
|
|
593
623
|
"""
|
|
594
624
|
return _config.get_autotune_interval()
|
|
@@ -606,13 +636,12 @@ def get_enable_shared_mem():
|
|
|
606
636
|
|
|
607
637
|
Examples:
|
|
608
638
|
>>> # Get the flag of shared memory feature.
|
|
639
|
+
>>> import mindspore.dataset as ds
|
|
609
640
|
>>> shared_mem_flag = ds.config.get_enable_shared_mem()
|
|
610
641
|
"""
|
|
611
642
|
# For Windows and MacOS we forbid shared mem function temporarily
|
|
612
643
|
enable_shared_mem = _config.get_enable_shared_mem()
|
|
613
644
|
if enable_shared_mem and platform.system().lower() in {"windows", "darwin"}:
|
|
614
|
-
logger.warning(
|
|
615
|
-
"For Windows and MacOS we forbid shared mem function temporarily.")
|
|
616
645
|
_config.set_enable_shared_mem(False)
|
|
617
646
|
return False
|
|
618
647
|
return enable_shared_mem
|
|
@@ -620,20 +649,24 @@ def get_enable_shared_mem():
|
|
|
620
649
|
|
|
621
650
|
def set_enable_shared_mem(enable):
|
|
622
651
|
"""
|
|
623
|
-
Set
|
|
624
|
-
|
|
652
|
+
Set whether to use shared memory for interprocess communication when data processing multiprocessing is turned on.
|
|
653
|
+
|
|
654
|
+
Using shared memory can speed up the efficiency of data transfer between processes.
|
|
655
|
+
|
|
656
|
+
Shared memory is used by default.
|
|
625
657
|
|
|
626
658
|
Note:
|
|
627
|
-
|
|
659
|
+
Windows and MacOS systems are not supported yet.
|
|
628
660
|
|
|
629
661
|
Args:
|
|
630
|
-
enable (bool): Whether to use shared memory
|
|
662
|
+
enable (bool): Whether to use shared memory for interprocess communication.
|
|
631
663
|
|
|
632
664
|
Raises:
|
|
633
|
-
TypeError: If `enable` is not
|
|
665
|
+
TypeError: If `enable` is not of type bool.
|
|
634
666
|
|
|
635
667
|
Examples:
|
|
636
668
|
>>> # Enable shared memory feature to improve the performance of Python multiprocessing.
|
|
669
|
+
>>> import mindspore.dataset as ds
|
|
637
670
|
>>> ds.config.set_enable_shared_mem(True)
|
|
638
671
|
"""
|
|
639
672
|
if not isinstance(enable, bool):
|
|
@@ -650,17 +683,26 @@ def set_enable_shared_mem(enable):
|
|
|
650
683
|
|
|
651
684
|
def set_sending_batches(batch_num):
|
|
652
685
|
"""
|
|
653
|
-
Set the
|
|
686
|
+
Set the upper limit on the number of batches of data that the Host can send to the Device.
|
|
687
|
+
|
|
688
|
+
Can be used to implement customized data sending control logic to solve the problem of
|
|
689
|
+
Device out of memory. In each epoch, when the actual number of batches sent to the Device
|
|
690
|
+
reaches this value, the Host will stop continuing to send until the user increases this
|
|
691
|
+
upper limit again through this interface.
|
|
692
|
+
|
|
693
|
+
Currently, it is only supported when training in sink mode with Ascend backend, which can
|
|
694
|
+
be enabled via the :class:`mindspore.train.Model.train` interface.
|
|
654
695
|
|
|
655
696
|
Args:
|
|
656
|
-
batch_num (int):
|
|
657
|
-
|
|
697
|
+
batch_num (int): The upper limit on the number of batches of data that the Host can
|
|
698
|
+
send to the Device. ``0`` indicates that there is no upper limit for sending.
|
|
658
699
|
|
|
659
700
|
Raises:
|
|
660
701
|
TypeError: If `batch_num` is not of type int.
|
|
661
702
|
|
|
662
703
|
Examples:
|
|
663
704
|
>>> # Set a new global configuration value for the sending batches
|
|
705
|
+
>>> import mindspore.dataset as ds
|
|
664
706
|
>>> ds.config.set_sending_batches(10)
|
|
665
707
|
"""
|
|
666
708
|
if not isinstance(batch_num, int) or isinstance(batch_num, bool):
|
|
@@ -681,6 +723,7 @@ def set_auto_offload(offload):
|
|
|
681
723
|
|
|
682
724
|
Examples:
|
|
683
725
|
>>> # Enable automatic offload feature
|
|
726
|
+
>>> import mindspore.dataset as ds
|
|
684
727
|
>>> ds.config.set_auto_offload(True)
|
|
685
728
|
"""
|
|
686
729
|
if not isinstance(offload, bool):
|
|
@@ -690,13 +733,14 @@ def set_auto_offload(offload):
|
|
|
690
733
|
|
|
691
734
|
def get_auto_offload():
|
|
692
735
|
"""
|
|
693
|
-
Get the state of the automatic offload flag (True or False)
|
|
736
|
+
Get the state of the automatic offload flag (True or False), it is disabled by default.
|
|
694
737
|
|
|
695
738
|
Returns:
|
|
696
739
|
bool, Whether the automatic offload feature is enabled.
|
|
697
740
|
|
|
698
741
|
Examples:
|
|
699
742
|
>>> # Get the global configuration of the automatic offload feature.
|
|
743
|
+
>>> import mindspore.dataset as ds
|
|
700
744
|
>>> auto_offload = ds.config.get_auto_offload()
|
|
701
745
|
"""
|
|
702
746
|
return _config.get_auto_offload()
|
|
@@ -708,13 +752,14 @@ def set_enable_watchdog(enable):
|
|
|
708
752
|
Watchdog is a thread which cleans up hanging subprocesses.
|
|
709
753
|
|
|
710
754
|
Args:
|
|
711
|
-
enable (bool): Whether to launch a watchdog Python thread.
|
|
755
|
+
enable (bool): Whether to launch a watchdog Python thread.
|
|
712
756
|
|
|
713
757
|
Raises:
|
|
714
758
|
TypeError: If `enable` is not a boolean data type.
|
|
715
759
|
|
|
716
760
|
Examples:
|
|
717
761
|
>>> # Set a new global configuration value for the state of watchdog Python thread as enabled.
|
|
762
|
+
>>> import mindspore.dataset as ds
|
|
718
763
|
>>> ds.config.set_enable_watchdog(True)
|
|
719
764
|
"""
|
|
720
765
|
if not isinstance(enable, bool):
|
|
@@ -725,13 +770,14 @@ def set_enable_watchdog(enable):
|
|
|
725
770
|
def get_enable_watchdog():
|
|
726
771
|
"""
|
|
727
772
|
Get the state of watchdog Python thread to indicate enabled or disabled state.
|
|
728
|
-
This is
|
|
773
|
+
This is a Python watchdog thread used for all processes, it is enabled by default.
|
|
729
774
|
|
|
730
775
|
Returns:
|
|
731
776
|
bool, the default state of watchdog Python thread enabled.
|
|
732
777
|
|
|
733
778
|
Examples:
|
|
734
779
|
>>> # Get the global configuration of watchdog Python thread.
|
|
780
|
+
>>> import mindspore.dataset as ds
|
|
735
781
|
>>> watchdog_state = ds.config.get_enable_watchdog()
|
|
736
782
|
"""
|
|
737
783
|
return _config.get_enable_watchdog()
|
|
@@ -744,7 +790,7 @@ def set_multiprocessing_timeout_interval(interval):
|
|
|
744
790
|
|
|
745
791
|
Args:
|
|
746
792
|
interval (int): Interval (in seconds) to be used for multiprocessing/multithreading timeout when main
|
|
747
|
-
|
|
793
|
+
process/thread gets data from subprocess/child threads.
|
|
748
794
|
|
|
749
795
|
Raises:
|
|
750
796
|
TypeError: If `interval` is not of type int.
|
|
@@ -752,6 +798,7 @@ def set_multiprocessing_timeout_interval(interval):
|
|
|
752
798
|
|
|
753
799
|
Examples:
|
|
754
800
|
>>> # Set a new global configuration value for multiprocessing/multithreading timeout when getting data.
|
|
801
|
+
>>> import mindspore.dataset as ds
|
|
755
802
|
>>> ds.config.set_multiprocessing_timeout_interval(300)
|
|
756
803
|
"""
|
|
757
804
|
if not isinstance(interval, int) or isinstance(interval, bool):
|
|
@@ -769,12 +816,14 @@ def get_multiprocessing_timeout_interval():
|
|
|
769
816
|
|
|
770
817
|
Returns:
|
|
771
818
|
int, interval (in seconds) for multiprocessing/multithreading timeout when main process/thread gets data from
|
|
772
|
-
subprocesses/child threads.
|
|
819
|
+
subprocesses/child threads. If `set_multiprocessing_timeout_interval` is never called before,
|
|
820
|
+
the default value(300) will be returned.
|
|
773
821
|
|
|
774
822
|
Examples:
|
|
775
823
|
>>> # Get the global configuration of multiprocessing/multithreading timeout when main process/thread gets data
|
|
776
824
|
>>> # from subprocesses/child threads. If set_multiprocessing_timeout_interval() is never called before, the
|
|
777
825
|
>>> # default value(300) will be returned.
|
|
826
|
+
>>> import mindspore.dataset as ds
|
|
778
827
|
>>> multiprocessing_timeout_interval = ds.config.get_multiprocessing_timeout_interval()
|
|
779
828
|
"""
|
|
780
829
|
return _config.get_multiprocessing_timeout_interval()
|
|
@@ -791,6 +840,7 @@ def set_dynamic_shape(is_dynamic):
|
|
|
791
840
|
TypeError: If `is_dynamic` is not a boolean data type.
|
|
792
841
|
|
|
793
842
|
Examples:
|
|
843
|
+
>>> import mindspore.dataset as ds
|
|
794
844
|
>>> ds.config.set_dynamic_shape(True)
|
|
795
845
|
"""
|
|
796
846
|
if not isinstance(is_dynamic, bool):
|
|
@@ -800,11 +850,13 @@ def set_dynamic_shape(is_dynamic):
|
|
|
800
850
|
|
|
801
851
|
def get_dynamic_shape():
|
|
802
852
|
"""
|
|
803
|
-
Get the dynamic shape flag of the dataset
|
|
853
|
+
Get the dynamic shape flag of the dataset, it is set to False by default.
|
|
854
|
+
|
|
804
855
|
Returns:
|
|
805
856
|
bool, whether the dataset is dynamic shape.
|
|
806
857
|
|
|
807
858
|
Examples:
|
|
859
|
+
>>> import mindspore.dataset as ds
|
|
808
860
|
>>> is_dynamic_shape = ds.config.get_dynamic_shape()
|
|
809
861
|
"""
|
|
810
862
|
return _config.get_dynamic_shape()
|
|
@@ -816,12 +868,13 @@ def set_fast_recovery(fast_recovery):
|
|
|
816
868
|
(In fast mode, random augmentations may not get same results as before the failure occurred).
|
|
817
869
|
|
|
818
870
|
Args:
|
|
819
|
-
fast_recovery (bool): Whether the dataset pipeline recovers in fast mode.
|
|
871
|
+
fast_recovery (bool): Whether the dataset pipeline recovers in fast mode.
|
|
820
872
|
|
|
821
873
|
Raises:
|
|
822
874
|
TypeError: If `fast_recovery` is not a boolean data type.
|
|
823
875
|
|
|
824
876
|
Examples:
|
|
877
|
+
>>> import mindspore.dataset as ds
|
|
825
878
|
>>> ds.config.set_fast_recovery(False)
|
|
826
879
|
"""
|
|
827
880
|
if not isinstance(fast_recovery, bool):
|
|
@@ -832,11 +885,13 @@ def set_fast_recovery(fast_recovery):
|
|
|
832
885
|
def get_fast_recovery():
|
|
833
886
|
"""
|
|
834
887
|
Get whether the fast recovery mode is enabled for the current dataset pipeline.
|
|
888
|
+
It is set to True by default.
|
|
835
889
|
|
|
836
890
|
Returns:
|
|
837
891
|
bool, whether the dataset recovers fast in failover reset.
|
|
838
892
|
|
|
839
893
|
Examples:
|
|
894
|
+
>>> import mindspore.dataset as ds
|
|
840
895
|
>>> is_fast_recovery = ds.config.get_fast_recovery()
|
|
841
896
|
"""
|
|
842
897
|
return _config.get_fast_recovery()
|
|
@@ -848,28 +903,27 @@ def set_debug_mode(debug_mode_flag: bool, debug_hook_list: list = None):
|
|
|
848
903
|
sequentially with a single thread.
|
|
849
904
|
|
|
850
905
|
Note:
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
-
|
|
864
|
-
|
|
865
|
-
-
|
|
866
|
-
- If both debug_mode and profiling are enabled, then dataset profiling is ignored.
|
|
906
|
+
When debug_mode is enabled,
|
|
907
|
+
|
|
908
|
+
- If random seed has not been set, will internally set the seed to 1.
|
|
909
|
+
So that debug mode execution of the dataset pipeline can produce deterministic results.
|
|
910
|
+
|
|
911
|
+
- The following configuration settings are ignored:
|
|
912
|
+
|
|
913
|
+
- auto_offload (False is used.)
|
|
914
|
+
- enable_autotune (False is used.)
|
|
915
|
+
- error_samples_mode (ErrorSamplesMode.RETURN is used.)
|
|
916
|
+
- num_parallel_workers (Value 1 is used.)
|
|
917
|
+
|
|
918
|
+
- The `offload` parameter in `map` operation will be ignored.
|
|
919
|
+
- The `python_multiprocessing` parameter in `GeneratorDataset`, `map`/`batch` operation will be ignored.
|
|
920
|
+
- The `cache` parameter in Dataset loading API will be ignored.
|
|
867
921
|
|
|
868
922
|
Args:
|
|
869
923
|
debug_mode_flag (bool): Whether dataset pipeline debug mode is enabled, which forces the pipeline
|
|
870
924
|
to run synchronously and sequentially.
|
|
871
925
|
debug_hook_list (list[DebugHook]): a list of debug hook objects to be inserted before and after each
|
|
872
|
-
transform operation in map operation. Default: None
|
|
926
|
+
transform operation in map operation. Default: ``None``, which means to use basic print hook,
|
|
873
927
|
which prints shape/size/type of each input/output data of each transformation.
|
|
874
928
|
|
|
875
929
|
Raises:
|
|
@@ -878,19 +932,19 @@ def set_debug_mode(debug_mode_flag: bool, debug_hook_list: list = None):
|
|
|
878
932
|
TypeError: If any item in `debug_hook_list` is not DebugHook type.
|
|
879
933
|
|
|
880
934
|
Examples:
|
|
881
|
-
|
|
935
|
+
>>> import mindspore.dataset as ds
|
|
936
|
+
>>> import mindspore.dataset.vision as vision
|
|
937
|
+
>>> import mindspore.dataset.debug as debug
|
|
938
|
+
>>>
|
|
939
|
+
>>> # 1. Enable dataset pipeline debug mode and use default debug hook.
|
|
882
940
|
>>> # Print shape and type of input/output data of each transform op in map operator.
|
|
883
941
|
>>> ds.config.set_debug_mode(True)
|
|
884
|
-
|
|
885
|
-
2. Enable dataset pipeline debug mode and use pre-defined debug hook provided by MindData.
|
|
886
|
-
>>> import mindspore.dataset.debug as debug
|
|
887
942
|
>>>
|
|
943
|
+
>>> # 2. Enable dataset pipeline debug mode and use pre-defined debug hook provided by MindData.
|
|
888
944
|
>>> ds.config.set_debug_mode(True, debug_hook_list=[debug.PrintDataHook()])
|
|
889
|
-
|
|
890
|
-
3. Enable dataset pipeline debug mode and use user-defined debug hook. It must define a
|
|
891
|
-
class inherited from DebugHook.
|
|
892
|
-
>>> import mindspore.dataset.debug as debug
|
|
893
945
|
>>>
|
|
946
|
+
>>> # 3. Enable dataset pipeline debug mode and use user-defined debug hook. It must define a
|
|
947
|
+
>>> # class inherited from DebugHook.
|
|
894
948
|
>>> class CustomizedHook(debug.DebugHook):
|
|
895
949
|
... def __init__(self):
|
|
896
950
|
... super().__init__()
|
|
@@ -900,16 +954,17 @@ def set_debug_mode(debug_mode_flag: bool, debug_hook_list: list = None):
|
|
|
900
954
|
... return args
|
|
901
955
|
>>>
|
|
902
956
|
>>> ds.config.set_debug_mode(True, debug_hook_list=[CustomizedHook()])
|
|
903
|
-
|
|
904
|
-
4. Enable dataset pipeline debug mode and use user-defined debug hook and insert by users manually.
|
|
957
|
+
>>>
|
|
958
|
+
>>> # 4. Enable dataset pipeline debug mode and use user-defined debug hook and insert by users manually.
|
|
905
959
|
>>> ds.config.set_debug_mode(True)
|
|
906
960
|
>>> dataset = ds.ImageFolderDataset(dataset_dir="/path/to/image_folder_dataset_directory")
|
|
907
|
-
>>>
|
|
961
|
+
>>>
|
|
962
|
+
>>> # The debug hook is added after Decode operation.
|
|
908
963
|
>>> dataset = dataset.map([vision.Decode(), CustomizedHook(), vision.CenterCrop(100)])
|
|
909
964
|
"""
|
|
910
965
|
if not isinstance(debug_mode_flag, bool):
|
|
911
966
|
raise TypeError("debug_mode_flag isn't of type boolean.")
|
|
912
|
-
if
|
|
967
|
+
if debug_hook_list is None:
|
|
913
968
|
debug_hook_list = [PrintMetaDataHook()]
|
|
914
969
|
if not isinstance(debug_hook_list, list):
|
|
915
970
|
raise TypeError("debug_hook_list is not a list.")
|
|
@@ -927,12 +982,13 @@ def set_debug_mode(debug_mode_flag: bool, debug_hook_list: list = None):
|
|
|
927
982
|
|
|
928
983
|
def get_debug_mode():
|
|
929
984
|
"""
|
|
930
|
-
Get
|
|
985
|
+
Get whether debug mode is currently enabled for the data pipeline.
|
|
931
986
|
|
|
932
987
|
Returns:
|
|
933
|
-
bool, whether
|
|
988
|
+
bool, whether data pipeline debug mode is enabled.
|
|
934
989
|
|
|
935
990
|
Examples:
|
|
991
|
+
>>> import mindspore.dataset as ds
|
|
936
992
|
>>> debug_mode = ds.config.get_debug_mode()
|
|
937
993
|
"""
|
|
938
994
|
return _config.get_debug_mode()
|
|
@@ -986,8 +1042,8 @@ def set_error_samples_mode(error_samples_mode):
|
|
|
986
1042
|
|
|
987
1043
|
Note:
|
|
988
1044
|
- This error samples feature is only applicable to the Map operation in a dataset pipeline.
|
|
989
|
-
- For
|
|
990
|
-
- If
|
|
1045
|
+
- For ``ErrorSamplesMode.REPLACE`` mode, a cache of other samples will be used.
|
|
1046
|
+
- If ``ErrorSamplesMode.SKIP`` mode is used in a distributed setting, beware to manually ensure the
|
|
991
1047
|
number of valid samples are the same for each shard (otherwise one may encounter hangs).
|
|
992
1048
|
One technique is to manually concat a dataset of all valid samples plus a
|
|
993
1049
|
take operation for the number of skipped erroneous samples.
|
|
@@ -995,18 +1051,18 @@ def set_error_samples_mode(error_samples_mode):
|
|
|
995
1051
|
Args:
|
|
996
1052
|
error_samples_mode (ErrorSamplesMode): The method in which erroneous samples should be processed in a dataset
|
|
997
1053
|
pipeline. It can be any of [ErrorSamplesMode.RETURN, ErrorSamplesMode.REPLACE, ErrorSamplesMode.SKIP].
|
|
998
|
-
System default: ErrorSamplesMode.RETURN.
|
|
999
1054
|
|
|
1000
|
-
- ErrorSamplesMode.RETURN
|
|
1055
|
+
- ``ErrorSamplesMode.RETURN``: means erroneous sample results in error raised and returned.
|
|
1001
1056
|
|
|
1002
|
-
- ErrorSamplesMode.REPLACE
|
|
1057
|
+
- ``ErrorSamplesMode.REPLACE``: means erroneous sample is replaced with a correct sample.
|
|
1003
1058
|
|
|
1004
|
-
- ErrorSamplesMode.SKIP
|
|
1059
|
+
- ``ErrorSamplesMode.SKIP``: means erroneous sample is skipped.
|
|
1005
1060
|
|
|
1006
1061
|
Raises:
|
|
1007
1062
|
TypeError: If `error_samples_mode` is not of type ErrorSamplesMode.
|
|
1008
1063
|
|
|
1009
1064
|
Examples:
|
|
1065
|
+
>>> import mindspore.dataset as ds
|
|
1010
1066
|
>>> ds.config.set_error_samples_mode(ds.config.ErrorSamplesMode.SKIP)
|
|
1011
1067
|
"""
|
|
1012
1068
|
type_check(error_samples_mode, (ErrorSamplesMode,), "error_samples_mode")
|
|
@@ -1016,6 +1072,7 @@ def set_error_samples_mode(error_samples_mode):
|
|
|
1016
1072
|
def get_error_samples_mode():
|
|
1017
1073
|
"""
|
|
1018
1074
|
Get the current configuration for strategy for processing erroneous samples in a dataset pipeline.
|
|
1075
|
+
If `set_error_samples_mode` is never called before, the default setting is ErrorSamplesMode.RETURN.
|
|
1019
1076
|
|
|
1020
1077
|
Returns:
|
|
1021
1078
|
ErrorSamplesMode, The method in which erroneous samples should be processed in a dataset pipeline.
|
|
@@ -1025,6 +1082,7 @@ def get_error_samples_mode():
|
|
|
1025
1082
|
- ErrorSamplesMode.SKIP: means erroneous sample is skipped.
|
|
1026
1083
|
|
|
1027
1084
|
Examples:
|
|
1085
|
+
>>> import mindspore.dataset as ds
|
|
1028
1086
|
>>> error_samples_mode = ds.config.get_error_samples_mode()
|
|
1029
1087
|
"""
|
|
1030
1088
|
return _CDE_TO_PYTHON_ERROR_SAMPLES_MODE.get(_config.get_error_samples_mode())
|