mindspore 2.0.0rc1__cp38-cp38-manylinux1_x86_64.whl → 2.2.0__cp38-cp38-manylinux1_x86_64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mindspore might be problematic. Click here for more details.

Files changed (884) hide show
  1. mindspore/.commit_id +1 -1
  2. mindspore/Third_Party_Open_Source_Software_Notice +2 -2
  3. mindspore/__init__.py +5 -2
  4. mindspore/_akg/akg/build_module.py +5 -6
  5. mindspore/_akg/akg/composite/build_module.py +49 -16
  6. mindspore/_akg/akg/composite/split_stitch.py +10 -11
  7. mindspore/_akg/akg/config/repository.json +195 -0
  8. mindspore/_akg/akg/global_configs.py +5 -1
  9. mindspore/_akg/akg/ms/info_version_adapt.py +67 -1
  10. mindspore/_akg/akg/tvm/api.py +4 -3
  11. mindspore/_akg/akg/tvm/autotvm/__init__.py +1 -2
  12. mindspore/_akg/akg/tvm/autotvm/graph_tuner/base_graph_tuner.py +1 -5
  13. mindspore/_akg/akg/tvm/autotvm/measure/__init__.py +1 -1
  14. mindspore/_akg/akg/tvm/autotvm/measure/measure.py +1 -10
  15. mindspore/_akg/akg/tvm/autotvm/measure/measure_methods.py +1 -372
  16. mindspore/_akg/akg/tvm/build_module.py +16 -1
  17. mindspore/_akg/akg/tvm/contrib/graph_runtime.py +0 -53
  18. mindspore/_akg/akg/tvm/hybrid/parser.py +7 -6
  19. mindspore/_akg/akg/tvm/ir_builder.py +1 -1
  20. mindspore/_akg/akg/tvm/module.py +1 -2
  21. mindspore/_akg/akg/tvm/stmt.py +2 -2
  22. mindspore/_akg/akg/utils/composite_op_helper.py +9 -10
  23. mindspore/_akg/akg/utils/kernel_exec.py +58 -260
  24. mindspore/_akg/akg/utils/op_dsl.py +17 -1
  25. mindspore/_akg/akg/utils/result_analysis.py +4 -24
  26. mindspore/_akg/akg/utils/tbe_codegen_utils.py +198 -0
  27. mindspore/_c_dataengine.cpython-38-x86_64-linux-gnu.so +0 -0
  28. mindspore/_c_expression.cpython-38-x86_64-linux-gnu.so +0 -0
  29. mindspore/_c_mindrecord.cpython-38-x86_64-linux-gnu.so +0 -0
  30. mindspore/_check_jit_forbidden_api.py +5 -1
  31. mindspore/_checkparam.py +79 -62
  32. mindspore/_extends/graph_kernel/__init__.py +0 -1
  33. mindspore/_extends/graph_kernel/model/graph_split.py +2 -0
  34. mindspore/_extends/graph_kernel/model/model_builder.py +9 -50
  35. mindspore/_extends/graph_kernel/splitter.py +1 -9
  36. mindspore/_extends/parallel_compile/akg_compiler/akg_process.py +128 -21
  37. mindspore/_extends/parallel_compile/akg_compiler/build_tbe_kernel.py +2 -2
  38. mindspore/_extends/parallel_compile/akg_compiler/tbe_topi.py +4 -2
  39. mindspore/_extends/parallel_compile/tbe_compiler/tbe_adapter.py +18 -13
  40. mindspore/_extends/parallel_compile/tbe_compiler/tbe_helper.py +13 -9
  41. mindspore/_extends/parallel_compile/tbe_compiler/tbe_job.py +1 -1
  42. mindspore/_extends/parallel_compile/tbe_compiler/tbe_job_manager.py +1 -1
  43. mindspore/_extends/parse/__init__.py +19 -17
  44. mindspore/_extends/parse/namespace.py +7 -36
  45. mindspore/_extends/parse/parser.py +375 -189
  46. mindspore/_extends/parse/resources.py +36 -41
  47. mindspore/_extends/parse/standard_method.py +350 -245
  48. mindspore/_extends/parse/trope.py +2 -12
  49. mindspore/_extends/remote/kernel_build_server.py +24 -7
  50. mindspore/_extends/remote/kernel_build_server_akg_v2.py +55 -0
  51. mindspore/_install_custom.py +43 -0
  52. mindspore/_mindspore_offline_debug.cpython-38-x86_64-linux-gnu.so +0 -0
  53. mindspore/amp.py +85 -19
  54. mindspore/bin/cache_admin +0 -0
  55. mindspore/bin/cache_server +0 -0
  56. mindspore/boost/base.py +2 -2
  57. mindspore/boost/boost.py +27 -32
  58. mindspore/boost/boost_cell_wrapper.py +37 -13
  59. mindspore/boost/grad_accumulation.py +1 -1
  60. mindspore/boost/grad_freeze.py +34 -6
  61. mindspore/boost/group_loss_scale_manager.py +15 -14
  62. mindspore/boost/less_batch_normalization.py +28 -3
  63. mindspore/common/__init__.py +15 -11
  64. mindspore/common/_auto_dynamic.py +68 -0
  65. mindspore/common/_jit_fallback_utils.py +111 -0
  66. mindspore/common/_register_for_adapter.py +17 -5
  67. mindspore/common/_register_for_tensor.py +2 -2
  68. mindspore/common/_stub_tensor.py +18 -15
  69. mindspore/common/_utils.py +31 -7
  70. mindspore/common/api.py +269 -101
  71. mindspore/common/auto_dynamic_shape.py +498 -0
  72. mindspore/common/dtype.py +61 -21
  73. mindspore/common/dump.py +9 -7
  74. mindspore/common/initializer.py +106 -76
  75. mindspore/common/jit_config.py +35 -14
  76. mindspore/common/lazy_inline.py +187 -0
  77. mindspore/common/mindir_util.py +101 -0
  78. mindspore/common/mutable.py +10 -13
  79. mindspore/common/parameter.py +246 -55
  80. mindspore/common/seed.py +13 -7
  81. mindspore/common/sparse_tensor.py +29 -33
  82. mindspore/common/tensor.py +907 -251
  83. mindspore/communication/__init__.py +7 -4
  84. mindspore/communication/_comm_helper.py +84 -4
  85. mindspore/communication/management.py +160 -88
  86. mindspore/config/op_info.config +99 -75
  87. mindspore/config/super_bar_config.json +36 -4
  88. mindspore/context.py +526 -219
  89. mindspore/dataset/__init__.py +9 -46
  90. mindspore/dataset/audio/__init__.py +4 -19
  91. mindspore/dataset/audio/transforms.py +545 -233
  92. mindspore/dataset/audio/utils.py +21 -18
  93. mindspore/dataset/callback/ds_callback.py +42 -13
  94. mindspore/dataset/core/config.py +158 -100
  95. mindspore/dataset/core/validator_helpers.py +1 -63
  96. mindspore/dataset/debug/debug_hook.py +45 -13
  97. mindspore/dataset/debug/pre_defined_hook.py +5 -5
  98. mindspore/dataset/engine/__init__.py +0 -5
  99. mindspore/dataset/engine/cache_client.py +38 -15
  100. mindspore/dataset/engine/datasets.py +615 -278
  101. mindspore/dataset/engine/datasets_audio.py +154 -283
  102. mindspore/dataset/engine/datasets_standard_format.py +104 -116
  103. mindspore/dataset/engine/datasets_text.py +443 -326
  104. mindspore/dataset/engine/datasets_user_defined.py +251 -164
  105. mindspore/dataset/engine/datasets_vision.py +839 -1443
  106. mindspore/dataset/engine/iterators.py +11 -4
  107. mindspore/dataset/engine/obs/obs_mindrecord_dataset.py +7 -3
  108. mindspore/dataset/engine/obs/util.py +3 -0
  109. mindspore/dataset/engine/offload.py +6 -6
  110. mindspore/dataset/engine/queue.py +15 -14
  111. mindspore/dataset/engine/samplers.py +39 -23
  112. mindspore/dataset/engine/serializer_deserializer.py +22 -6
  113. mindspore/dataset/engine/validators.py +21 -331
  114. mindspore/dataset/text/__init__.py +5 -33
  115. mindspore/dataset/text/transforms.py +334 -165
  116. mindspore/dataset/text/utils.py +215 -145
  117. mindspore/dataset/transforms/__init__.py +1 -1
  118. mindspore/dataset/transforms/c_transforms.py +3 -2
  119. mindspore/dataset/transforms/py_transforms_util.py +40 -12
  120. mindspore/dataset/transforms/transforms.py +174 -71
  121. mindspore/dataset/utils/browse_dataset.py +25 -17
  122. mindspore/dataset/utils/line_reader.py +24 -21
  123. mindspore/dataset/vision/__init__.py +5 -26
  124. mindspore/dataset/vision/c_transforms.py +177 -165
  125. mindspore/dataset/vision/py_transforms.py +114 -119
  126. mindspore/dataset/vision/py_transforms_util.py +54 -51
  127. mindspore/dataset/vision/transforms.py +1127 -381
  128. mindspore/dataset/vision/utils.py +54 -38
  129. mindspore/dataset/vision/validators.py +12 -2
  130. mindspore/experimental/map_parameter.py +38 -4
  131. mindspore/{dataset/datapreprocess → experimental/optim}/__init__.py +14 -4
  132. mindspore/experimental/optim/adam.py +192 -0
  133. mindspore/experimental/optim/adamw.py +181 -0
  134. mindspore/experimental/optim/lr_scheduler.py +1427 -0
  135. mindspore/experimental/optim/optimizer.py +252 -0
  136. mindspore/experimental/optim/sgd.py +147 -0
  137. mindspore/gen_ops.py +273 -0
  138. mindspore/include/OWNERS +1 -2
  139. mindspore/include/api/context.h +21 -1
  140. mindspore/include/api/data_type.h +2 -1
  141. mindspore/include/api/graph.h +0 -15
  142. mindspore/include/api/kernel.h +2 -0
  143. mindspore/include/api/kernel_api.h +37 -12
  144. mindspore/include/api/model.h +29 -42
  145. mindspore/include/api/model_group.h +14 -3
  146. mindspore/include/api/model_parallel_runner.h +18 -2
  147. mindspore/include/api/serialization.h +26 -0
  148. mindspore/include/api/status.h +1 -0
  149. mindspore/include/api/types.h +38 -4
  150. mindspore/include/c_api/ms/abstract.h +67 -0
  151. mindspore/include/c_api/ms/attribute.h +197 -0
  152. mindspore/include/c_api/ms/base/handle_types.h +43 -0
  153. mindspore/include/c_api/ms/base/macros.h +32 -0
  154. mindspore/include/c_api/ms/base/status.h +33 -0
  155. mindspore/include/c_api/ms/base/types.h +282 -0
  156. mindspore/include/c_api/ms/context.h +102 -0
  157. mindspore/include/c_api/ms/graph.h +160 -0
  158. mindspore/include/c_api/ms/node.h +606 -0
  159. mindspore/include/c_api/ms/tensor.h +161 -0
  160. mindspore/include/c_api/ms/value.h +84 -0
  161. mindspore/include/c_api/status_c.h +3 -0
  162. mindspore/include/dataset/constants.h +6 -12
  163. mindspore/include/dataset/execute.h +23 -13
  164. mindspore/include/dataset/text.h +26 -26
  165. mindspore/include/dataset/transforms.h +25 -31
  166. mindspore/include/dataset/vision.h +60 -60
  167. mindspore/include/dataset/vision_ascend.h +5 -6
  168. mindspore/include/dataset/vision_lite.h +17 -17
  169. mindspore/include/mindapi/base/format.h +0 -1
  170. mindspore/include/mindapi/base/type_id.h +2 -1
  171. mindspore/include/mindapi/base/types.h +5 -1
  172. mindspore/lib/libdnnl.so.2 +0 -0
  173. mindspore/lib/libjemalloc.so.2 +0 -0
  174. mindspore/lib/libmindspore.so +0 -0
  175. mindspore/lib/libmindspore_backend.so +0 -0
  176. mindspore/lib/libmindspore_common.so +0 -0
  177. mindspore/lib/libmindspore_core.so +0 -0
  178. mindspore/lib/libmindspore_glog.so.0 +0 -0
  179. mindspore/lib/libmindspore_gpr.so.15 +0 -0
  180. mindspore/lib/libmindspore_grpc++.so.1 +0 -0
  181. mindspore/lib/libmindspore_grpc.so.15 +0 -0
  182. mindspore/lib/libmindspore_shared_lib.so +0 -0
  183. mindspore/lib/libmpi_adapter.so +0 -0
  184. mindspore/lib/libnnacl.so +0 -0
  185. mindspore/lib/libopencv_core.so.4.5 +0 -0
  186. mindspore/lib/libopencv_imgcodecs.so.4.5 +0 -0
  187. mindspore/lib/libopencv_imgproc.so.4.5 +0 -0
  188. mindspore/lib/libps_cache.so +0 -0
  189. mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/aicpu_kernel/impl/libcust_aicpu_kernels.so +0 -0
  190. mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/aicpu_kernel/impl/libcust_cpu_kernels.so +0 -0
  191. mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/config/cust_aicpu_kernel.json +9000 -0
  192. mindspore/lib/plugin/ascend/custom_aicpu_ops/op_proto/libcust_op_proto.so +0 -0
  193. mindspore/lib/plugin/ascend/libakg.so +0 -0
  194. mindspore/lib/plugin/ascend/libascend_collective.so +0 -0
  195. mindspore/lib/plugin/ascend/libdvpp_utils.so +0 -0
  196. mindspore/lib/plugin/ascend/libhccl_plugin.so +0 -0
  197. mindspore/lib/plugin/ascend/libmindspore_aicpu_kernels.so +0 -0
  198. mindspore/lib/plugin/ascend/libmindspore_cpu_kernels.so +0 -0
  199. mindspore/lib/plugin/cpu/libakg.so +0 -0
  200. mindspore/lib/plugin/gpu/libcuda_ops.so.10 +0 -0
  201. mindspore/lib/plugin/gpu/libcuda_ops.so.11 +0 -0
  202. mindspore/lib/plugin/gpu10.1/libakg.so +0 -0
  203. mindspore/lib/plugin/gpu10.1/libnccl.so.2 +0 -0
  204. mindspore/lib/plugin/gpu10.1/libnvidia_collective.so +0 -0
  205. mindspore/lib/plugin/gpu11.1/libakg.so +0 -0
  206. mindspore/lib/plugin/gpu11.1/libnccl.so.2 +0 -0
  207. mindspore/lib/plugin/gpu11.1/libnvidia_collective.so +0 -0
  208. mindspore/lib/plugin/gpu11.6/libakg.so +0 -0
  209. mindspore/lib/plugin/gpu11.6/libnccl.so.2 +0 -0
  210. mindspore/lib/plugin/gpu11.6/libnvidia_collective.so +0 -0
  211. mindspore/lib/plugin/libmindspore_ascend.so.1 +0 -0
  212. mindspore/lib/plugin/libmindspore_ascend.so.2 +0 -0
  213. mindspore/lib/plugin/libmindspore_gpu.so.10.1 +0 -0
  214. mindspore/lib/plugin/libmindspore_gpu.so.11.1 +0 -0
  215. mindspore/lib/plugin/libmindspore_gpu.so.11.6 +0 -0
  216. mindspore/log.py +9 -6
  217. mindspore/mindrecord/filereader.py +33 -4
  218. mindspore/mindrecord/filewriter.py +70 -35
  219. mindspore/mindrecord/mindpage.py +40 -34
  220. mindspore/mindrecord/shardreader.py +1 -1
  221. mindspore/mindrecord/shardsegment.py +1 -1
  222. mindspore/mindrecord/tools/cifar100_to_mr.py +25 -18
  223. mindspore/mindrecord/tools/cifar10_to_mr.py +25 -18
  224. mindspore/mindrecord/tools/csv_to_mr.py +29 -13
  225. mindspore/mindrecord/tools/imagenet_to_mr.py +24 -10
  226. mindspore/mindrecord/tools/mnist_to_mr.py +24 -11
  227. mindspore/mindrecord/tools/tfrecord_to_mr.py +31 -26
  228. mindspore/nn/cell.py +463 -169
  229. mindspore/nn/dynamic_lr.py +47 -43
  230. mindspore/nn/layer/activation.py +225 -82
  231. mindspore/nn/layer/basic.py +121 -79
  232. mindspore/nn/layer/channel_shuffle.py +21 -21
  233. mindspore/nn/layer/combined.py +33 -26
  234. mindspore/nn/layer/container.py +277 -22
  235. mindspore/nn/layer/conv.py +441 -304
  236. mindspore/nn/layer/dense.py +19 -13
  237. mindspore/nn/layer/embedding.py +62 -49
  238. mindspore/nn/layer/flash_attention.py +264 -0
  239. mindspore/nn/layer/image.py +50 -39
  240. mindspore/nn/layer/math.py +62 -51
  241. mindspore/nn/layer/normalization.py +219 -167
  242. mindspore/nn/layer/padding.py +58 -70
  243. mindspore/nn/layer/pooling.py +334 -287
  244. mindspore/nn/layer/rnn_cells.py +53 -38
  245. mindspore/nn/layer/rnns.py +59 -56
  246. mindspore/nn/layer/thor_layer.py +52 -44
  247. mindspore/nn/layer/timedistributed.py +6 -4
  248. mindspore/nn/layer/transformer.py +284 -164
  249. mindspore/nn/learning_rate_schedule.py +34 -25
  250. mindspore/nn/loss/__init__.py +3 -2
  251. mindspore/nn/loss/loss.py +554 -311
  252. mindspore/nn/optim/ada_grad.py +12 -9
  253. mindspore/nn/optim/adadelta.py +14 -11
  254. mindspore/nn/optim/adafactor.py +19 -16
  255. mindspore/nn/optim/adam.py +62 -47
  256. mindspore/nn/optim/adamax.py +13 -10
  257. mindspore/nn/optim/adasum.py +12 -8
  258. mindspore/nn/optim/asgd.py +10 -9
  259. mindspore/nn/optim/ftrl.py +20 -17
  260. mindspore/nn/optim/lamb.py +16 -12
  261. mindspore/nn/optim/lars.py +8 -6
  262. mindspore/nn/optim/lazyadam.py +25 -20
  263. mindspore/nn/optim/momentum.py +10 -7
  264. mindspore/nn/optim/optimizer.py +61 -9
  265. mindspore/nn/optim/proximal_ada_grad.py +14 -13
  266. mindspore/nn/optim/rmsprop.py +17 -13
  267. mindspore/nn/optim/rprop.py +30 -17
  268. mindspore/nn/optim/sgd.py +40 -23
  269. mindspore/nn/optim/thor.py +24 -26
  270. mindspore/nn/probability/bijector/bijector.py +11 -11
  271. mindspore/nn/probability/bijector/exp.py +1 -1
  272. mindspore/nn/probability/bijector/gumbel_cdf.py +3 -3
  273. mindspore/nn/probability/bijector/invert.py +1 -1
  274. mindspore/nn/probability/bijector/power_transform.py +29 -29
  275. mindspore/nn/probability/bijector/scalar_affine.py +3 -3
  276. mindspore/nn/probability/bijector/softplus.py +5 -5
  277. mindspore/nn/probability/bnn_layers/bnn_cell_wrapper.py +4 -2
  278. mindspore/nn/probability/bnn_layers/conv_variational.py +13 -13
  279. mindspore/nn/probability/bnn_layers/dense_variational.py +12 -12
  280. mindspore/nn/probability/bnn_layers/layer_distribution.py +9 -8
  281. mindspore/nn/probability/distribution/_utils/custom_ops.py +19 -3
  282. mindspore/nn/probability/distribution/_utils/utils.py +1 -1
  283. mindspore/nn/probability/distribution/bernoulli.py +9 -9
  284. mindspore/nn/probability/distribution/beta.py +8 -8
  285. mindspore/nn/probability/distribution/categorical.py +23 -15
  286. mindspore/nn/probability/distribution/cauchy.py +5 -6
  287. mindspore/nn/probability/distribution/distribution.py +3 -3
  288. mindspore/nn/probability/distribution/exponential.py +4 -4
  289. mindspore/nn/probability/distribution/gamma.py +10 -10
  290. mindspore/nn/probability/distribution/geometric.py +8 -8
  291. mindspore/nn/probability/distribution/gumbel.py +8 -9
  292. mindspore/nn/probability/distribution/half_normal.py +5 -5
  293. mindspore/nn/probability/distribution/laplace.py +5 -5
  294. mindspore/nn/probability/distribution/log_normal.py +12 -11
  295. mindspore/nn/probability/distribution/logistic.py +8 -8
  296. mindspore/nn/probability/distribution/normal.py +6 -5
  297. mindspore/nn/probability/distribution/poisson.py +10 -11
  298. mindspore/nn/probability/distribution/student_t.py +8 -9
  299. mindspore/nn/probability/distribution/transformed_distribution.py +5 -5
  300. mindspore/nn/probability/distribution/uniform.py +11 -11
  301. mindspore/nn/reinforcement/tensor_array.py +2 -2
  302. mindspore/nn/sparse/sparse.py +9 -9
  303. mindspore/nn/wrap/cell_wrapper.py +188 -63
  304. mindspore/nn/wrap/grad_reducer.py +21 -12
  305. mindspore/nn/wrap/loss_scale.py +136 -49
  306. mindspore/numpy/__init__.py +4 -4
  307. mindspore/numpy/array_creations.py +55 -56
  308. mindspore/numpy/array_ops.py +134 -35
  309. mindspore/numpy/logic_ops.py +66 -20
  310. mindspore/numpy/math_ops.py +142 -139
  311. mindspore/numpy/utils_const.py +2 -2
  312. mindspore/offline_debug/convert_async.py +2 -2
  313. mindspore/ops/_grad_experimental/__init__.py +7 -5
  314. mindspore/ops/_grad_experimental/grad_array_ops.py +231 -348
  315. mindspore/ops/{_grad → _grad_experimental}/grad_base.py +1 -33
  316. mindspore/ops/{_grad → _grad_experimental}/grad_comm_ops.py +25 -13
  317. mindspore/ops/{_grad/__init__.py → _grad_experimental/grad_debug_ops.py} +15 -7
  318. mindspore/ops/{_grad → _grad_experimental}/grad_implementations.py +17 -11
  319. mindspore/ops/_grad_experimental/grad_inner_ops.py +33 -52
  320. mindspore/ops/_grad_experimental/grad_math_ops.py +151 -1224
  321. mindspore/ops/_grad_experimental/grad_nn_ops.py +141 -414
  322. mindspore/ops/{_grad → _grad_experimental}/grad_quant_ops.py +10 -6
  323. mindspore/ops/_grad_experimental/grad_sparse.py +317 -2
  324. mindspore/ops/_grad_experimental/grad_sparse_ops.py +3 -13
  325. mindspore/ops/{_grad → _grad_experimental}/taylor_rule.py +1 -1
  326. mindspore/ops/_op_impl/_custom_op/dsd_back_impl.py +1 -1
  327. mindspore/ops/_op_impl/_custom_op/flash_attention/__init__.py +0 -0
  328. mindspore/ops/_op_impl/_custom_op/flash_attention/attention.py +406 -0
  329. mindspore/{_extends/graph_kernel/expanders/complex/__init__.py → ops/_op_impl/_custom_op/flash_attention/constants.py} +27 -8
  330. mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_bwd.py +467 -0
  331. mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_fwd.py +563 -0
  332. mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_impl.py +193 -0
  333. mindspore/ops/_op_impl/_custom_op/flash_attention/tik_ops_utils.py +435 -0
  334. mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/__init__.py +0 -0
  335. mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/sparse_tiling.py +45 -0
  336. mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/strategy.py +67 -0
  337. mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/wukong_tiling.py +62 -0
  338. mindspore/ops/_op_impl/_custom_op/matmul_cube_dense_left_impl.py +2 -2
  339. mindspore/ops/_op_impl/aicpu/__init__.py +41 -1
  340. mindspore/ops/_op_impl/aicpu/adaptive_max_pool_2d.py +37 -0
  341. mindspore/ops/_op_impl/aicpu/bias_add_grad.py +0 -1
  342. mindspore/ops/_op_impl/aicpu/cast.py +52 -0
  343. mindspore/ops/_op_impl/aicpu/coalesce.py +2 -0
  344. mindspore/ops/_op_impl/aicpu/col2im.py +3 -1
  345. mindspore/ops/_op_impl/aicpu/count_nonzero.py +43 -0
  346. mindspore/ops/_op_impl/aicpu/dropout_genmask.py +6 -0
  347. mindspore/ops/_op_impl/aicpu/eps.py +32 -0
  348. mindspore/ops/_op_impl/aicpu/eye.py +4 -4
  349. mindspore/ops/_op_impl/aicpu/fft_with_size.py +6 -0
  350. mindspore/ops/_op_impl/aicpu/fill_diagonal.py +5 -0
  351. mindspore/ops/_op_impl/aicpu/gamma.py +2 -2
  352. mindspore/ops/_op_impl/aicpu/im2col.py +3 -5
  353. mindspore/ops/_op_impl/aicpu/lgamma.py +1 -0
  354. mindspore/ops/_op_impl/aicpu/log_uniform_candidate_sampler.py +6 -3
  355. mindspore/ops/_op_impl/aicpu/lu.py +39 -0
  356. mindspore/ops/_op_impl/aicpu/lu_unpack_grad.py +0 -1
  357. mindspore/ops/_op_impl/aicpu/masked_scatter.py +1 -0
  358. mindspore/ops/_op_impl/aicpu/masked_select_grad.py +3 -0
  359. mindspore/ops/_op_impl/aicpu/matrix_band_part.py +59 -0
  360. mindspore/ops/_op_impl/aicpu/matrix_power.py +6 -1
  361. mindspore/ops/_op_impl/aicpu/median.py +1 -0
  362. mindspore/ops/_op_impl/aicpu/multinomial.py +9 -9
  363. mindspore/ops/_op_impl/aicpu/not_equal.py +0 -5
  364. mindspore/ops/_op_impl/aicpu/pad_v3.py +3 -1
  365. mindspore/ops/_op_impl/aicpu/pad_v3_grad.py +2 -0
  366. mindspore/ops/_op_impl/aicpu/parameterized_truncated_normal.py +15 -7
  367. mindspore/ops/_op_impl/aicpu/random_categorical.py +39 -19
  368. mindspore/ops/_op_impl/aicpu/random_choice_with_mask.py +5 -2
  369. mindspore/ops/_op_impl/aicpu/random_poisson.py +103 -52
  370. mindspore/ops/_op_impl/aicpu/random_shuffle.py +17 -15
  371. mindspore/ops/_op_impl/aicpu/resize_bilinear_grad.py +0 -1
  372. mindspore/ops/_op_impl/aicpu/resize_nearest_neighbor_v2.py +0 -6
  373. mindspore/ops/_op_impl/aicpu/resize_nearest_neighbor_v2_grad.py +0 -7
  374. mindspore/ops/_op_impl/aicpu/scatter_nd.py +2 -0
  375. mindspore/ops/_op_impl/aicpu/sequence_concat.py +40 -0
  376. mindspore/ops/_op_impl/aicpu/sequence_stack.py +40 -0
  377. mindspore/ops/_op_impl/aicpu/{sparseaddmm.py → sparse_addmm.py} +2 -2
  378. mindspore/ops/_op_impl/aicpu/{sparsesparsemaximum.py → sparse_sparse_maximum.py} +4 -4
  379. mindspore/ops/_op_impl/aicpu/standard_laplace.py +5 -4
  380. mindspore/ops/_op_impl/aicpu/standard_normal.py +5 -4
  381. mindspore/ops/_op_impl/aicpu/truncated_normal.py +9 -7
  382. mindspore/ops/_op_impl/aicpu/uniform.py +5 -3
  383. mindspore/ops/_op_impl/aicpu/uniform_candidate_sampler.py +8 -4
  384. mindspore/ops/_op_impl/aicpu/uniform_int.py +5 -5
  385. mindspore/ops/_op_impl/aicpu/uniform_real.py +4 -4
  386. mindspore/ops/_op_impl/aicpu/upsample_nearest_3d.py +14 -6
  387. mindspore/ops/_op_impl/aicpu/upsample_nearest_3d_grad.py +22 -8
  388. mindspore/ops/_op_impl/aicpu/upsample_trilinear_3d.py +11 -6
  389. mindspore/ops/_op_impl/aicpu/upsample_trilinear_3d_grad.py +21 -10
  390. mindspore/ops/_op_impl/tbe/__init__.py +6 -4
  391. mindspore/ops/_op_impl/tbe/atomic_addr_clean.py +1 -1
  392. mindspore/ops/_op_impl/tbe/avg_pool.py +2 -2
  393. mindspore/ops/_op_impl/tbe/avg_pool_3d.py +3 -3
  394. mindspore/ops/_op_impl/tbe/avg_pool_3d_grad.py +4 -4
  395. mindspore/ops/_op_impl/tbe/avg_pool_ds.py +2 -2
  396. mindspore/ops/_op_impl/tbe/avg_pool_grad.py +3 -3
  397. mindspore/ops/_op_impl/tbe/avg_pool_grad_vm.py +3 -3
  398. mindspore/ops/_op_impl/tbe/batch_to_space.py +1 -1
  399. mindspore/ops/_op_impl/tbe/batch_to_space_nd.py +2 -2
  400. mindspore/ops/_op_impl/tbe/bn_infer.py +2 -2
  401. mindspore/ops/_op_impl/tbe/bn_infer_ds.py +3 -2
  402. mindspore/ops/_op_impl/tbe/broadcast_to.py +1 -1
  403. mindspore/ops/_op_impl/tbe/depthwise_conv2d.py +3 -3
  404. mindspore/ops/_op_impl/tbe/expand_dims.py +1 -1
  405. mindspore/ops/_op_impl/tbe/gather_v2.py +56 -0
  406. mindspore/ops/_op_impl/tbe/im2col.py +4 -4
  407. mindspore/ops/_op_impl/tbe/inplace_index_add.py +7 -3
  408. mindspore/ops/_op_impl/tbe/mem_set.py +38 -0
  409. mindspore/ops/_op_impl/tbe/scatter_nd_add.py +3 -0
  410. mindspore/ops/_op_impl/tbe/scatter_nd_d.py +1 -1
  411. mindspore/ops/_op_impl/tbe/space_to_batch.py +1 -1
  412. mindspore/ops/_op_impl/tbe/space_to_batch_nd.py +2 -2
  413. mindspore/ops/_op_impl/tbe/trans_data_ds.py +2 -0
  414. mindspore/ops/_primitive_cache.py +1 -1
  415. mindspore/ops/_tracefunc.py +241 -0
  416. mindspore/ops/_utils/utils.py +10 -2
  417. mindspore/ops/_vmap/vmap_array_ops.py +5 -3
  418. mindspore/ops/_vmap/vmap_base.py +5 -4
  419. mindspore/ops/_vmap/vmap_convolution_ops.py +1 -1
  420. mindspore/ops/_vmap/vmap_grad_math_ops.py +6 -4
  421. mindspore/ops/_vmap/vmap_grad_nn_ops.py +11 -6
  422. mindspore/ops/_vmap/vmap_math_ops.py +5 -2
  423. mindspore/ops/_vmap/vmap_nn_ops.py +135 -11
  424. mindspore/ops/arg_dtype_cast.py +54 -0
  425. mindspore/ops/composite/__init__.py +7 -5
  426. mindspore/ops/composite/base.py +78 -34
  427. mindspore/ops/composite/math_ops.py +5 -695
  428. mindspore/ops/composite/multitype_ops/_compile_utils.py +403 -97
  429. mindspore/ops/composite/multitype_ops/_constexpr_utils.py +28 -22
  430. mindspore/ops/composite/multitype_ops/add_impl.py +69 -7
  431. mindspore/ops/composite/multitype_ops/bitwise_and_impl.py +2 -1
  432. mindspore/ops/composite/multitype_ops/bitwise_or_impl.py +2 -1
  433. mindspore/ops/composite/multitype_ops/bitwise_xor_impl.py +2 -0
  434. mindspore/ops/composite/multitype_ops/div_impl.py +1 -0
  435. mindspore/ops/composite/multitype_ops/floordiv_impl.py +1 -0
  436. mindspore/ops/composite/multitype_ops/getitem_impl.py +48 -10
  437. mindspore/ops/composite/multitype_ops/greater_equal_impl.py +2 -0
  438. mindspore/ops/composite/multitype_ops/greater_impl.py +2 -0
  439. mindspore/ops/composite/multitype_ops/left_shift_impl.py +2 -0
  440. mindspore/ops/composite/multitype_ops/less_equal_impl.py +2 -0
  441. mindspore/ops/composite/multitype_ops/less_impl.py +2 -0
  442. mindspore/ops/composite/multitype_ops/logic_not_impl.py +2 -2
  443. mindspore/ops/composite/multitype_ops/mod_impl.py +1 -0
  444. mindspore/ops/composite/multitype_ops/mul_impl.py +1 -0
  445. mindspore/ops/composite/multitype_ops/negative_impl.py +1 -0
  446. mindspore/ops/composite/multitype_ops/not_in_impl.py +1 -0
  447. mindspore/ops/composite/multitype_ops/ones_like_impl.py +6 -0
  448. mindspore/ops/composite/multitype_ops/pow_impl.py +1 -0
  449. mindspore/ops/composite/multitype_ops/right_shift_impl.py +2 -0
  450. mindspore/ops/composite/multitype_ops/setitem_impl.py +10 -7
  451. mindspore/ops/composite/multitype_ops/sub_impl.py +1 -0
  452. mindspore/ops/composite/multitype_ops/uadd_impl.py +2 -0
  453. mindspore/ops/composite/multitype_ops/zeros_like_impl.py +9 -0
  454. mindspore/ops/deprecated.py +304 -0
  455. mindspore/ops/function/__init__.py +41 -4
  456. mindspore/ops/function/array_func.py +1108 -467
  457. mindspore/ops/function/clip_func.py +94 -27
  458. mindspore/ops/function/debug_func.py +3 -1
  459. mindspore/ops/function/grad/grad_func.py +82 -73
  460. mindspore/ops/function/image_func.py +28 -12
  461. mindspore/ops/function/linalg_func.py +135 -39
  462. mindspore/ops/function/math_func.py +3779 -894
  463. mindspore/ops/function/nn_func.py +1584 -657
  464. mindspore/ops/function/parameter_func.py +13 -3
  465. mindspore/ops/function/random_func.py +247 -153
  466. mindspore/ops/function/sparse_func.py +14 -11
  467. mindspore/ops/function/sparse_unary_func.py +173 -47
  468. mindspore/ops/function/spectral_func.py +8 -4
  469. mindspore/ops/function/vmap_func.py +8 -7
  470. mindspore/ops/functional.py +47 -16
  471. mindspore/ops/op_info_register.py +346 -86
  472. mindspore/ops/operations/__init__.py +38 -22
  473. mindspore/ops/operations/_grad_ops.py +145 -149
  474. mindspore/ops/operations/_inner_ops.py +298 -56
  475. mindspore/ops/operations/_ms_kernel.py +3 -3
  476. mindspore/ops/operations/_quant_ops.py +24 -28
  477. mindspore/ops/operations/_rl_inner_ops.py +9 -7
  478. mindspore/ops/operations/_scalar_ops.py +115 -0
  479. mindspore/ops/operations/_sequence_ops.py +148 -10
  480. mindspore/ops/operations/_tensor_array.py +1 -1
  481. mindspore/ops/operations/_thor_ops.py +2 -2
  482. mindspore/ops/operations/array_ops.py +1239 -561
  483. mindspore/ops/operations/comm_ops.py +166 -90
  484. mindspore/ops/operations/control_ops.py +3 -3
  485. mindspore/ops/operations/custom_ops.py +124 -102
  486. mindspore/ops/operations/debug_ops.py +24 -11
  487. mindspore/ops/operations/image_ops.py +86 -71
  488. mindspore/ops/operations/inner_ops.py +18 -13
  489. mindspore/ops/operations/linalg_ops.py +30 -11
  490. mindspore/ops/operations/math_ops.py +1730 -435
  491. mindspore/ops/operations/nn_ops.py +1953 -943
  492. mindspore/ops/operations/other_ops.py +65 -43
  493. mindspore/ops/operations/random_ops.py +258 -98
  494. mindspore/ops/operations/rl_ops.py +4 -36
  495. mindspore/ops/operations/sparse_ops.py +38 -33
  496. mindspore/ops/operations/spectral_ops.py +8 -4
  497. mindspore/ops/primitive.py +66 -44
  498. mindspore/ops/signature.py +5 -5
  499. mindspore/parallel/_auto_parallel_context.py +80 -19
  500. mindspore/parallel/_cost_model_context.py +42 -0
  501. mindspore/parallel/_offload_context.py +162 -72
  502. mindspore/parallel/_parallel_serialization.py +2 -2
  503. mindspore/parallel/_ps_context.py +16 -4
  504. mindspore/parallel/_recovery_context.py +2 -1
  505. mindspore/parallel/_tensor.py +15 -13
  506. mindspore/parallel/_transformer/layers.py +8 -6
  507. mindspore/parallel/_transformer/loss.py +1 -0
  508. mindspore/parallel/_transformer/moe.py +7 -7
  509. mindspore/parallel/_transformer/op_parallel_config.py +12 -1
  510. mindspore/parallel/_transformer/transformer.py +34 -14
  511. mindspore/parallel/_utils.py +36 -14
  512. mindspore/parallel/algo_parameter_config.py +114 -20
  513. mindspore/parallel/checkpoint_transform.py +16 -18
  514. mindspore/parallel/shard.py +16 -13
  515. mindspore/profiler/__init__.py +1 -1
  516. mindspore/profiler/common/struct_type.py +3 -3
  517. mindspore/profiler/common/util.py +3 -2
  518. mindspore/profiler/envprofiling.py +11 -4
  519. mindspore/profiler/parser/aicpu_data_parser.py +5 -3
  520. mindspore/profiler/parser/ascend_flops_generator.py +94 -0
  521. mindspore/profiler/parser/ascend_fpbp_generator.py +76 -0
  522. mindspore/profiler/parser/ascend_hccl_generator.py +288 -0
  523. mindspore/profiler/parser/ascend_msprof_exporter.py +213 -0
  524. mindspore/profiler/parser/ascend_msprof_generator.py +199 -0
  525. mindspore/profiler/parser/ascend_op_generator.py +276 -0
  526. mindspore/profiler/parser/ascend_steptrace_generator.py +94 -0
  527. mindspore/profiler/parser/ascend_timeline_generator.py +110 -54
  528. mindspore/profiler/parser/base_timeline_generator.py +11 -7
  529. mindspore/profiler/parser/cpu_gpu_timeline_generator.py +45 -46
  530. mindspore/profiler/parser/flops_parser.py +15 -11
  531. mindspore/profiler/parser/framework_parser.py +92 -73
  532. mindspore/profiler/parser/hccl_parser.py +16 -12
  533. mindspore/profiler/parser/integrator.py +22 -11
  534. mindspore/profiler/parser/memory_usage_parser.py +36 -11
  535. mindspore/profiler/parser/minddata_analyzer.py +12 -14
  536. mindspore/profiler/parser/minddata_pipeline_parser.py +1 -1
  537. mindspore/profiler/parser/msadvisor_parser.py +8 -4
  538. mindspore/profiler/parser/op_intermediate_parser.py +5 -2
  539. mindspore/profiler/parser/optime_parser.py +1 -1
  540. mindspore/profiler/parser/profiler_info.py +4 -5
  541. mindspore/profiler/parser/step_trace_parser.py +11 -14
  542. mindspore/profiler/profiling.py +678 -377
  543. mindspore/rewrite/api/node.py +211 -54
  544. mindspore/rewrite/api/node_type.py +5 -0
  545. mindspore/rewrite/api/pattern_engine.py +22 -23
  546. mindspore/rewrite/api/scoped_value.py +20 -17
  547. mindspore/rewrite/api/symbol_tree.py +252 -106
  548. mindspore/rewrite/api/tree_node_helper.py +3 -0
  549. mindspore/rewrite/ast_helpers/__init__.py +2 -1
  550. mindspore/rewrite/ast_helpers/ast_finder.py +129 -0
  551. mindspore/rewrite/ast_helpers/ast_modifier.py +116 -104
  552. mindspore/rewrite/ast_transformers/flatten_recursive_stmt.py +97 -46
  553. mindspore/rewrite/common/rewrite_elog.py +5 -1
  554. mindspore/rewrite/namer.py +51 -51
  555. mindspore/rewrite/namespace.py +14 -5
  556. mindspore/{ops/bprop_mindir → rewrite/node}/__init__.py +9 -4
  557. mindspore/rewrite/node/call_function.py +79 -0
  558. mindspore/rewrite/node/cell_container.py +135 -0
  559. mindspore/rewrite/node/control_flow.py +88 -0
  560. mindspore/rewrite/{node.py → node/node.py} +313 -247
  561. mindspore/rewrite/node/node_manager.py +254 -0
  562. mindspore/rewrite/node/node_topological_manager.py +243 -0
  563. mindspore/rewrite/parsers/arguments_parser.py +22 -21
  564. mindspore/rewrite/parsers/assign_parser.py +225 -239
  565. mindspore/rewrite/parsers/attribute_parser.py +9 -7
  566. mindspore/rewrite/parsers/class_def_parser.py +179 -218
  567. mindspore/rewrite/parsers/constant_parser.py +9 -6
  568. mindspore/rewrite/parsers/container_parser.py +9 -7
  569. mindspore/rewrite/parsers/for_parser.py +36 -15
  570. mindspore/rewrite/parsers/function_def_parser.py +23 -20
  571. mindspore/rewrite/parsers/if_parser.py +28 -24
  572. mindspore/rewrite/parsers/module_parser.py +202 -25
  573. mindspore/rewrite/{parser.py → parsers/parser.py} +4 -2
  574. mindspore/rewrite/{parser_register.py → parsers/parser_register.py} +1 -1
  575. mindspore/rewrite/parsers/return_parser.py +6 -6
  576. mindspore/rewrite/sparsify/sparse_transformer.py +12 -3
  577. mindspore/rewrite/sparsify/sparsify.py +4 -1
  578. mindspore/rewrite/sparsify/utils.py +11 -5
  579. mindspore/rewrite/symbol_tree.py +577 -732
  580. mindspore/rewrite/symbol_tree_builder.py +9 -175
  581. mindspore/rewrite/symbol_tree_dumper.py +2 -2
  582. mindspore/run_check/_check_version.py +46 -39
  583. mindspore/run_check/run_check.py +3 -2
  584. mindspore/{scipy/sparse → safeguard}/__init__.py +4 -5
  585. mindspore/safeguard/rewrite_obfuscation.py +517 -0
  586. mindspore/scipy/__init__.py +1 -1
  587. mindspore/scipy/linalg.py +67 -61
  588. mindspore/scipy/ops.py +5 -41
  589. mindspore/scipy/ops_grad.py +3 -2
  590. mindspore/scipy/ops_wrapper.py +5 -5
  591. mindspore/scipy/optimize/line_search.py +8 -8
  592. mindspore/scipy/optimize/linear_sum_assignment.py +4 -4
  593. mindspore/scipy/optimize/minimize.py +16 -12
  594. mindspore/scipy/utils.py +1 -52
  595. mindspore/scipy/utils_const.py +4 -4
  596. mindspore/train/__init__.py +4 -4
  597. mindspore/train/_utils.py +13 -5
  598. mindspore/train/amp.py +410 -148
  599. mindspore/train/anf_ir_pb2.py +16 -4
  600. mindspore/train/callback/_backup_and_restore.py +8 -11
  601. mindspore/train/callback/_callback.py +80 -3
  602. mindspore/train/callback/_checkpoint.py +82 -51
  603. mindspore/train/callback/_early_stop.py +12 -15
  604. mindspore/train/callback/_history.py +1 -1
  605. mindspore/train/callback/_lambda_callback.py +13 -13
  606. mindspore/train/callback/_landscape.py +21 -17
  607. mindspore/train/callback/_loss_monitor.py +9 -10
  608. mindspore/train/callback/_on_request_exit.py +16 -33
  609. mindspore/train/callback/_reduce_lr_on_plateau.py +21 -24
  610. mindspore/train/callback/_summary_collector.py +44 -30
  611. mindspore/train/callback/_time_monitor.py +62 -12
  612. mindspore/train/data_sink.py +10 -16
  613. mindspore/train/dataset_helper.py +154 -86
  614. mindspore/train/loss_scale_manager.py +14 -9
  615. mindspore/train/metrics/__init__.py +10 -2
  616. mindspore/train/metrics/accuracy.py +1 -1
  617. mindspore/train/metrics/auc.py +1 -1
  618. mindspore/train/metrics/bleu_score.py +2 -2
  619. mindspore/train/metrics/confusion_matrix.py +14 -14
  620. mindspore/train/metrics/cosine_similarity.py +3 -3
  621. mindspore/train/metrics/dice.py +1 -1
  622. mindspore/train/metrics/fbeta.py +1 -1
  623. mindspore/train/metrics/hausdorff_distance.py +8 -6
  624. mindspore/train/metrics/mean_surface_distance.py +5 -4
  625. mindspore/train/metrics/metric.py +49 -17
  626. mindspore/train/metrics/occlusion_sensitivity.py +4 -4
  627. mindspore/train/metrics/perplexity.py +1 -1
  628. mindspore/train/metrics/precision.py +2 -2
  629. mindspore/train/metrics/recall.py +2 -3
  630. mindspore/train/metrics/roc.py +7 -7
  631. mindspore/train/metrics/root_mean_square_surface_distance.py +5 -4
  632. mindspore/train/metrics/topk.py +7 -4
  633. mindspore/train/mind_ir_pb2.py +193 -48
  634. mindspore/train/model.py +377 -133
  635. mindspore/train/serialization.py +697 -245
  636. mindspore/train/summary/_summary_adapter.py +5 -2
  637. mindspore/train/summary/_writer_pool.py +4 -3
  638. mindspore/train/summary/summary_record.py +25 -23
  639. mindspore/train/train_thor/convert_utils.py +39 -23
  640. mindspore/train/train_thor/dataset_helper.py +4 -3
  641. mindspore/train/train_thor/model_thor.py +8 -8
  642. mindspore/version.py +1 -1
  643. {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/METADATA +7 -8
  644. {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/RECORD +647 -818
  645. {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/entry_points.txt +0 -1
  646. mindspore/_akg/akg/tvm/contrib/debugger/__init__.py +0 -16
  647. mindspore/_akg/akg/tvm/contrib/debugger/debug_result.py +0 -274
  648. mindspore/_akg/akg/tvm/contrib/debugger/debug_runtime.py +0 -259
  649. mindspore/_akg/akg/tvm/contrib/peak.py +0 -341
  650. mindspore/_akg/akg/tvm/contrib/rpc.py +0 -25
  651. mindspore/_akg/akg/tvm/contrib/xcode.py +0 -257
  652. mindspore/_akg/akg/tvm/exec/__init__.py +0 -17
  653. mindspore/_akg/akg/tvm/exec/autotvm_log_editor.py +0 -60
  654. mindspore/_akg/akg/tvm/exec/measure_peak.py +0 -48
  655. mindspore/_akg/akg/tvm/exec/query_rpc_tracker.py +0 -48
  656. mindspore/_akg/akg/tvm/exec/rpc_proxy.py +0 -98
  657. mindspore/_akg/akg/tvm/exec/rpc_server.py +0 -88
  658. mindspore/_akg/akg/tvm/exec/rpc_tracker.py +0 -62
  659. mindspore/_akg/akg/tvm/rpc/__init__.py +0 -29
  660. mindspore/_akg/akg/tvm/rpc/base.py +0 -182
  661. mindspore/_akg/akg/tvm/rpc/client.py +0 -436
  662. mindspore/_akg/akg/tvm/rpc/proxy.py +0 -595
  663. mindspore/_akg/akg/tvm/rpc/server.py +0 -413
  664. mindspore/_akg/akg/tvm/rpc/tornado_util.py +0 -121
  665. mindspore/_akg/akg/tvm/rpc/tracker.py +0 -431
  666. mindspore/_extends/graph_kernel/expander.py +0 -80
  667. mindspore/_extends/graph_kernel/expanders/__init__.py +0 -57
  668. mindspore/_extends/graph_kernel/expanders/_utils.py +0 -269
  669. mindspore/_extends/graph_kernel/expanders/addn.py +0 -33
  670. mindspore/_extends/graph_kernel/expanders/batchnorm.py +0 -152
  671. mindspore/_extends/graph_kernel/expanders/batchnorm_grad.py +0 -105
  672. mindspore/_extends/graph_kernel/expanders/bias_add_grad.py +0 -49
  673. mindspore/_extends/graph_kernel/expanders/clip_by_norm_no_div_sum.py +0 -33
  674. mindspore/_extends/graph_kernel/expanders/complex/abs.py +0 -30
  675. mindspore/_extends/graph_kernel/expanders/complex/add.py +0 -44
  676. mindspore/_extends/graph_kernel/expanders/complex/div.py +0 -62
  677. mindspore/_extends/graph_kernel/expanders/complex/mul.py +0 -52
  678. mindspore/_extends/graph_kernel/expanders/complex/real_div.py +0 -62
  679. mindspore/_extends/graph_kernel/expanders/complex/sub.py +0 -45
  680. mindspore/_extends/graph_kernel/expanders/conv2d.py +0 -200
  681. mindspore/_extends/graph_kernel/expanders/dropout_grad.py +0 -30
  682. mindspore/_extends/graph_kernel/expanders/equal_count.py +0 -50
  683. mindspore/_extends/graph_kernel/expanders/erfc.py +0 -35
  684. mindspore/_extends/graph_kernel/expanders/expand_dims.py +0 -50
  685. mindspore/_extends/graph_kernel/expanders/fused_adam.py +0 -44
  686. mindspore/_extends/graph_kernel/expanders/fused_adam_weight_decay.py +0 -47
  687. mindspore/_extends/graph_kernel/expanders/fused_mul_add.py +0 -28
  688. mindspore/_extends/graph_kernel/expanders/gather.py +0 -43
  689. mindspore/_extends/graph_kernel/expanders/gelu_grad.py +0 -70
  690. mindspore/_extends/graph_kernel/expanders/gkdropout.py +0 -40
  691. mindspore/_extends/graph_kernel/expanders/identity.py +0 -25
  692. mindspore/_extends/graph_kernel/expanders/layernorm.py +0 -93
  693. mindspore/_extends/graph_kernel/expanders/layernorm_grad.py +0 -113
  694. mindspore/_extends/graph_kernel/expanders/logsoftmax.py +0 -46
  695. mindspore/_extends/graph_kernel/expanders/logsoftmax_grad.py +0 -36
  696. mindspore/_extends/graph_kernel/expanders/matmul.py +0 -80
  697. mindspore/_extends/graph_kernel/expanders/maximum_grad.py +0 -59
  698. mindspore/_extends/graph_kernel/expanders/minimum_grad.py +0 -80
  699. mindspore/_extends/graph_kernel/expanders/oneslike.py +0 -26
  700. mindspore/_extends/graph_kernel/expanders/reduce_mean.py +0 -43
  701. mindspore/_extends/graph_kernel/expanders/relu_grad.py +0 -32
  702. mindspore/_extends/graph_kernel/expanders/sigmoid_cross_entropy_with_logits.py +0 -41
  703. mindspore/_extends/graph_kernel/expanders/sigmoid_cross_entropy_with_logits_grad.py +0 -35
  704. mindspore/_extends/graph_kernel/expanders/sigmoid_grad.py +0 -31
  705. mindspore/_extends/graph_kernel/expanders/slice.py +0 -35
  706. mindspore/_extends/graph_kernel/expanders/softmax_cross_entropy_with_logits.py +0 -42
  707. mindspore/_extends/graph_kernel/expanders/softmax_grad_ext.py +0 -41
  708. mindspore/_extends/graph_kernel/expanders/softsign.py +0 -28
  709. mindspore/_extends/graph_kernel/expanders/sqrt_grad.py +0 -29
  710. mindspore/_extends/graph_kernel/expanders/square_sum_all.py +0 -44
  711. mindspore/_extends/graph_kernel/expanders/square_sum_v1.py +0 -37
  712. mindspore/_extends/graph_kernel/expanders/squared_difference.py +0 -43
  713. mindspore/_extends/graph_kernel/expanders/tanh_grad.py +0 -31
  714. mindspore/_extends/graph_kernel/expanders/tile.py +0 -54
  715. mindspore/_extends/graph_kernel/model/op_infer.py +0 -506
  716. mindspore/_extends/parse/jit_fallback_modules.py +0 -51
  717. mindspore/dataset/datapreprocess/preprocess_imagenet_validate_dataset.py +0 -54
  718. mindspore/dataset/engine/graphdata.py +0 -1586
  719. mindspore/include/api/net.h +0 -142
  720. mindspore/ops/_grad/grad_array_ops.py +0 -1347
  721. mindspore/ops/_grad/grad_clip_ops.py +0 -84
  722. mindspore/ops/_grad/grad_debug_ops.py +0 -68
  723. mindspore/ops/_grad/grad_inner_ops.py +0 -235
  724. mindspore/ops/_grad/grad_math_ops.py +0 -1684
  725. mindspore/ops/_grad/grad_nn_ops.py +0 -1529
  726. mindspore/ops/_grad/grad_other_ops.py +0 -89
  727. mindspore/ops/_grad/grad_sequence_ops.py +0 -296
  728. mindspore/ops/_grad/grad_sparse.py +0 -323
  729. mindspore/ops/_grad_experimental/grad_image_ops.py +0 -249
  730. mindspore/ops/_grad_experimental/grad_linalg_ops.py +0 -195
  731. mindspore/ops/_grad_experimental/grad_scalar_ops.py +0 -112
  732. mindspore/ops/bprop_mindir/AdaptiveAvgPool2D_bprop.mindir +0 -0
  733. mindspore/ops/bprop_mindir/AdaptiveMaxPool2D_bprop.mindir +0 -0
  734. mindspore/ops/bprop_mindir/ApproximateEqual_bprop.mindir +0 -19
  735. mindspore/ops/bprop_mindir/Argmax_bprop.mindir +0 -15
  736. mindspore/ops/bprop_mindir/Argmin_bprop.mindir +0 -15
  737. mindspore/ops/bprop_mindir/AssignSub_bprop.mindir +0 -19
  738. mindspore/ops/bprop_mindir/Assign_bprop.mindir +0 -17
  739. mindspore/ops/bprop_mindir/AvgPool3D_bprop.mindir +0 -150
  740. mindspore/ops/bprop_mindir/AvgPool_bprop.mindir +0 -66
  741. mindspore/ops/bprop_mindir/BCEWithLogitsLoss_bprop.mindir +0 -0
  742. mindspore/ops/bprop_mindir/BNTrainingReduce_bprop.mindir +0 -15
  743. mindspore/ops/bprop_mindir/BatchNormGrad_bprop.mindir +0 -0
  744. mindspore/ops/bprop_mindir/BatchToSpaceND_bprop.mindir +0 -28
  745. mindspore/ops/bprop_mindir/BiasAddGrad_bprop.mindir +0 -0
  746. mindspore/ops/bprop_mindir/BinaryCrossEntropy_bprop.mindir +0 -33
  747. mindspore/ops/bprop_mindir/BroadcastTo_bprop.mindir +0 -306
  748. mindspore/ops/bprop_mindir/Broadcast_bprop.mindir +0 -13
  749. mindspore/ops/bprop_mindir/CTCLoss_bprop.mindir +0 -0
  750. mindspore/ops/bprop_mindir/Concat_bprop.mindir +0 -0
  751. mindspore/ops/bprop_mindir/Conv2DBackpropFilter_bprop.mindir +0 -240
  752. mindspore/ops/bprop_mindir/Conv2DBackpropInput_bprop.mindir +0 -247
  753. mindspore/ops/bprop_mindir/Conv2DTranspose_bprop.mindir +0 -247
  754. mindspore/ops/bprop_mindir/Conv3DTranspose_bprop.mindir +0 -315
  755. mindspore/ops/bprop_mindir/Conv3D_bprop.mindir +0 -278
  756. mindspore/ops/bprop_mindir/DType_bprop.mindir +0 -14
  757. mindspore/ops/bprop_mindir/DeformableOffsets_bprop.mindir +0 -58
  758. mindspore/ops/bprop_mindir/Depend_bprop.mindir +0 -13
  759. mindspore/ops/bprop_mindir/DepthToSpace_bprop.mindir +0 -23
  760. mindspore/ops/bprop_mindir/DepthwiseConv2dNative_bprop.mindir +0 -138
  761. mindspore/ops/bprop_mindir/DiagPart_bprop.mindir +0 -15
  762. mindspore/ops/bprop_mindir/Dropout2D_bprop.mindir +0 -0
  763. mindspore/ops/bprop_mindir/Dropout3D_bprop.mindir +0 -0
  764. mindspore/ops/bprop_mindir/DropoutDoMask_bprop.mindir +0 -25
  765. mindspore/ops/bprop_mindir/DropoutGenMask_bprop.mindir +0 -18
  766. mindspore/ops/bprop_mindir/DropoutGrad_bprop.mindir +0 -27
  767. mindspore/ops/bprop_mindir/Dropout_bprop.mindir +0 -0
  768. mindspore/ops/bprop_mindir/DynamicGRUV2_bprop.mindir +0 -0
  769. mindspore/ops/bprop_mindir/DynamicRNN_bprop.mindir +0 -0
  770. mindspore/ops/bprop_mindir/DynamicShape_bprop.mindir +0 -14
  771. mindspore/ops/bprop_mindir/Elu_bprop.mindir +0 -16
  772. mindspore/ops/bprop_mindir/EmbeddingLookup_bprop.mindir +0 -0
  773. mindspore/ops/bprop_mindir/Equal_bprop.mindir +0 -19
  774. mindspore/ops/bprop_mindir/ExpandDims_bprop.mindir +0 -58
  775. mindspore/ops/bprop_mindir/FastGeLU_bprop.mindir +0 -16
  776. mindspore/ops/bprop_mindir/Flatten_bprop.mindir +0 -54
  777. mindspore/ops/bprop_mindir/FloorDiv_bprop.mindir +0 -19
  778. mindspore/ops/bprop_mindir/GatherD_bprop.mindir +0 -26
  779. mindspore/ops/bprop_mindir/GatherNd_bprop.mindir +0 -57
  780. mindspore/ops/bprop_mindir/Gather_bprop.mindir +0 -0
  781. mindspore/ops/bprop_mindir/GreaterEqual_bprop.mindir +0 -19
  782. mindspore/ops/bprop_mindir/Greater_bprop.mindir +0 -19
  783. mindspore/ops/bprop_mindir/HSigmoid_bprop.mindir +0 -16
  784. mindspore/ops/bprop_mindir/HSwish_bprop.mindir +0 -16
  785. mindspore/ops/bprop_mindir/IOU_bprop.mindir +0 -19
  786. mindspore/ops/bprop_mindir/InstanceNorm_bprop.mindir +0 -0
  787. mindspore/ops/bprop_mindir/IsFinite_bprop.mindir +0 -15
  788. mindspore/ops/bprop_mindir/IsInf_bprop.mindir +0 -15
  789. mindspore/ops/bprop_mindir/IsNan_bprop.mindir +0 -15
  790. mindspore/ops/bprop_mindir/KLDivLoss_bprop.mindir +0 -126
  791. mindspore/ops/bprop_mindir/L2Loss_bprop.mindir +0 -15
  792. mindspore/ops/bprop_mindir/L2Normalize_bprop.mindir +0 -30
  793. mindspore/ops/bprop_mindir/LRN_bprop.mindir +0 -43
  794. mindspore/ops/bprop_mindir/LayerNormGrad_bprop.mindir +0 -0
  795. mindspore/ops/bprop_mindir/LessEqual_bprop.mindir +0 -19
  796. mindspore/ops/bprop_mindir/Less_bprop.mindir +0 -19
  797. mindspore/ops/bprop_mindir/LinSpace_bprop.mindir +0 -23
  798. mindspore/ops/bprop_mindir/Load_bprop.mindir +0 -13
  799. mindspore/ops/bprop_mindir/LogSoftmax_bprop.mindir +0 -23
  800. mindspore/ops/bprop_mindir/LogicalAnd_bprop.mindir +0 -19
  801. mindspore/ops/bprop_mindir/LogicalNot_bprop.mindir +0 -15
  802. mindspore/ops/bprop_mindir/MaskedSelect_bprop.mindir +0 -21
  803. mindspore/ops/bprop_mindir/MaxPool3DGradGrad_bprop.mindir +0 -74
  804. mindspore/ops/bprop_mindir/MaxPool3DGrad_bprop.mindir +0 -74
  805. mindspore/ops/bprop_mindir/MaxPool3D_bprop.mindir +0 -75
  806. mindspore/ops/bprop_mindir/MaxPoolGradGrad_bprop.mindir +0 -65
  807. mindspore/ops/bprop_mindir/MaxPoolWithArgmax_bprop.mindir +0 -0
  808. mindspore/ops/bprop_mindir/Maximum_bprop.mindir +0 -0
  809. mindspore/ops/bprop_mindir/Minimum_bprop.mindir +0 -0
  810. mindspore/ops/bprop_mindir/MirrorPad_bprop.mindir +0 -27
  811. mindspore/ops/bprop_mindir/Mish_bprop.mindir +0 -35
  812. mindspore/ops/bprop_mindir/MulNoNan_bprop.mindir +0 -0
  813. mindspore/ops/bprop_mindir/NLLLoss_bprop.mindir +0 -0
  814. mindspore/ops/bprop_mindir/NonZero_bprop.mindir +0 -14
  815. mindspore/ops/bprop_mindir/NotEqual_bprop.mindir +0 -19
  816. mindspore/ops/bprop_mindir/OneHot_bprop.mindir +0 -26
  817. mindspore/ops/bprop_mindir/OnesLike_bprop.mindir +0 -14
  818. mindspore/ops/bprop_mindir/PReLU_bprop.mindir +0 -0
  819. mindspore/ops/bprop_mindir/Pad_bprop.mindir +0 -0
  820. mindspore/ops/bprop_mindir/Padding_bprop.mindir +0 -0
  821. mindspore/ops/bprop_mindir/RNNTLoss_bprop.mindir +0 -29
  822. mindspore/ops/bprop_mindir/ROIAlign_bprop.mindir +0 -82
  823. mindspore/ops/bprop_mindir/Range_bprop.mindir +0 -22
  824. mindspore/ops/bprop_mindir/Rank_bprop.mindir +0 -14
  825. mindspore/ops/bprop_mindir/ReLU6_bprop.mindir +0 -16
  826. mindspore/ops/bprop_mindir/ReLUV2_bprop.mindir +0 -0
  827. mindspore/ops/bprop_mindir/ReduceAll_bprop.mindir +0 -19
  828. mindspore/ops/bprop_mindir/ReduceAny_bprop.mindir +0 -19
  829. mindspore/ops/bprop_mindir/ReluGrad_bprop.mindir +0 -20
  830. mindspore/ops/bprop_mindir/Reshape_bprop.mindir +0 -60
  831. mindspore/ops/bprop_mindir/ResizeBilinear_bprop.mindir +0 -29
  832. mindspore/ops/bprop_mindir/ResizeNearestNeighbor_bprop.mindir +0 -89
  833. mindspore/ops/bprop_mindir/ReverseSequence_bprop.mindir +0 -52
  834. mindspore/ops/bprop_mindir/ReverseV2_bprop.mindir +0 -22
  835. mindspore/ops/bprop_mindir/Round_bprop.mindir +0 -15
  836. mindspore/ops/bprop_mindir/ScatterMax_bprop.mindir +0 -0
  837. mindspore/ops/bprop_mindir/ScatterMin_bprop.mindir +0 -0
  838. mindspore/ops/bprop_mindir/ScatterNdUpdate_bprop.mindir +0 -22
  839. mindspore/ops/bprop_mindir/ScatterNd_bprop.mindir +0 -24
  840. mindspore/ops/bprop_mindir/ScatterNonAliasingAdd_bprop.mindir +0 -22
  841. mindspore/ops/bprop_mindir/ScatterUpdate_bprop.mindir +0 -0
  842. mindspore/ops/bprop_mindir/SeLU_bprop.mindir +0 -21
  843. mindspore/ops/bprop_mindir/Select_bprop.mindir +0 -31
  844. mindspore/ops/bprop_mindir/Shape_bprop.mindir +0 -14
  845. mindspore/ops/bprop_mindir/SigmoidCrossEntropyWithLogits_bprop.mindir +0 -21
  846. mindspore/ops/bprop_mindir/SigmoidGrad_bprop.mindir +0 -0
  847. mindspore/ops/bprop_mindir/Sigmoid_bprop.mindir +0 -16
  848. mindspore/ops/bprop_mindir/Sign_bprop.mindir +0 -15
  849. mindspore/ops/bprop_mindir/Slice_bprop.mindir +0 -26
  850. mindspore/ops/bprop_mindir/SmoothL1Loss_bprop.mindir +0 -36
  851. mindspore/ops/bprop_mindir/SoftmaxCrossEntropyWithLogits_bprop.mindir +0 -0
  852. mindspore/ops/bprop_mindir/Softplus_bprop.mindir +0 -16
  853. mindspore/ops/bprop_mindir/Softsign_bprop.mindir +0 -33
  854. mindspore/ops/bprop_mindir/Sort_bprop.mindir +0 -0
  855. mindspore/ops/bprop_mindir/SpaceToBatchND_bprop.mindir +0 -28
  856. mindspore/ops/bprop_mindir/SpaceToDepth_bprop.mindir +0 -23
  857. mindspore/ops/bprop_mindir/SparseGatherV2_bprop.mindir +0 -0
  858. mindspore/ops/bprop_mindir/SparseSoftmaxCrossEntropyWithLogits_bprop.mindir +0 -0
  859. mindspore/ops/bprop_mindir/Split_bprop.mindir +0 -22
  860. mindspore/ops/bprop_mindir/Squeeze_bprop.mindir +0 -54
  861. mindspore/ops/bprop_mindir/StridedSliceGrad_bprop.mindir +0 -95
  862. mindspore/ops/bprop_mindir/StridedSlice_bprop.mindir +0 -98
  863. mindspore/ops/bprop_mindir/Switch_bprop.mindir +0 -29
  864. mindspore/ops/bprop_mindir/TanhGrad_bprop.mindir +0 -0
  865. mindspore/ops/bprop_mindir/Tanh_bprop.mindir +0 -66
  866. mindspore/ops/bprop_mindir/TensorScatterAdd_bprop.mindir +0 -22
  867. mindspore/ops/bprop_mindir/TensorScatterUpdate_bprop.mindir +0 -29
  868. mindspore/ops/bprop_mindir/TensorShape_bprop.mindir +0 -14
  869. mindspore/ops/bprop_mindir/Tile_bprop.mindir +0 -0
  870. mindspore/ops/bprop_mindir/TopK_bprop.mindir +0 -0
  871. mindspore/ops/bprop_mindir/TransShape_bprop.mindir +0 -23
  872. mindspore/ops/bprop_mindir/TruncateDiv_bprop.mindir +0 -19
  873. mindspore/ops/bprop_mindir/TupleGetItem_bprop.mindir +0 -20
  874. mindspore/ops/bprop_mindir/Unique_bprop.mindir +0 -16
  875. mindspore/ops/bprop_mindir/Unstack_bprop.mindir +0 -22
  876. mindspore/ops/bprop_mindir/UpsampleNearest3D_bprop.mindir +0 -32
  877. mindspore/ops/bprop_mindir/UpsampleTrilinear3D_bprop.mindir +0 -38
  878. mindspore/ops/bprop_mindir/ZerosLike_bprop.mindir +0 -15
  879. mindspore/ops/bprop_mindir/generate_mindir.py +0 -114
  880. mindspore/rewrite/node_visitor.py +0 -44
  881. mindspore/rewrite/topological_manager.py +0 -203
  882. mindspore/scipy/sparse/linalg.py +0 -192
  883. {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/WHEEL +0 -0
  884. {mindspore-2.0.0rc1.dist-info → mindspore-2.2.0.dist-info}/top_level.txt +0 -0
@@ -104,23 +104,26 @@ def _init_device_info():
104
104
 
105
105
  def set_seed(seed):
106
106
  """
107
- Set the seed so the random generated number will be fixed for deterministic results.
107
+ Set the seed for the random number generator in data pipeline.
108
+
109
+ The seed can be set to control the initial state of the random generator
110
+ for the purpose of fixing the result of random number generation.
108
111
 
109
112
  Note:
110
- This set_seed function sets the seed in the Python random library and numpy.random library
111
- for deterministic Python augmentations using randomness. This set_seed function should
112
- be called when iterator is created to reset the random seed.
113
+ This interface will set the random seed of the `random`, `numpy.random`
114
+ and `mindspore.dataset` modules to the specified value at the same time.
113
115
 
114
116
  Args:
115
- seed(int): Random number seed. It is used to generate deterministic random numbers.
117
+ seed (int): The desired seed. Must be non-negative.
116
118
 
117
119
  Raises:
118
- TypeError: If `seed` isn't of type int.
119
- ValueError: If `seed` < 0 or `seed` > UINT32_MAX(4294967295).
120
+ TypeError: If `seed` is not of type int.
121
+ ValueError: If `seed` is a negative value.
120
122
 
121
123
  Examples:
122
124
  >>> # Set a new global configuration value for the seed value.
123
125
  >>> # Operations with randomness will use the seed value to generate random values.
126
+ >>> import mindspore.dataset as ds
124
127
  >>> ds.config.set_seed(1000)
125
128
  """
126
129
  if not isinstance(seed, int) or isinstance(seed, bool):
@@ -146,6 +149,7 @@ def get_seed():
146
149
  Examples:
147
150
  >>> # Get the global configuration of seed.
148
151
  >>> # If set_seed() is never called before, the default value(std::mt19937::default_seed) will be returned.
152
+ >>> import mindspore.dataset as ds
149
153
  >>> seed = ds.config.get_seed()
150
154
  """
151
155
  return _config.get_seed()
@@ -153,23 +157,32 @@ def get_seed():
153
157
 
154
158
  def set_prefetch_size(size):
155
159
  """
156
- Set the queue capacity of the thread in pipeline.
160
+ Set the buffer queue size between dataset operations in the pipeline.
161
+
162
+ The presence of a buffer queue allows the current operation to start
163
+ processing subsequent data before the next operation fetches it, so the
164
+ operations can execute asynchronously and concurrently.
165
+
166
+ A larger buffer queue size reduces the overall processing latency when
167
+ neighboring operations have unbalanced throughput rates, but also consumes
168
+ more system memory.
157
169
 
158
170
  Args:
159
- size (int): The length of the cache queue. The `size` must be greater than 0, otherwise the queue capacity of
160
- the thread is invalid.
171
+ size (int): The size of the buffer queue, must be greater than 0.
161
172
 
162
173
  Raises:
163
174
  TypeError: If `size` is not of type int.
164
175
  ValueError: If `size` is not a positive number.
165
176
 
166
177
  Note:
167
- Since total memory used for prefetch can grow very large with high number of workers,
168
- when the number of workers is greater than 4, the per worker prefetch size will be reduced.
169
- The actual prefetch size at runtime per-worker will be prefetchsize * (4 / num_parallel_workers).
178
+ The total memory consumed by the buffer queue is proportional to the number
179
+ of worker threads. To avoid overuse of memory, when the number of worker
180
+ threads is greater than 4, the actual buffer queue size used will be adjusted
181
+ to the greater of (`size` * 4 / number of worker threads) and 1.
170
182
 
171
183
  Examples:
172
184
  >>> # Set a new global configuration value for the prefetch size.
185
+ >>> import mindspore.dataset as ds
173
186
  >>> ds.config.set_prefetch_size(1000)
174
187
  """
175
188
  if not isinstance(size, int) or isinstance(size, bool):
@@ -191,6 +204,7 @@ def get_prefetch_size():
191
204
  Examples:
192
205
  >>> # Get the global configuration of prefetch size.
193
206
  >>> # If set_prefetch_size() is never called before, the default value(16) will be returned.
207
+ >>> import mindspore.dataset as ds
194
208
  >>> prefetch_size = ds.config.get_prefetch_size()
195
209
  """
196
210
  return _config.get_op_connector_size()
@@ -211,6 +225,7 @@ def set_num_parallel_workers(num):
211
225
  Examples:
212
226
  >>> # Set a new global configuration value for the number of parallel workers.
213
227
  >>> # Now parallel dataset operations will run with 8 workers.
228
+ >>> import mindspore.dataset as ds
214
229
  >>> ds.config.set_num_parallel_workers(8)
215
230
  """
216
231
  if not isinstance(num, int) or isinstance(num, bool):
@@ -225,6 +240,7 @@ def get_num_parallel_workers():
225
240
  """
226
241
  Get the global configuration of number of parallel workers.
227
242
  This is the DEFAULT num_parallel_workers value used for each operation.
243
+ If `set_num_parallel_workers` is never called before, the default value(8) will be returned.
228
244
 
229
245
  Returns:
230
246
  int, number of parallel workers to be used as a default for each operation.
@@ -232,6 +248,7 @@ def get_num_parallel_workers():
232
248
  Examples:
233
249
  >>> # Get the global configuration of parallel workers.
234
250
  >>> # If set_num_parallel_workers() is never called before, the default value(8) will be returned.
251
+ >>> import mindspore.dataset as ds
235
252
  >>> num_parallel_workers = ds.config.get_num_parallel_workers()
236
253
  """
237
254
  return _config.get_num_parallel_workers()
@@ -239,7 +256,7 @@ def get_num_parallel_workers():
239
256
 
240
257
  def set_numa_enable(numa_enable):
241
258
  """
242
- Set the default state of numa enabled. If `numa_enable` is True, need to
259
+ Set the default state of numa enabled. If `numa_enable` is ``True``, need to
243
260
  ensure `numa library <http://rpmfind.net/linux/rpm2html/search.php?query=libnuma-devel>`_ is installed.
244
261
 
245
262
  Args:
@@ -251,6 +268,7 @@ def set_numa_enable(numa_enable):
251
268
  Examples:
252
269
  >>> # Set a new global configuration value for the state of numa enabled.
253
270
  >>> # Now parallel dataset operations will run with numa bind function
271
+ >>> import mindspore.dataset as ds
254
272
  >>> ds.config.set_numa_enable(True)
255
273
  """
256
274
  if not isinstance(numa_enable, bool):
@@ -261,13 +279,14 @@ def set_numa_enable(numa_enable):
261
279
  def get_numa_enable():
262
280
  """
263
281
  Get the state of numa to indicate enabled/disabled.
264
- This is the DEFAULT numa enabled value used for the all process.
282
+ This is the numa config used for all processes, it is disabled by default.
265
283
 
266
284
  Returns:
267
285
  bool, the default state of numa enabled.
268
286
 
269
287
  Examples:
270
288
  >>> # Get the global configuration of numa.
289
+ >>> import mindspore.dataset as ds
271
290
  >>> numa_state = ds.config.get_numa_enable()
272
291
  """
273
292
  return _config.get_numa_enable()
@@ -286,6 +305,7 @@ def set_monitor_sampling_interval(interval):
286
305
 
287
306
  Examples:
288
307
  >>> # Set a new global configuration value for the monitor sampling interval.
308
+ >>> import mindspore.dataset as ds
289
309
  >>> ds.config.set_monitor_sampling_interval(100)
290
310
  """
291
311
  if not isinstance(interval, int) or isinstance(interval, bool):
@@ -307,6 +327,7 @@ def get_monitor_sampling_interval():
307
327
  Examples:
308
328
  >>> # Get the global configuration of monitor sampling interval.
309
329
  >>> # If set_monitor_sampling_interval() is never called before, the default value(1000) will be returned.
330
+ >>> import mindspore.dataset as ds
310
331
  >>> sampling_interval = ds.config.get_monitor_sampling_interval()
311
332
  """
312
333
  return _config.get_monitor_sampling_interval()
@@ -318,7 +339,7 @@ def set_auto_num_workers(enable):
318
339
 
319
340
  If turned on, the num_parallel_workers in each op will be adjusted automatically, possibly overwriting the
320
341
  num_parallel_workers passed in by user or the default value (if user doesn't pass anything) set by
321
- ds.config.set_num_parallel_workers().
342
+ :func:`mindspore.dataset.config.set_num_parallel_workers`.
322
343
 
323
344
  For now, this function is only optimized for YoloV3 dataset with per_batch_map (running map in batch).
324
345
  This feature aims to provide a baseline for optimized num_workers assignment for each operation.
@@ -332,6 +353,7 @@ def set_auto_num_workers(enable):
332
353
 
333
354
  Examples:
334
355
  >>> # Enable auto_num_worker feature, this might override the num_parallel_workers passed in by user
356
+ >>> import mindspore.dataset as ds
335
357
  >>> ds.config.set_auto_num_workers(True)
336
358
  """
337
359
  if not isinstance(enable, bool):
@@ -367,13 +389,14 @@ def _set_auto_workers_config(option):
367
389
 
368
390
  def get_auto_num_workers():
369
391
  """
370
- Get the setting (turned on or off) automatic number of workers.
392
+ Get the setting (turned on or off) automatic number of workers, it is disabled by default.
371
393
 
372
394
  Returns:
373
395
  bool, whether auto number worker feature is turned on.
374
396
 
375
397
  Examples:
376
398
  >>> # Get the global configuration of auto number worker feature.
399
+ >>> import mindspore.dataset as ds
377
400
  >>> flag = ds.config.get_auto_num_workers()
378
401
  """
379
402
  return _config.get_auto_num_workers()
@@ -393,6 +416,7 @@ def set_callback_timeout(timeout):
393
416
 
394
417
  Examples:
395
418
  >>> # Set a new global configuration value for the timeout value.
419
+ >>> import mindspore.dataset as ds
396
420
  >>> ds.config.set_callback_timeout(100)
397
421
  """
398
422
  if not isinstance(timeout, int) or isinstance(timeout, bool):
@@ -404,7 +428,8 @@ def set_callback_timeout(timeout):
404
428
 
405
429
  def get_callback_timeout():
406
430
  """
407
- Get the default timeout for :class:`mindspore.dataset.WaitedDSCallback` .
431
+ Get the default timeout (in seconds) for :class:`mindspore.dataset.WaitedDSCallback` .
432
+ If `set_callback_timeout` is never called before, the default value(60) will be returned.
408
433
 
409
434
  Returns:
410
435
  int, Timeout (in seconds) to be used to end the wait in :class:`mindspore.dataset.WaitedDSCallback` in case of
@@ -413,6 +438,7 @@ def get_callback_timeout():
413
438
  Examples:
414
439
  >>> # Get the global configuration of callback timeout.
415
440
  >>> # If set_callback_timeout() is never called before, the default value(60) will be returned.
441
+ >>> import mindspore.dataset as ds
416
442
  >>> callback_timeout = ds.config.get_callback_timeout()
417
443
  """
418
444
  return _config.get_callback_timeout()
@@ -447,6 +473,7 @@ def load(file):
447
473
  >>> # "seed": 5489,
448
474
  >>> # "monitorSamplingInterval": 30
449
475
  >>> # }
476
+ >>> import mindspore.dataset as ds
450
477
  >>> config_file = "/path/to/config/file"
451
478
  >>> ds.config.load(config_file)
452
479
  """
@@ -455,43 +482,40 @@ def load(file):
455
482
 
456
483
  def set_enable_autotune(enable, filepath_prefix=None):
457
484
  """
458
- Set whether to enable AutoTune. AutoTune is disabled by default.
485
+ Set whether to enable AutoTune for data pipeline parameters.
459
486
 
460
- AutoTune is used to automatically adjust the global configuration of the data pipeline
461
- according to the workload of environmental resources during the training process to
462
- improve the speed of data processing.
487
+ It can be used to automatically adjust the parameter configurations
488
+ of each operation in the data processing pipeline, such as parallelism
489
+ and buffer queue size, according to the load of the environment resources
490
+ during training, so as to improve the overall processing speed.
463
491
 
464
- The optimized global configuration can be saved as a JSON file by setting `json_filepath`
465
- for subsequent reuse.
492
+ AutoTune is not enabled by default.
466
493
 
467
494
  Args:
468
495
  enable (bool): Whether to enable AutoTune.
469
- filepath_prefix (str, optional): The prefix filepath to save the optimized global configuration.
470
- The rank id and the json extension will be appended to the filepath_prefix string in multi-device training,
471
- rank id will be set to 0 in standalone training.
472
- For example, if filepath_prefix="/path/to/some/dir/prefixname" and rank_id is 1, then the path
473
- of the generated file will be "/path/to/some/dir/prefixname_1.json"
474
- If the file already exists, it will be automatically overwritten. Default: None,
475
- means not to save the configuration file, but the tuned result still can be checked through INFO log.
496
+ filepath_prefix (str, optional): The path where the optimized parameter configuration will be saved.
497
+ Effective only if `enable` is `True`. The parameter configuration file on each Device will be
498
+ saved separately, and the final file name will be `filepath_prefix + RANK_ID + ".json"`,
499
+ where RANK_ID is the Device ID corresponding to the file. Default: ``None`` , no configuration
500
+ file is saved.
476
501
 
477
502
  Raises:
478
503
  TypeError: If `enable` is not of type boolean.
479
- TypeError: If `json_filepath` is not of type str.
480
- RuntimeError: If `json_filepath` is an empty string.
481
- RuntimeError: If `json_filepath` is a directory.
482
- RuntimeError: If `json_filepath` does not exist.
483
- RuntimeError: If `json_filepath` does not have write permission.
504
+ TypeError: If `filepath_prefix` is not of type str.
505
+ RuntimeError: If `filepath_prefix` is an empty string.
506
+ RuntimeError: If `filepath_prefix` is a directory.
507
+ RuntimeError: If `filepath_prefix` does not exist.
508
+ RuntimeError: If `filepath_prefix` does not have write permission.
484
509
 
485
510
  Note:
486
- - When `enable` is False, `json_filepath` will be ignored.
487
- - The JSON file can be loaded by API `mindspore.dataset.deserialize` to build a tuned pipeline.
488
- - In distributed training scenario, set_enable_autotune() must be called after cluster communication has been
489
- initialized (mindspore.communication.management.init()), otherwise the AutoTune file will always suffix with
490
- rank id 0.
511
+ - Saved parameter profiles can be loaded via the `mindspore.dataset.deserialize` interface to
512
+ directly obtain a data processing pipeline object configured with optimal parameters.
513
+ - The parameter tuning process can be viewed by turning on INFO level logging.
491
514
 
492
- An example of the generated JSON file is as follows. "remark" file will conclude that if the dataset has been
493
- tuned or not. "summary" filed will show the tuned configuration of dataset pipeline. Users can modify scripts
494
- based on the tuned result.
515
+ An example of the generated configuration file is as follows, the "remark" field describes whether or not data
516
+ processing parameter tuning has been performed, the "summary" field briefly shows each operation in the data
517
+ processing pipeline and its corresponding optimal configuration, and the "tree" field provides complete
518
+ information about the structure of the data processing pipeline.
495
519
 
496
520
  .. code-block::
497
521
 
@@ -509,6 +533,8 @@ def set_enable_autotune(enable, filepath_prefix=None):
509
533
  }
510
534
 
511
535
  Examples:
536
+ >>> import mindspore.dataset as ds
537
+ >>>
512
538
  >>> # enable AutoTune and save optimized data pipeline configuration
513
539
  >>> ds.config.set_enable_autotune(True, "/path/to/autotune_out.json")
514
540
  >>>
@@ -542,13 +568,14 @@ def set_enable_autotune(enable, filepath_prefix=None):
542
568
 
543
569
  def get_enable_autotune():
544
570
  """
545
- Get whether AutoTune is currently enabled.
571
+ Get whether AutoTune is currently enabled, it is disabled by default.
546
572
 
547
573
  Returns:
548
574
  bool, whether AutoTune is currently enabled.
549
575
 
550
576
  Examples:
551
577
  >>> # get the state of AutoTune
578
+ >>> import mindspore.dataset as ds
552
579
  >>> autotune_flag = ds.config.get_enable_autotune()
553
580
  """
554
581
  return _config.get_enable_autotune()
@@ -558,7 +585,7 @@ def set_autotune_interval(interval):
558
585
  """
559
586
  Set the configuration adjustment interval (in steps) for AutoTune.
560
587
 
561
- The default setting is 0, which will adjust the configuration after each epoch.
588
+ The default setting is ``0``, which will adjust the configuration after each epoch.
562
589
  Otherwise, the configuration will be adjusted every `interval` steps.
563
590
 
564
591
  Args:
@@ -570,6 +597,7 @@ def set_autotune_interval(interval):
570
597
 
571
598
  Examples:
572
599
  >>> # set a new interval for AutoTune
600
+ >>> import mindspore.dataset as ds
573
601
  >>> ds.config.set_autotune_interval(30)
574
602
  """
575
603
  if not isinstance(interval, int) or isinstance(interval, bool):
@@ -583,12 +611,14 @@ def set_autotune_interval(interval):
583
611
  def get_autotune_interval():
584
612
  """
585
613
  Get the current configuration adjustment interval (in steps) for AutoTune.
614
+ If `set_autotune_interval` is never called before, the default value(0) will be returned.
586
615
 
587
616
  Returns:
588
617
  int, the configuration adjustment interval (in steps) for AutoTune.
589
618
 
590
619
  Examples:
591
620
  >>> # get the global configuration of the autotuning interval
621
+ >>> import mindspore.dataset as ds
592
622
  >>> autotune_interval = ds.config.get_autotune_interval()
593
623
  """
594
624
  return _config.get_autotune_interval()
@@ -606,13 +636,12 @@ def get_enable_shared_mem():
606
636
 
607
637
  Examples:
608
638
  >>> # Get the flag of shared memory feature.
639
+ >>> import mindspore.dataset as ds
609
640
  >>> shared_mem_flag = ds.config.get_enable_shared_mem()
610
641
  """
611
642
  # For Windows and MacOS we forbid shared mem function temporarily
612
643
  enable_shared_mem = _config.get_enable_shared_mem()
613
644
  if enable_shared_mem and platform.system().lower() in {"windows", "darwin"}:
614
- logger.warning(
615
- "For Windows and MacOS we forbid shared mem function temporarily.")
616
645
  _config.set_enable_shared_mem(False)
617
646
  return False
618
647
  return enable_shared_mem
@@ -620,20 +649,24 @@ def get_enable_shared_mem():
620
649
 
621
650
  def set_enable_shared_mem(enable):
622
651
  """
623
- Set the default state of shared memory flag. If shared_mem_enable is True, will use shared memory queues
624
- to pass data to processes that are created for operations that set python_multiprocessing=True.
652
+ Set whether to use shared memory for interprocess communication when data processing multiprocessing is turned on.
653
+
654
+ Using shared memory can speed up the efficiency of data transfer between processes.
655
+
656
+ Shared memory is used by default.
625
657
 
626
658
  Note:
627
- `set_enable_shared_mem` is not supported on Windows and MacOS platforms yet.
659
+ Windows and MacOS systems are not supported yet.
628
660
 
629
661
  Args:
630
- enable (bool): Whether to use shared memory in operations when python_multiprocessing=True.
662
+ enable (bool): Whether to use shared memory for interprocess communication.
631
663
 
632
664
  Raises:
633
- TypeError: If `enable` is not a boolean data type.
665
+ TypeError: If `enable` is not of type bool.
634
666
 
635
667
  Examples:
636
668
  >>> # Enable shared memory feature to improve the performance of Python multiprocessing.
669
+ >>> import mindspore.dataset as ds
637
670
  >>> ds.config.set_enable_shared_mem(True)
638
671
  """
639
672
  if not isinstance(enable, bool):
@@ -650,17 +683,26 @@ def set_enable_shared_mem(enable):
650
683
 
651
684
  def set_sending_batches(batch_num):
652
685
  """
653
- Set the default sending batches when training with sink_mode=True in Ascend device.
686
+ Set the upper limit on the number of batches of data that the Host can send to the Device.
687
+
688
+ Can be used to implement customized data sending control logic to solve the problem of
689
+ Device out of memory. In each epoch, when the actual number of batches sent to the Device
690
+ reaches this value, the Host will stop continuing to send until the user increases this
691
+ upper limit again through this interface.
692
+
693
+ Currently, it is only supported when training in sink mode with Ascend backend, which can
694
+ be enabled via the :class:`mindspore.train.Model.train` interface.
654
695
 
655
696
  Args:
656
- batch_num (int): the total sending batches, when batch_num is set, it will wait unless sending batches
657
- increase, default is 0 which means will send all batches in dataset.
697
+ batch_num (int): The upper limit on the number of batches of data that the Host can
698
+ send to the Device. ``0`` indicates that there is no upper limit for sending.
658
699
 
659
700
  Raises:
660
701
  TypeError: If `batch_num` is not of type int.
661
702
 
662
703
  Examples:
663
704
  >>> # Set a new global configuration value for the sending batches
705
+ >>> import mindspore.dataset as ds
664
706
  >>> ds.config.set_sending_batches(10)
665
707
  """
666
708
  if not isinstance(batch_num, int) or isinstance(batch_num, bool):
@@ -681,6 +723,7 @@ def set_auto_offload(offload):
681
723
 
682
724
  Examples:
683
725
  >>> # Enable automatic offload feature
726
+ >>> import mindspore.dataset as ds
684
727
  >>> ds.config.set_auto_offload(True)
685
728
  """
686
729
  if not isinstance(offload, bool):
@@ -690,13 +733,14 @@ def set_auto_offload(offload):
690
733
 
691
734
  def get_auto_offload():
692
735
  """
693
- Get the state of the automatic offload flag (True or False)
736
+ Get the state of the automatic offload flag (True or False), it is disabled by default.
694
737
 
695
738
  Returns:
696
739
  bool, Whether the automatic offload feature is enabled.
697
740
 
698
741
  Examples:
699
742
  >>> # Get the global configuration of the automatic offload feature.
743
+ >>> import mindspore.dataset as ds
700
744
  >>> auto_offload = ds.config.get_auto_offload()
701
745
  """
702
746
  return _config.get_auto_offload()
@@ -708,13 +752,14 @@ def set_enable_watchdog(enable):
708
752
  Watchdog is a thread which cleans up hanging subprocesses.
709
753
 
710
754
  Args:
711
- enable (bool): Whether to launch a watchdog Python thread. System default: True.
755
+ enable (bool): Whether to launch a watchdog Python thread.
712
756
 
713
757
  Raises:
714
758
  TypeError: If `enable` is not a boolean data type.
715
759
 
716
760
  Examples:
717
761
  >>> # Set a new global configuration value for the state of watchdog Python thread as enabled.
762
+ >>> import mindspore.dataset as ds
718
763
  >>> ds.config.set_enable_watchdog(True)
719
764
  """
720
765
  if not isinstance(enable, bool):
@@ -725,13 +770,14 @@ def set_enable_watchdog(enable):
725
770
  def get_enable_watchdog():
726
771
  """
727
772
  Get the state of watchdog Python thread to indicate enabled or disabled state.
728
- This is the DEFAULT watchdog Python thread state value used for the all processes.
773
+ This is a Python watchdog thread used for all processes, it is enabled by default.
729
774
 
730
775
  Returns:
731
776
  bool, the default state of watchdog Python thread enabled.
732
777
 
733
778
  Examples:
734
779
  >>> # Get the global configuration of watchdog Python thread.
780
+ >>> import mindspore.dataset as ds
735
781
  >>> watchdog_state = ds.config.get_enable_watchdog()
736
782
  """
737
783
  return _config.get_enable_watchdog()
@@ -744,7 +790,7 @@ def set_multiprocessing_timeout_interval(interval):
744
790
 
745
791
  Args:
746
792
  interval (int): Interval (in seconds) to be used for multiprocessing/multithreading timeout when main
747
- process/thread gets data from subprocess/child threads. System default: 300s.
793
+ process/thread gets data from subprocess/child threads.
748
794
 
749
795
  Raises:
750
796
  TypeError: If `interval` is not of type int.
@@ -752,6 +798,7 @@ def set_multiprocessing_timeout_interval(interval):
752
798
 
753
799
  Examples:
754
800
  >>> # Set a new global configuration value for multiprocessing/multithreading timeout when getting data.
801
+ >>> import mindspore.dataset as ds
755
802
  >>> ds.config.set_multiprocessing_timeout_interval(300)
756
803
  """
757
804
  if not isinstance(interval, int) or isinstance(interval, bool):
@@ -769,12 +816,14 @@ def get_multiprocessing_timeout_interval():
769
816
 
770
817
  Returns:
771
818
  int, interval (in seconds) for multiprocessing/multithreading timeout when main process/thread gets data from
772
- subprocesses/child threads. Default: 300s.
819
+ subprocesses/child threads. If `set_multiprocessing_timeout_interval` is never called before,
820
+ the default value(300) will be returned.
773
821
 
774
822
  Examples:
775
823
  >>> # Get the global configuration of multiprocessing/multithreading timeout when main process/thread gets data
776
824
  >>> # from subprocesses/child threads. If set_multiprocessing_timeout_interval() is never called before, the
777
825
  >>> # default value(300) will be returned.
826
+ >>> import mindspore.dataset as ds
778
827
  >>> multiprocessing_timeout_interval = ds.config.get_multiprocessing_timeout_interval()
779
828
  """
780
829
  return _config.get_multiprocessing_timeout_interval()
@@ -791,6 +840,7 @@ def set_dynamic_shape(is_dynamic):
791
840
  TypeError: If `is_dynamic` is not a boolean data type.
792
841
 
793
842
  Examples:
843
+ >>> import mindspore.dataset as ds
794
844
  >>> ds.config.set_dynamic_shape(True)
795
845
  """
796
846
  if not isinstance(is_dynamic, bool):
@@ -800,11 +850,13 @@ def set_dynamic_shape(is_dynamic):
800
850
 
801
851
  def get_dynamic_shape():
802
852
  """
803
- Get the dynamic shape flag of the dataset
853
+ Get the dynamic shape flag of the dataset, it is set to False by default.
854
+
804
855
  Returns:
805
856
  bool, whether the dataset is dynamic shape.
806
857
 
807
858
  Examples:
859
+ >>> import mindspore.dataset as ds
808
860
  >>> is_dynamic_shape = ds.config.get_dynamic_shape()
809
861
  """
810
862
  return _config.get_dynamic_shape()
@@ -816,12 +868,13 @@ def set_fast_recovery(fast_recovery):
816
868
  (In fast mode, random augmentations may not get same results as before the failure occurred).
817
869
 
818
870
  Args:
819
- fast_recovery (bool): Whether the dataset pipeline recovers in fast mode. System default: True.
871
+ fast_recovery (bool): Whether the dataset pipeline recovers in fast mode.
820
872
 
821
873
  Raises:
822
874
  TypeError: If `fast_recovery` is not a boolean data type.
823
875
 
824
876
  Examples:
877
+ >>> import mindspore.dataset as ds
825
878
  >>> ds.config.set_fast_recovery(False)
826
879
  """
827
880
  if not isinstance(fast_recovery, bool):
@@ -832,11 +885,13 @@ def set_fast_recovery(fast_recovery):
832
885
  def get_fast_recovery():
833
886
  """
834
887
  Get whether the fast recovery mode is enabled for the current dataset pipeline.
888
+ It is set to True by default.
835
889
 
836
890
  Returns:
837
891
  bool, whether the dataset recovers fast in failover reset.
838
892
 
839
893
  Examples:
894
+ >>> import mindspore.dataset as ds
840
895
  >>> is_fast_recovery = ds.config.get_fast_recovery()
841
896
  """
842
897
  return _config.get_fast_recovery()
@@ -848,28 +903,27 @@ def set_debug_mode(debug_mode_flag: bool, debug_hook_list: list = None):
848
903
  sequentially with a single thread.
849
904
 
850
905
  Note:
851
- - When debug_mode is enabled, if set_seed has not yet been issued, MindData will internally set the seed to 1
852
- so that debug mode execution of the dataset pipeline can produce deterministic results.
853
- - When debug_mode is enabled, many configuration settings are ignored, including the following noteworthy
854
- settings:
855
- - auto_offload (False is used.)
856
- - enable_autotune (False is used.)
857
- - error_samples_mode (ErrorSamplesMode.RETURN is used.)
858
- - num_parallel_workers (Value 1 is used.)
859
- - If both debug_mode is enabled and a dataset pipeline has Map operation with offload set, then offloading is
860
- ignored.
861
- - If both debug_mode is enabled and a dataset pipeline has Map operation or Batch operation with
862
- python_multiprocessing=True, then Python multiprocessing is ignored.
863
- - If both debug_mode is enabled and a dataset pipeline has GeneratorDataset with
864
- python_multiprocessing=True (the default value), then Python multiprocessing is ignored.
865
- - If both debug_mode is enabled and a dataset operation has cache set, then the cache is dropped.
866
- - If both debug_mode and profiling are enabled, then dataset profiling is ignored.
906
+ When debug_mode is enabled,
907
+
908
+ - If random seed has not been set, will internally set the seed to 1.
909
+ So that debug mode execution of the dataset pipeline can produce deterministic results.
910
+
911
+ - The following configuration settings are ignored:
912
+
913
+ - auto_offload (False is used.)
914
+ - enable_autotune (False is used.)
915
+ - error_samples_mode (ErrorSamplesMode.RETURN is used.)
916
+ - num_parallel_workers (Value 1 is used.)
917
+
918
+ - The `offload` parameter in `map` operation will be ignored.
919
+ - The `python_multiprocessing` parameter in `GeneratorDataset`, `map`/`batch` operation will be ignored.
920
+ - The `cache` parameter in Dataset loading API will be ignored.
867
921
 
868
922
  Args:
869
923
  debug_mode_flag (bool): Whether dataset pipeline debug mode is enabled, which forces the pipeline
870
924
  to run synchronously and sequentially.
871
925
  debug_hook_list (list[DebugHook]): a list of debug hook objects to be inserted before and after each
872
- transform operation in map operation. Default: None, which means to use `[PrintMetaDataHook]`,
926
+ transform operation in map operation. Default: ``None``, which means to use basic print hook,
873
927
  which prints shape/size/type of each input/output data of each transformation.
874
928
 
875
929
  Raises:
@@ -878,19 +932,19 @@ def set_debug_mode(debug_mode_flag: bool, debug_hook_list: list = None):
878
932
  TypeError: If any item in `debug_hook_list` is not DebugHook type.
879
933
 
880
934
  Examples:
881
- 1. Enable dataset pipeline debug mode and use default debug hook.
935
+ >>> import mindspore.dataset as ds
936
+ >>> import mindspore.dataset.vision as vision
937
+ >>> import mindspore.dataset.debug as debug
938
+ >>>
939
+ >>> # 1. Enable dataset pipeline debug mode and use default debug hook.
882
940
  >>> # Print shape and type of input/output data of each transform op in map operator.
883
941
  >>> ds.config.set_debug_mode(True)
884
-
885
- 2. Enable dataset pipeline debug mode and use pre-defined debug hook provided by MindData.
886
- >>> import mindspore.dataset.debug as debug
887
942
  >>>
943
+ >>> # 2. Enable dataset pipeline debug mode and use pre-defined debug hook provided by MindData.
888
944
  >>> ds.config.set_debug_mode(True, debug_hook_list=[debug.PrintDataHook()])
889
-
890
- 3. Enable dataset pipeline debug mode and use user-defined debug hook. It must define a
891
- class inherited from DebugHook.
892
- >>> import mindspore.dataset.debug as debug
893
945
  >>>
946
+ >>> # 3. Enable dataset pipeline debug mode and use user-defined debug hook. It must define a
947
+ >>> # class inherited from DebugHook.
894
948
  >>> class CustomizedHook(debug.DebugHook):
895
949
  ... def __init__(self):
896
950
  ... super().__init__()
@@ -900,16 +954,17 @@ def set_debug_mode(debug_mode_flag: bool, debug_hook_list: list = None):
900
954
  ... return args
901
955
  >>>
902
956
  >>> ds.config.set_debug_mode(True, debug_hook_list=[CustomizedHook()])
903
-
904
- 4. Enable dataset pipeline debug mode and use user-defined debug hook and insert by users manually.
957
+ >>>
958
+ >>> # 4. Enable dataset pipeline debug mode and use user-defined debug hook and insert by users manually.
905
959
  >>> ds.config.set_debug_mode(True)
906
960
  >>> dataset = ds.ImageFolderDataset(dataset_dir="/path/to/image_folder_dataset_directory")
907
- >>> # the debug hook is added after `Decode` operation.
961
+ >>>
962
+ >>> # The debug hook is added after Decode operation.
908
963
  >>> dataset = dataset.map([vision.Decode(), CustomizedHook(), vision.CenterCrop(100)])
909
964
  """
910
965
  if not isinstance(debug_mode_flag, bool):
911
966
  raise TypeError("debug_mode_flag isn't of type boolean.")
912
- if not debug_hook_list:
967
+ if debug_hook_list is None:
913
968
  debug_hook_list = [PrintMetaDataHook()]
914
969
  if not isinstance(debug_hook_list, list):
915
970
  raise TypeError("debug_hook_list is not a list.")
@@ -927,12 +982,13 @@ def set_debug_mode(debug_mode_flag: bool, debug_hook_list: list = None):
927
982
 
928
983
  def get_debug_mode():
929
984
  """
930
- Get the debug_mode flag of the dataset pipeline
985
+ Get whether debug mode is currently enabled for the data pipeline.
931
986
 
932
987
  Returns:
933
- bool, whether dataset pipeline debug mode is enabled
988
+ bool, whether data pipeline debug mode is enabled.
934
989
 
935
990
  Examples:
991
+ >>> import mindspore.dataset as ds
936
992
  >>> debug_mode = ds.config.get_debug_mode()
937
993
  """
938
994
  return _config.get_debug_mode()
@@ -986,8 +1042,8 @@ def set_error_samples_mode(error_samples_mode):
986
1042
 
987
1043
  Note:
988
1044
  - This error samples feature is only applicable to the Map operation in a dataset pipeline.
989
- - For 'ErrorSamplesMode.REPLACE' mode, a cache of other samples will be used.
990
- - If 'ErrorSamplesMode.SKIP' mode is used in a distributed setting, beware to manually ensure the
1045
+ - For ``ErrorSamplesMode.REPLACE`` mode, a cache of other samples will be used.
1046
+ - If ``ErrorSamplesMode.SKIP`` mode is used in a distributed setting, beware to manually ensure the
991
1047
  number of valid samples are the same for each shard (otherwise one may encounter hangs).
992
1048
  One technique is to manually concat a dataset of all valid samples plus a
993
1049
  take operation for the number of skipped erroneous samples.
@@ -995,18 +1051,18 @@ def set_error_samples_mode(error_samples_mode):
995
1051
  Args:
996
1052
  error_samples_mode (ErrorSamplesMode): The method in which erroneous samples should be processed in a dataset
997
1053
  pipeline. It can be any of [ErrorSamplesMode.RETURN, ErrorSamplesMode.REPLACE, ErrorSamplesMode.SKIP].
998
- System default: ErrorSamplesMode.RETURN.
999
1054
 
1000
- - ErrorSamplesMode.RETURN: means erroneous sample results in error raised and returned.
1055
+ - ``ErrorSamplesMode.RETURN``: means erroneous sample results in error raised and returned.
1001
1056
 
1002
- - ErrorSamplesMode.REPLACE: means erroneous sample is replaced with a correct sample.
1057
+ - ``ErrorSamplesMode.REPLACE``: means erroneous sample is replaced with a correct sample.
1003
1058
 
1004
- - ErrorSamplesMode.SKIP: means erroneous sample is skipped.
1059
+ - ``ErrorSamplesMode.SKIP``: means erroneous sample is skipped.
1005
1060
 
1006
1061
  Raises:
1007
1062
  TypeError: If `error_samples_mode` is not of type ErrorSamplesMode.
1008
1063
 
1009
1064
  Examples:
1065
+ >>> import mindspore.dataset as ds
1010
1066
  >>> ds.config.set_error_samples_mode(ds.config.ErrorSamplesMode.SKIP)
1011
1067
  """
1012
1068
  type_check(error_samples_mode, (ErrorSamplesMode,), "error_samples_mode")
@@ -1016,6 +1072,7 @@ def set_error_samples_mode(error_samples_mode):
1016
1072
  def get_error_samples_mode():
1017
1073
  """
1018
1074
  Get the current configuration for strategy for processing erroneous samples in a dataset pipeline.
1075
+ If `set_error_samples_mode` is never called before, the default setting is ErrorSamplesMode.RETURN.
1019
1076
 
1020
1077
  Returns:
1021
1078
  ErrorSamplesMode, The method in which erroneous samples should be processed in a dataset pipeline.
@@ -1025,6 +1082,7 @@ def get_error_samples_mode():
1025
1082
  - ErrorSamplesMode.SKIP: means erroneous sample is skipped.
1026
1083
 
1027
1084
  Examples:
1085
+ >>> import mindspore.dataset as ds
1028
1086
  >>> error_samples_mode = ds.config.get_error_samples_mode()
1029
1087
  """
1030
1088
  return _CDE_TO_PYTHON_ERROR_SAMPLES_MODE.get(_config.get_error_samples_mode())