mindspore 2.5.0__cp311-cp311-win_amd64.whl → 2.6.0rc1__cp311-cp311-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mindspore might be problematic. Click here for more details.

Files changed (491) hide show
  1. mindspore/.commit_id +1 -1
  2. mindspore/Microsoft.VisualStudio.Telemetry.dll +0 -0
  3. mindspore/Newtonsoft.Json.dll +0 -0
  4. mindspore/__init__.py +6 -4
  5. mindspore/_c_dataengine.cp311-win_amd64.pyd +0 -0
  6. mindspore/_c_expression.cp311-win_amd64.pyd +0 -0
  7. mindspore/_c_mindrecord.cp311-win_amd64.pyd +0 -0
  8. mindspore/_check_jit_forbidden_api.py +3 -0
  9. mindspore/_checkparam.py +3 -33
  10. mindspore/_deprecated/__init__.py +17 -0
  11. mindspore/_deprecated/jit.py +198 -0
  12. mindspore/_extends/builtin_operations.py +1 -1
  13. mindspore/_extends/parse/__init__.py +6 -7
  14. mindspore/_extends/parse/compile_config.py +19 -0
  15. mindspore/_extends/parse/deprecated/deprecated_tensor_method.py +22 -3
  16. mindspore/_extends/parse/jit_fallback_modules/__init__.py +0 -0
  17. mindspore/_extends/parse/jit_fallback_modules/check_utils.py +123 -0
  18. mindspore/_extends/parse/jit_fallback_modules/third_party_modules.py +50 -0
  19. mindspore/_extends/parse/parser.py +24 -193
  20. mindspore/_extends/parse/resources.py +1 -5
  21. mindspore/_extends/parse/standard_method.py +97 -74
  22. mindspore/_extends/pijit/__init__.py +2 -2
  23. mindspore/_extends/pijit/pijit_func_white_list.py +16 -11
  24. mindspore/_extends/pijit/tensor_func_list.py +27 -0
  25. mindspore/_extends/utils.py +1 -1
  26. mindspore/amp.py +4 -4
  27. mindspore/atlprov.dll +0 -0
  28. mindspore/avcodec-59.dll +0 -0
  29. mindspore/avdevice-59.dll +0 -0
  30. mindspore/avfilter-8.dll +0 -0
  31. mindspore/avformat-59.dll +0 -0
  32. mindspore/avutil-57.dll +0 -0
  33. mindspore/boost/__init__.py +2 -2
  34. mindspore/boost/base.py +3 -7
  35. mindspore/boost/boost_cell_wrapper.py +2 -2
  36. mindspore/c1.dll +0 -0
  37. mindspore/c1xx.dll +0 -0
  38. mindspore/c2.dll +0 -0
  39. mindspore/common/__init__.py +4 -3
  40. mindspore/common/_grad_function.py +56 -0
  41. mindspore/common/_pijit_context.py +14 -5
  42. mindspore/common/_register_for_tensor.py +1 -1
  43. mindspore/common/_stub_tensor.py +5 -10
  44. mindspore/common/_tensor_cpp_method.py +1 -1
  45. mindspore/common/_tensor_docs.py +1915 -3287
  46. mindspore/common/api.py +341 -354
  47. mindspore/common/auto_dynamic_shape.py +41 -44
  48. mindspore/common/dtype.py +5 -2
  49. mindspore/common/dump.py +7 -5
  50. mindspore/common/file_system.py +3 -0
  51. mindspore/common/hook_handle.py +5 -3
  52. mindspore/common/initializer.py +10 -6
  53. mindspore/common/jit_begin_end.py +94 -0
  54. mindspore/common/jit_config.py +6 -1
  55. mindspore/common/jit_context.py +76 -0
  56. mindspore/common/jit_trace.py +378 -0
  57. mindspore/common/lazy_inline.py +2 -2
  58. mindspore/common/mutable.py +5 -4
  59. mindspore/common/parameter.py +106 -39
  60. mindspore/common/seed.py +2 -2
  61. mindspore/common/sparse_tensor.py +23 -17
  62. mindspore/common/tensor.py +297 -714
  63. mindspore/communication/__init__.py +7 -5
  64. mindspore/communication/_comm_helper.py +47 -2
  65. mindspore/communication/comm_func.py +70 -53
  66. mindspore/communication/management.py +83 -17
  67. mindspore/context.py +214 -560
  68. mindspore/dataset/__init__.py +44 -20
  69. mindspore/dataset/audio/__init__.py +2 -8
  70. mindspore/dataset/audio/transforms.py +3 -17
  71. mindspore/dataset/core/config.py +3 -3
  72. mindspore/dataset/engine/cache_client.py +1 -1
  73. mindspore/dataset/engine/datasets.py +102 -120
  74. mindspore/dataset/engine/datasets_audio.py +22 -22
  75. mindspore/dataset/engine/datasets_standard_format.py +43 -24
  76. mindspore/dataset/engine/datasets_text.py +78 -85
  77. mindspore/dataset/engine/datasets_user_defined.py +108 -76
  78. mindspore/dataset/engine/datasets_vision.py +111 -108
  79. mindspore/dataset/engine/iterators.py +5 -3
  80. mindspore/dataset/engine/obs/obs_mindrecord_dataset.py +1 -1
  81. mindspore/dataset/engine/samplers.py +279 -57
  82. mindspore/dataset/engine/serializer_deserializer.py +2 -1
  83. mindspore/dataset/engine/validators.py +10 -0
  84. mindspore/dataset/text/__init__.py +7 -6
  85. mindspore/dataset/text/transforms.py +6 -5
  86. mindspore/dataset/text/utils.py +3 -3
  87. mindspore/dataset/transforms/__init__.py +0 -9
  88. mindspore/dataset/transforms/transforms.py +3 -3
  89. mindspore/dataset/utils/browse_dataset.py +1 -1
  90. mindspore/dataset/vision/__init__.py +2 -9
  91. mindspore/dataset/vision/transforms.py +202 -158
  92. mindspore/dataset/vision/utils.py +7 -5
  93. mindspore/device_context/ascend/op_debug.py +60 -1
  94. mindspore/device_context/ascend/op_tuning.py +0 -4
  95. mindspore/device_manager.py +39 -3
  96. mindspore/dnnl.dll +0 -0
  97. mindspore/dpcmi.dll +0 -0
  98. mindspore/experimental/es/embedding_service.py +35 -27
  99. mindspore/experimental/map_parameter.py +4 -4
  100. mindspore/experimental/optim/adadelta.py +22 -26
  101. mindspore/experimental/optim/adagrad.py +4 -4
  102. mindspore/experimental/optim/adam.py +4 -0
  103. mindspore/experimental/optim/adamax.py +4 -4
  104. mindspore/experimental/optim/adamw.py +4 -0
  105. mindspore/experimental/optim/asgd.py +1 -1
  106. mindspore/experimental/optim/lr_scheduler.py +40 -22
  107. mindspore/experimental/optim/radam.py +5 -5
  108. mindspore/experimental/optim/rprop.py +1 -1
  109. mindspore/experimental/optim/sgd.py +1 -1
  110. mindspore/hal/contiguous_tensors_handle.py +6 -10
  111. mindspore/hal/device.py +55 -81
  112. mindspore/hal/event.py +38 -55
  113. mindspore/hal/memory.py +93 -144
  114. mindspore/hal/stream.py +81 -125
  115. mindspore/include/dataset/constants.h +7 -4
  116. mindspore/include/dataset/execute.h +2 -2
  117. mindspore/jpeg62.dll +0 -0
  118. mindspore/log.py +40 -2
  119. mindspore/mindrecord/__init__.py +20 -7
  120. mindspore/mindspore_backend_common.dll +0 -0
  121. mindspore/mindspore_backend_manager.dll +0 -0
  122. mindspore/mindspore_common.dll +0 -0
  123. mindspore/mindspore_core.dll +0 -0
  124. mindspore/mindspore_dump.dll +0 -0
  125. mindspore/mindspore_frontend.dll +0 -0
  126. mindspore/mindspore_glog.dll +0 -0
  127. mindspore/mindspore_memory_pool.dll +0 -0
  128. mindspore/mindspore_ms_backend.dll +0 -0
  129. mindspore/mindspore_ops.dll +0 -0
  130. mindspore/{mindspore_backend.dll → mindspore_ops_host.dll} +0 -0
  131. mindspore/mindspore_ops_kernel_common.dll +0 -0
  132. mindspore/mindspore_profiler.dll +0 -0
  133. mindspore/mindspore_pyboost.dll +0 -0
  134. mindspore/mindspore_pynative.dll +0 -0
  135. mindspore/mindspore_res_manager.dll +0 -0
  136. mindspore/mindspore_runtime_pipeline.dll +0 -0
  137. mindspore/mint/__init__.py +131 -700
  138. mindspore/mint/distributed/__init__.py +5 -1
  139. mindspore/mint/distributed/distributed.py +194 -109
  140. mindspore/mint/linalg/__init__.py +2 -0
  141. mindspore/mint/nn/__init__.py +280 -18
  142. mindspore/mint/nn/functional.py +282 -64
  143. mindspore/mint/nn/layer/__init__.py +4 -0
  144. mindspore/mint/nn/layer/_functions.py +7 -3
  145. mindspore/mint/nn/layer/activation.py +120 -13
  146. mindspore/mint/nn/layer/conv.py +218 -24
  147. mindspore/mint/nn/layer/normalization.py +15 -16
  148. mindspore/mint/nn/layer/padding.py +1 -1
  149. mindspore/mint/nn/layer/pooling.py +66 -1
  150. mindspore/mint/optim/__init__.py +2 -1
  151. mindspore/mint/optim/sgd.py +171 -0
  152. mindspore/msobj140.dll +0 -0
  153. mindspore/mspdb140.dll +0 -0
  154. mindspore/mspdbcore.dll +0 -0
  155. mindspore/mspdbst.dll +0 -0
  156. mindspore/mspft140.dll +0 -0
  157. mindspore/msvcdis140.dll +0 -0
  158. mindspore/msvcp140_1.dll +0 -0
  159. mindspore/msvcp140_2.dll +0 -0
  160. mindspore/msvcp140_atomic_wait.dll +0 -0
  161. mindspore/msvcp140_codecvt_ids.dll +0 -0
  162. mindspore/nn/__init__.py +4 -1
  163. mindspore/nn/cell.py +1250 -176
  164. mindspore/nn/layer/activation.py +23 -21
  165. mindspore/nn/layer/basic.py +22 -16
  166. mindspore/nn/layer/container.py +1 -1
  167. mindspore/nn/layer/conv.py +22 -17
  168. mindspore/nn/layer/embedding.py +9 -8
  169. mindspore/nn/layer/normalization.py +48 -42
  170. mindspore/nn/layer/pooling.py +75 -31
  171. mindspore/nn/layer/transformer.py +11 -10
  172. mindspore/nn/learning_rate_schedule.py +4 -2
  173. mindspore/nn/loss/loss.py +27 -19
  174. mindspore/nn/optim/ada_grad.py +6 -5
  175. mindspore/nn/optim/adadelta.py +9 -7
  176. mindspore/nn/optim/adafactor.py +1 -1
  177. mindspore/nn/optim/adam.py +16 -12
  178. mindspore/nn/optim/adamax.py +8 -7
  179. mindspore/nn/optim/adasum.py +5 -5
  180. mindspore/nn/optim/asgd.py +1 -1
  181. mindspore/nn/optim/ftrl.py +11 -9
  182. mindspore/nn/optim/lamb.py +1 -1
  183. mindspore/nn/optim/lazyadam.py +12 -10
  184. mindspore/nn/optim/momentum.py +7 -6
  185. mindspore/nn/optim/optimizer.py +2 -2
  186. mindspore/nn/optim/proximal_ada_grad.py +12 -10
  187. mindspore/nn/optim/rmsprop.py +13 -12
  188. mindspore/nn/optim/rprop.py +9 -7
  189. mindspore/nn/optim/sgd.py +9 -6
  190. mindspore/nn/optim/tft_wrapper.py +5 -2
  191. mindspore/nn/probability/bijector/bijector.py +17 -11
  192. mindspore/nn/probability/bijector/gumbel_cdf.py +5 -5
  193. mindspore/nn/probability/bijector/invert.py +2 -2
  194. mindspore/nn/probability/bijector/scalar_affine.py +3 -3
  195. mindspore/nn/probability/bijector/softplus.py +3 -2
  196. mindspore/nn/probability/distribution/beta.py +3 -3
  197. mindspore/nn/probability/distribution/categorical.py +1 -1
  198. mindspore/nn/probability/distribution/cauchy.py +4 -2
  199. mindspore/nn/probability/distribution/exponential.py +6 -7
  200. mindspore/nn/probability/distribution/gamma.py +2 -2
  201. mindspore/nn/probability/distribution/gumbel.py +2 -2
  202. mindspore/nn/probability/distribution/half_normal.py +5 -3
  203. mindspore/nn/probability/distribution/logistic.py +5 -3
  204. mindspore/nn/probability/distribution/poisson.py +1 -1
  205. mindspore/nn/probability/distribution/uniform.py +5 -3
  206. mindspore/nn/reinforcement/_tensors_queue.py +1 -1
  207. mindspore/nn/reinforcement/tensor_array.py +1 -1
  208. mindspore/nn/wrap/__init__.py +6 -6
  209. mindspore/nn/wrap/cell_wrapper.py +178 -117
  210. mindspore/nn/wrap/grad_reducer.py +45 -36
  211. mindspore/nn/wrap/loss_scale.py +3 -3
  212. mindspore/numpy/array_creations.py +3 -3
  213. mindspore/numpy/array_ops.py +1 -1
  214. mindspore/numpy/math_ops.py +4 -4
  215. mindspore/numpy/utils.py +1 -2
  216. mindspore/numpy/utils_const.py +1 -2
  217. mindspore/opencv_core452.dll +0 -0
  218. mindspore/opencv_imgcodecs452.dll +0 -0
  219. mindspore/opencv_imgproc452.dll +0 -0
  220. mindspore/ops/__init__.py +3 -2
  221. mindspore/ops/_grad_experimental/grad_comm_ops.py +18 -3
  222. mindspore/ops/_grad_experimental/grad_debug_ops.py +8 -1
  223. mindspore/ops/_grad_experimental/taylor_rule.py +29 -0
  224. mindspore/ops/_register_for_op.py +0 -11
  225. mindspore/{ops_generate → ops/_utils}/arg_dtype_cast.py +123 -4
  226. mindspore/{ops_generate → ops/_utils}/arg_handler.py +3 -4
  227. mindspore/ops/_vmap/vmap_array_ops.py +7 -6
  228. mindspore/ops/_vmap/vmap_grad_nn_ops.py +2 -1
  229. mindspore/ops/_vmap/vmap_math_ops.py +4 -7
  230. mindspore/ops/_vmap/vmap_nn_ops.py +9 -8
  231. mindspore/ops/auto_generate/__init__.py +4 -3
  232. mindspore/ops/auto_generate/cpp_create_prim_instance_helper.py +102 -49
  233. mindspore/ops/auto_generate/gen_extend_func.py +281 -135
  234. mindspore/ops/auto_generate/gen_ops_def.py +2574 -2326
  235. mindspore/ops/auto_generate/gen_ops_prim.py +8566 -2755
  236. mindspore/ops/auto_generate/pyboost_inner_prim.py +106 -76
  237. mindspore/ops/composite/__init__.py +2 -1
  238. mindspore/ops/composite/base.py +19 -24
  239. mindspore/ops/composite/math_ops.py +6 -16
  240. mindspore/ops/composite/multitype_ops/__init__.py +5 -2
  241. mindspore/ops/composite/multitype_ops/_compile_utils.py +2 -3
  242. mindspore/ops/composite/multitype_ops/_constexpr_utils.py +1 -2
  243. mindspore/ops/composite/multitype_ops/add_impl.py +2 -1
  244. mindspore/ops/composite/multitype_ops/bitwise_and_impl.py +2 -1
  245. mindspore/ops/composite/multitype_ops/bitwise_or_impl.py +2 -1
  246. mindspore/ops/composite/multitype_ops/bitwise_xor_impl.py +2 -1
  247. mindspore/ops/composite/multitype_ops/div_impl.py +6 -4
  248. mindspore/ops/composite/multitype_ops/equal_impl.py +4 -3
  249. mindspore/ops/composite/multitype_ops/floordiv_impl.py +2 -1
  250. mindspore/ops/composite/multitype_ops/getitem_impl.py +3 -2
  251. mindspore/ops/composite/multitype_ops/greater_equal_impl.py +4 -3
  252. mindspore/ops/composite/multitype_ops/greater_impl.py +4 -3
  253. mindspore/ops/composite/multitype_ops/in_impl.py +2 -1
  254. mindspore/ops/composite/multitype_ops/invert_impl.py +50 -0
  255. mindspore/ops/composite/multitype_ops/left_shift_impl.py +2 -1
  256. mindspore/ops/composite/multitype_ops/less_equal_impl.py +4 -3
  257. mindspore/ops/composite/multitype_ops/less_impl.py +4 -3
  258. mindspore/ops/composite/multitype_ops/logic_not_impl.py +3 -2
  259. mindspore/ops/composite/multitype_ops/logical_and_impl.py +2 -1
  260. mindspore/ops/composite/multitype_ops/logical_or_impl.py +2 -1
  261. mindspore/ops/composite/multitype_ops/mod_impl.py +2 -1
  262. mindspore/ops/composite/multitype_ops/mul_impl.py +3 -2
  263. mindspore/ops/composite/multitype_ops/negative_impl.py +2 -1
  264. mindspore/ops/composite/multitype_ops/not_equal_impl.py +2 -1
  265. mindspore/ops/composite/multitype_ops/not_in_impl.py +2 -1
  266. mindspore/ops/composite/multitype_ops/ones_like_impl.py +18 -0
  267. mindspore/ops/composite/multitype_ops/pow_impl.py +2 -1
  268. mindspore/ops/composite/multitype_ops/right_shift_impl.py +2 -1
  269. mindspore/ops/composite/multitype_ops/setitem_impl.py +2 -1
  270. mindspore/ops/composite/multitype_ops/sub_impl.py +2 -1
  271. mindspore/ops/function/__init__.py +28 -2
  272. mindspore/ops/function/_add_attr_func.py +58 -0
  273. mindspore/ops/function/array_func.py +1629 -2345
  274. mindspore/ops/function/clip_func.py +38 -45
  275. mindspore/ops/function/debug_func.py +36 -44
  276. mindspore/ops/function/grad/__init__.py +1 -0
  277. mindspore/ops/function/grad/grad_func.py +104 -71
  278. mindspore/ops/function/image_func.py +1 -1
  279. mindspore/ops/function/linalg_func.py +46 -78
  280. mindspore/ops/function/math_func.py +3035 -3705
  281. mindspore/ops/function/nn_func.py +676 -241
  282. mindspore/ops/function/other_func.py +159 -1
  283. mindspore/ops/function/parameter_func.py +17 -30
  284. mindspore/ops/function/random_func.py +204 -361
  285. mindspore/ops/function/reshard_func.py +4 -70
  286. mindspore/ops/function/sparse_func.py +3 -3
  287. mindspore/ops/function/sparse_unary_func.py +5 -5
  288. mindspore/ops/function/spectral_func.py +25 -58
  289. mindspore/ops/function/vmap_func.py +24 -17
  290. mindspore/ops/functional.py +6 -4
  291. mindspore/ops/functional_overload.py +547 -4
  292. mindspore/ops/op_info_register.py +32 -244
  293. mindspore/ops/operations/__init__.py +10 -5
  294. mindspore/ops/operations/_custom_ops_utils.py +247 -0
  295. mindspore/ops/operations/_grad_ops.py +1 -10
  296. mindspore/ops/operations/_inner_ops.py +5 -76
  297. mindspore/ops/operations/_ms_kernel.py +4 -10
  298. mindspore/ops/operations/_rl_inner_ops.py +1 -1
  299. mindspore/ops/operations/_scalar_ops.py +3 -2
  300. mindspore/ops/operations/_sequence_ops.py +1 -1
  301. mindspore/ops/operations/_tensor_array.py +1 -1
  302. mindspore/ops/operations/array_ops.py +37 -22
  303. mindspore/ops/operations/comm_ops.py +150 -107
  304. mindspore/ops/operations/custom_ops.py +221 -23
  305. mindspore/ops/operations/debug_ops.py +115 -16
  306. mindspore/ops/operations/inner_ops.py +1 -1
  307. mindspore/ops/operations/linalg_ops.py +1 -58
  308. mindspore/ops/operations/manually_defined/_inner.py +1 -1
  309. mindspore/ops/operations/manually_defined/ops_def.py +746 -79
  310. mindspore/ops/operations/math_ops.py +21 -18
  311. mindspore/ops/operations/nn_ops.py +65 -191
  312. mindspore/ops/operations/other_ops.py +62 -9
  313. mindspore/ops/operations/random_ops.py +13 -7
  314. mindspore/ops/operations/reshard_ops.py +1 -1
  315. mindspore/ops/operations/sparse_ops.py +2 -2
  316. mindspore/ops/primitive.py +43 -32
  317. mindspore/ops/tensor_method.py +232 -13
  318. mindspore/ops_generate/__init__.py +0 -5
  319. mindspore/ops_generate/aclnn/__init__.py +0 -0
  320. mindspore/ops_generate/{aclnn_kernel_register_auto_cc_generator.py → aclnn/aclnn_kernel_register_auto_cc_generator.py} +43 -18
  321. mindspore/ops_generate/{gen_aclnn_implement.py → aclnn/gen_aclnn_implement.py} +49 -51
  322. mindspore/ops_generate/api/__init__.py +0 -0
  323. mindspore/ops_generate/{add_tensor_docs_generator.py → api/add_tensor_docs_generator.py} +9 -7
  324. mindspore/ops_generate/{cpp_create_prim_instance_helper_generator.py → api/cpp_create_prim_instance_helper_generator.py} +6 -9
  325. mindspore/ops_generate/{functional_map_cpp_generator.py → api/functional_map_cpp_generator.py} +25 -12
  326. mindspore/ops_generate/{functional_overload_py_generator.py → api/functional_overload_py_generator.py} +8 -6
  327. mindspore/ops_generate/{functions_cc_generator.py → api/functions_cc_generator.py} +14 -10
  328. mindspore/ops_generate/api/gen_api.py +103 -0
  329. mindspore/ops_generate/{op_api_proto.py → api/op_api_proto.py} +98 -69
  330. mindspore/ops_generate/{tensor_func_reg_cpp_generator.py → api/tensor_func_reg_cpp_generator.py} +82 -43
  331. mindspore/ops_generate/common/__init__.py +0 -0
  332. mindspore/ops_generate/common/gen_constants.py +91 -0
  333. mindspore/ops_generate/{gen_utils.py → common/gen_utils.py} +72 -19
  334. mindspore/ops_generate/{op_proto.py → common/op_proto.py} +64 -1
  335. mindspore/ops_generate/{template.py → common/template.py} +96 -84
  336. mindspore/ops_generate/gen_ops.py +23 -325
  337. mindspore/ops_generate/op_def/__init__.py +0 -0
  338. mindspore/ops_generate/op_def/gen_op_def.py +90 -0
  339. mindspore/ops_generate/{lite_ops_cpp_generator.py → op_def/lite_ops_cpp_generator.py} +47 -11
  340. mindspore/ops_generate/{ops_def_cc_generator.py → op_def/ops_def_cc_generator.py} +18 -7
  341. mindspore/ops_generate/{ops_def_h_generator.py → op_def/ops_def_h_generator.py} +5 -5
  342. mindspore/ops_generate/{ops_name_h_generator.py → op_def/ops_name_h_generator.py} +30 -15
  343. mindspore/ops_generate/op_def/ops_primitive_h_generator.py +125 -0
  344. mindspore/ops_generate/op_def_py/__init__.py +0 -0
  345. mindspore/ops_generate/op_def_py/gen_op_def_py.py +47 -0
  346. mindspore/ops_generate/{op_def_py_generator.py → op_def_py/op_def_py_generator.py} +6 -5
  347. mindspore/ops_generate/{op_prim_py_generator.py → op_def_py/op_prim_py_generator.py} +24 -15
  348. mindspore/ops_generate/pyboost/__init__.py +0 -0
  349. mindspore/ops_generate/{auto_grad_impl_cc_generator.py → pyboost/auto_grad_impl_cc_generator.py} +11 -7
  350. mindspore/ops_generate/{auto_grad_reg_cc_generator.py → pyboost/auto_grad_reg_cc_generator.py} +7 -7
  351. mindspore/ops_generate/{gen_pyboost_func.py → pyboost/gen_pyboost_func.py} +40 -16
  352. mindspore/ops_generate/{op_template_parser.py → pyboost/op_template_parser.py} +105 -24
  353. mindspore/ops_generate/{pyboost_functions_cpp_generator.py → pyboost/pyboost_functions_cpp_generator.py} +55 -18
  354. mindspore/ops_generate/{pyboost_functions_h_generator.py → pyboost/pyboost_functions_h_generator.py} +42 -10
  355. mindspore/ops_generate/{pyboost_functions_py_generator.py → pyboost/pyboost_functions_py_generator.py} +6 -6
  356. mindspore/ops_generate/{pyboost_grad_function_cpp_generator.py → pyboost/pyboost_grad_function_cpp_generator.py} +11 -10
  357. mindspore/ops_generate/{pyboost_inner_prim_generator.py → pyboost/pyboost_inner_prim_generator.py} +8 -7
  358. mindspore/ops_generate/{pyboost_native_grad_functions_generator.py → pyboost/pyboost_native_grad_functions_generator.py} +14 -10
  359. mindspore/ops_generate/{pyboost_op_cpp_code_generator.py → pyboost/pyboost_op_cpp_code_generator.py} +140 -53
  360. mindspore/ops_generate/{pyboost_overload_functions_cpp_generator.py → pyboost/pyboost_overload_functions_cpp_generator.py} +28 -15
  361. mindspore/ops_generate/{pyboost_utils.py → pyboost/pyboost_utils.py} +88 -4
  362. mindspore/ops_generate/resources/__init__.py +0 -0
  363. mindspore/ops_generate/resources/resource_list.py +30 -0
  364. mindspore/ops_generate/resources/resource_loader.py +36 -0
  365. mindspore/ops_generate/resources/resource_manager.py +64 -0
  366. mindspore/ops_generate/resources/yaml_loader.py +88 -0
  367. mindspore/ops_generate/tensor_py_cc_generator.py +122 -0
  368. mindspore/parallel/__init__.py +6 -2
  369. mindspore/parallel/_auto_parallel_context.py +133 -6
  370. mindspore/parallel/_cell_wrapper.py +130 -15
  371. mindspore/parallel/_parallel_serialization.py +95 -4
  372. mindspore/parallel/_ps_context.py +1 -1
  373. mindspore/parallel/_recovery_context.py +7 -2
  374. mindspore/parallel/_tensor.py +142 -18
  375. mindspore/parallel/_utils.py +198 -25
  376. mindspore/parallel/algo_parameter_config.py +3 -3
  377. mindspore/parallel/auto_parallel.py +732 -0
  378. mindspore/parallel/checkpoint_convert.py +159 -0
  379. mindspore/parallel/checkpoint_transform.py +656 -37
  380. mindspore/parallel/cluster/process_entity/_api.py +151 -19
  381. mindspore/parallel/cluster/run.py +1 -1
  382. mindspore/parallel/function/__init__.py +24 -0
  383. mindspore/parallel/function/reshard_func.py +259 -0
  384. mindspore/parallel/nn/__init__.py +25 -0
  385. mindspore/parallel/nn/parallel_cell_wrapper.py +263 -0
  386. mindspore/parallel/nn/parallel_grad_reducer.py +169 -0
  387. mindspore/parallel/parameter_broadcast.py +24 -13
  388. mindspore/parallel/shard.py +137 -61
  389. mindspore/parallel/transform_safetensors.py +287 -95
  390. mindspore/pgodb140.dll +0 -0
  391. mindspore/pgort140.dll +0 -0
  392. mindspore/profiler/__init__.py +9 -5
  393. mindspore/profiler/analysis/parser/ascend_cann_parser.py +6 -2
  394. mindspore/profiler/analysis/parser/ms_framework_parser.py +4 -4
  395. mindspore/profiler/analysis/parser/timeline_assembly_factory/ascend_timeline_assembler.py +7 -4
  396. mindspore/profiler/analysis/parser/timeline_assembly_factory/trace_view_container.py +22 -0
  397. mindspore/profiler/analysis/parser/timeline_creator/fwk_timeline_creator.py +3 -3
  398. mindspore/profiler/analysis/parser/timeline_event/fwk_event.py +241 -86
  399. mindspore/profiler/analysis/viewer/ascend_communication_viewer.py +41 -2
  400. mindspore/profiler/analysis/viewer/ascend_kernel_details_viewer.py +33 -35
  401. mindspore/profiler/analysis/viewer/ascend_memory_viewer.py +7 -0
  402. mindspore/profiler/analysis/viewer/ascend_op_memory_viewer.py +8 -3
  403. mindspore/profiler/analysis/viewer/ascend_step_trace_time_viewer.py +141 -30
  404. mindspore/profiler/analysis/viewer/ms_dataset_viewer.py +5 -6
  405. mindspore/profiler/common/ascend_msprof_exporter.py +5 -4
  406. mindspore/profiler/common/constant.py +12 -0
  407. mindspore/profiler/common/msprof_cmd_tool.py +42 -23
  408. mindspore/profiler/common/path_manager.py +24 -0
  409. mindspore/profiler/common/profiler_context.py +26 -2
  410. mindspore/profiler/common/profiler_meta_data.py +74 -0
  411. mindspore/profiler/common/profiler_parameters.py +59 -18
  412. mindspore/profiler/common/profiler_path_manager.py +66 -7
  413. mindspore/profiler/dynamic_profiler.py +112 -79
  414. mindspore/profiler/envprofiler.py +26 -1
  415. mindspore/profiler/experimental_config.py +197 -0
  416. mindspore/profiler/mstx.py +57 -14
  417. mindspore/profiler/platform/npu_profiler.py +33 -7
  418. mindspore/profiler/profiler.py +541 -45
  419. mindspore/profiler/profiler_action_controller.py +1 -1
  420. mindspore/profiler/profiler_interface.py +4 -0
  421. mindspore/profiler/schedule.py +57 -22
  422. mindspore/rewrite/api/node.py +15 -13
  423. mindspore/rewrite/api/symbol_tree.py +1 -1
  424. mindspore/run_check/_check_version.py +25 -14
  425. mindspore/run_check/run_check.py +1 -1
  426. mindspore/runtime/__init__.py +2 -2
  427. mindspore/runtime/executor.py +40 -11
  428. mindspore/runtime/memory.py +25 -8
  429. mindspore/safeguard/rewrite_obfuscation.py +12 -9
  430. mindspore/swresample-4.dll +0 -0
  431. mindspore/swscale-6.dll +0 -0
  432. mindspore/tbbmalloc.dll +0 -0
  433. mindspore/tinyxml2.dll +0 -0
  434. mindspore/train/__init__.py +8 -8
  435. mindspore/train/_utils.py +35 -7
  436. mindspore/train/amp.py +1 -1
  437. mindspore/train/callback/__init__.py +2 -2
  438. mindspore/train/callback/_callback.py +2 -16
  439. mindspore/train/callback/_checkpoint.py +24 -40
  440. mindspore/train/callback/_cluster_monitor.py +14 -18
  441. mindspore/train/callback/_flops_collector.py +2 -3
  442. mindspore/train/callback/_history.py +7 -4
  443. mindspore/train/callback/_lambda_callback.py +2 -2
  444. mindspore/train/callback/_landscape.py +0 -3
  445. mindspore/train/callback/_loss_monitor.py +2 -1
  446. mindspore/train/callback/_on_request_exit.py +6 -5
  447. mindspore/train/callback/_reduce_lr_on_plateau.py +11 -6
  448. mindspore/train/callback/_summary_collector.py +8 -13
  449. mindspore/train/callback/_time_monitor.py +2 -1
  450. mindspore/train/callback/{_tft_register.py → _train_fault_tolerance.py} +179 -103
  451. mindspore/train/data_sink.py +25 -2
  452. mindspore/train/dataset_helper.py +4 -5
  453. mindspore/train/loss_scale_manager.py +8 -7
  454. mindspore/train/metrics/accuracy.py +3 -3
  455. mindspore/train/metrics/confusion_matrix.py +9 -9
  456. mindspore/train/metrics/error.py +3 -3
  457. mindspore/train/metrics/hausdorff_distance.py +4 -4
  458. mindspore/train/metrics/mean_surface_distance.py +3 -3
  459. mindspore/train/metrics/metric.py +0 -12
  460. mindspore/train/metrics/occlusion_sensitivity.py +4 -2
  461. mindspore/train/metrics/precision.py +8 -6
  462. mindspore/train/metrics/recall.py +9 -9
  463. mindspore/train/metrics/root_mean_square_surface_distance.py +2 -2
  464. mindspore/train/mind_ir_pb2.py +19 -12
  465. mindspore/train/model.py +176 -103
  466. mindspore/train/serialization.py +246 -988
  467. mindspore/train/summary/_summary_adapter.py +2 -2
  468. mindspore/train/summary/summary_record.py +1 -1
  469. mindspore/turbojpeg.dll +0 -0
  470. mindspore/utils/__init__.py +3 -2
  471. mindspore/utils/dryrun.py +4 -2
  472. mindspore/utils/hooks.py +81 -0
  473. mindspore/utils/utils.py +138 -4
  474. mindspore/vcmeta.dll +0 -0
  475. mindspore/vcruntime140.dll +0 -0
  476. mindspore/vcruntime140_1.dll +0 -0
  477. mindspore/version.py +1 -1
  478. {mindspore-2.5.0.dist-info → mindspore-2.6.0rc1.dist-info}/METADATA +2 -1
  479. {mindspore-2.5.0.dist-info → mindspore-2.6.0rc1.dist-info}/RECORD +483 -438
  480. mindspore/_install_custom.py +0 -43
  481. mindspore/common/_register_for_adapter.py +0 -74
  482. mindspore/ops/auto_generate/gen_arg_dtype_cast.py +0 -252
  483. mindspore/ops/auto_generate/gen_arg_handler.py +0 -136
  484. mindspore/ops/operations/_opaque_predicate_registry.py +0 -41
  485. mindspore/ops_generate/gen_constants.py +0 -190
  486. mindspore/ops_generate/gen_ops_inner_prim.py +0 -131
  487. mindspore/ops_generate/ops_primitive_h_generator.py +0 -81
  488. /mindspore/ops_generate/{base_generator.py → common/base_generator.py} +0 -0
  489. {mindspore-2.5.0.dist-info → mindspore-2.6.0rc1.dist-info}/WHEEL +0 -0
  490. {mindspore-2.5.0.dist-info → mindspore-2.6.0rc1.dist-info}/entry_points.txt +0 -0
  491. {mindspore-2.5.0.dist-info → mindspore-2.6.0rc1.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,4 @@
1
- # Copyright 2023 Huawei Technologies Co., Ltd
1
+ # Copyright 2023-2025 Huawei Technologies Co., Ltd
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -13,342 +13,40 @@
13
13
  # limitations under the License.
14
14
  # ============================================================================
15
15
  """
16
- Generate operator definition from ops.yaml
16
+ Auto generate ops files.
17
17
  """
18
- import argparse
19
- import copy
20
18
  import logging
21
- import os
22
- import shutil
23
- import pathlib
24
- from gen_utils import (check_change_and_replace_file, merge_files,
25
- merge_files_append, safe_load_yaml)
26
- from op_prim_py_generator import OpPrimPyGenerator
27
- from op_def_py_generator import OpDefPyGenerator
28
- from aclnn_kernel_register_auto_cc_generator import AclnnKernelRegisterAutoCcGenerator
29
- from cpp_create_prim_instance_helper_generator import CppCreatePrimInstanceHelperGenerator
30
- from ops_def_cc_generator import OpsDefCcGenerator
31
- from ops_def_h_generator import OpsDefHGenerator
32
- from ops_primitive_h_generator import OpsPrimitiveHGenerator
33
- from lite_ops_cpp_generator import LiteOpsCcGenerator, LiteOpsHGenerator
34
- from ops_name_h_generator import OpsNameHGenerator
35
- from functional_map_cpp_generator import FunctionalMapCppGenerator
36
- from add_tensor_docs_generator import AddTensorDocsGenerator
37
- from functional_overload_py_generator import FunctionalOverloadPyGenerator
38
19
 
39
- from op_proto import OpProto
40
- from op_api_proto import load_api_protos_from_yaml
41
- from tensor_func_reg_cpp_generator import TensorFuncRegCppGenerator
42
- from gen_pyboost_func import gen_pyboost_code
20
+ from resources.resource_manager import prepare_resources
21
+ from common import gen_utils
43
22
 
44
- import gen_constants as K
23
+ from op_def.gen_op_def import generate_ops_def_files
24
+ from op_def_py.gen_op_def_py import generate_ops_py_files
25
+ from api.gen_api import generate_api_files
26
+ from aclnn.aclnn_kernel_register_auto_cc_generator import generate_aclnn_reg_file
27
+ from pyboost.gen_pyboost_func import gen_pyboost_code
45
28
 
46
29
 
47
- def generate_ops_prim_file(work_path, op_protos, doc_dict, file_pre):
48
- generator = OpPrimPyGenerator()
49
- generator.generate(work_path, op_protos, doc_dict, file_pre)
30
+ module_generators = [
31
+ generate_ops_py_files, # generate ops python files
32
+ generate_ops_def_files, # generate ops definition files
33
+ gen_pyboost_code, # generate pyboost code
34
+ generate_aclnn_reg_file, # generate aclnn kernelmod register
35
+ generate_api_files # generate api definition files
36
+ ]
50
37
 
51
38
 
52
- def generate_ops_def_file(work_path, os_protos, doc_dict, file_pre):
53
- generator = OpDefPyGenerator()
54
- generator.generate(work_path, os_protos, doc_dict, file_pre)
39
+ def main():
40
+ resource_mgr = prepare_resources()
55
41
 
42
+ for generator in module_generators:
43
+ generator(resource_mgr)
56
44
 
57
- def generate_ops_py_files(work_path, op_protos, doc_dict, file_pre):
58
- """
59
- Generate ops python file from yaml.
60
- """
61
- generate_ops_prim_file(work_path, op_protos, doc_dict, file_pre)
62
- generate_ops_def_file(work_path, op_protos, doc_dict, file_pre)
63
- shutil.copy(os.path.join(work_path, K.PY_OPS_GEN_PATH, 'ops_auto_generate_init.txt'),
64
- os.path.join(work_path, K.PY_AUTO_GEN_PATH, "__init__.py"))
65
-
66
-
67
- def call_ops_def_cc_generator(work_path, op_protos):
68
- generator = OpsDefCcGenerator()
69
- generator.generate(work_path, op_protos)
70
-
71
-
72
- def call_ops_def_h_generator(work_path, op_protos):
73
- generator = OpsDefHGenerator()
74
- generator.generate(work_path, op_protos)
75
-
76
-
77
- def call_ops_primitive_h_generator(work_path, op_protos):
78
- generator = OpsPrimitiveHGenerator()
79
- generator.generate(work_path, op_protos)
80
-
81
-
82
- def call_lite_ops_h_generator(work_path, op_protos):
83
- h_generator = LiteOpsHGenerator()
84
- h_generator.generate(work_path, op_protos)
85
-
86
-
87
- def call_lite_ops_cc_generator(work_path, op_protos):
88
- generator = LiteOpsCcGenerator()
89
- generator.generate(work_path, op_protos)
90
-
91
-
92
- def call_ops_name_h_generator(work_path, op_protos):
93
- h_generator = OpsNameHGenerator()
94
- h_generator.generate(work_path, op_protos)
95
-
96
-
97
- def generate_ops_cc_files(work_path, op_protos, op_protos_with_deprecated):
98
- """
99
- Generate ops c++ file from yaml.
100
- """
101
- call_ops_def_cc_generator(work_path, op_protos_with_deprecated)
102
- call_ops_def_h_generator(work_path, op_protos_with_deprecated)
103
- call_ops_primitive_h_generator(work_path, op_protos)
104
- call_lite_ops_h_generator(work_path, op_protos)
105
- call_lite_ops_cc_generator(work_path, op_protos)
106
- call_ops_name_h_generator(work_path, op_protos)
107
-
108
-
109
- def get_tensor_op_protos_with_deprecated(func_protos, op_protos):
110
- """
111
- Get op_protos with deprecated op_protos from func_protos.
112
- """
113
- tensor_op_protos = copy.deepcopy(op_protos)
114
- for _, item in func_protos.items():
115
- for func_proto in item:
116
- op_name = func_proto.op_proto.op_name
117
- if "deprecated" in func_proto.op_proto.op_name:
118
- func_proto.op_proto.op_class.name = ''.join(word.capitalize() for word in op_name.split('_'))
119
- if func_proto.op_proto.op_name[-1] == '_':
120
- func_proto.op_proto.op_class.name += '_'
121
- tensor_op_protos.append(func_proto.op_proto)
122
- return tensor_op_protos
123
-
124
-
125
- def generate_create_instance_helper_file(work_path, op_protos_with_deprecated):
126
- """
127
- Generate C++ helper file from yaml.
128
- """
129
- generator = CppCreatePrimInstanceHelperGenerator()
130
- generator.generate(work_path, op_protos_with_deprecated)
131
-
132
-
133
- def generate_aclnn_reg_file(work_path, op_protos):
134
- """
135
- Generate nnacl kernelmod register
136
- """
137
- generator = AclnnKernelRegisterAutoCcGenerator()
138
- generator.generate(work_path, op_protos)
139
-
140
-
141
- def generate_arg_handler_files(work_path):
142
- """
143
- Generate arg handler files.
144
- """
145
- dst_dir = os.path.join(work_path, K.PY_AUTO_GEN_PATH)
146
- src_arg_handler_path = os.path.join(work_path, K.PY_OPS_GEN_PATH, 'arg_handler.py')
147
- dst_arg_handler_path = os.path.join(dst_dir, 'gen_arg_handler.py')
148
- tmp_dst_arg_handler_path = os.path.join(dst_dir, 'tmp_gen_arg_handler.py')
149
- if not os.path.exists(dst_dir):
150
- os.makedirs(dst_dir, mode=0o700)
151
- shutil.copy(src_arg_handler_path, tmp_dst_arg_handler_path)
152
- check_change_and_replace_file(dst_arg_handler_path, tmp_dst_arg_handler_path)
153
-
154
- src_arg_dtype_cast_path = os.path.join(work_path, K.PY_OPS_GEN_PATH, 'arg_dtype_cast.py')
155
- dst_arg_dtype_cast_path = os.path.join(dst_dir, 'gen_arg_dtype_cast.py')
156
- tmp_arg_dtype_cast_path = os.path.join(dst_dir, 'tmp_arg_dtype_cast.py')
157
- shutil.copy(src_arg_dtype_cast_path, tmp_arg_dtype_cast_path)
158
- check_change_and_replace_file(dst_arg_dtype_cast_path, tmp_arg_dtype_cast_path)
159
-
160
-
161
- def gen_tensor_func_code(work_path, op_protos, func_protos, alias_api_mapping):
162
- generator = TensorFuncRegCppGenerator()
163
- generator.generate(work_path, op_protos, func_protos, alias_api_mapping)
164
-
165
-
166
- def gen_functional_map_code(work_path, tensor_method_protos, mint_func_protos, alias_api_mapping):
167
- generator = FunctionalMapCppGenerator()
168
- generator.generate(work_path, tensor_method_protos, mint_func_protos, alias_api_mapping)
169
-
170
-
171
- def gen_tensor_docs_code(work_path, tensor_docs_data):
172
- generator = AddTensorDocsGenerator()
173
- generator.generate(work_path, tensor_docs_data)
174
-
175
-
176
- def gen_functional_overload_py(work_path, mint_func_protos, function_doc_data, alias_api_mapping):
177
- generator = FunctionalOverloadPyGenerator()
178
- generator.generate(work_path, mint_func_protos, function_doc_data, alias_api_mapping)
179
-
180
-
181
- def main(args):
182
- current_path = os.path.dirname(os.path.realpath(__file__))
183
- work_path = os.path.join(current_path, '../../../../')
184
-
185
- if args.clear_auto_gen:
186
- delete_auto_gen_files(work_path)
187
-
188
- # merge ops yaml
189
- (doc_yaml_path, ops_yaml_path, deprecated_ops_yaml_path, ops_api_yaml_path,
190
- tensor_method_doc_yaml_path, mint_func_doc_yaml_path) = merge_ops_yaml(work_path)
191
-
192
- # make auto_generate dir
193
- cc_path = os.path.join(work_path, K.MS_OP_DEF_AUTO_GENERATE_PATH)
194
- pathlib.Path(cc_path).mkdir(parents=True, exist_ok=True)
195
-
196
- # generate arg_handler files
197
- generate_arg_handler_files(work_path)
198
-
199
- # read ops definition str and tensor method doc str
200
- ops_yaml_dict = safe_load_yaml(ops_yaml_path)
201
- doc_yaml_dict = safe_load_yaml(doc_yaml_path)
202
- deprecated_ops_yaml_dict = safe_load_yaml(deprecated_ops_yaml_path)
203
- ops_api_yaml_dict = safe_load_yaml(ops_api_yaml_path)
204
- tensor_method_doc_yaml_dict = safe_load_yaml(tensor_method_doc_yaml_path)
205
- mint_function_doc_yaml_dict = safe_load_yaml(mint_func_doc_yaml_path)
206
-
207
- op_protos = load_op_protos_from_ops_yaml(ops_yaml_dict)
208
- deprecated_op_protos = load_deprecated_op_protos_from_ops_yaml(deprecated_ops_yaml_dict)
209
- tensor_method_protos, mint_func_protos, alias_api_mapping \
210
- = load_api_protos_from_yaml(ops_api_yaml_dict, op_protos, deprecated_op_protos)
211
- # for generate tensor method deprecated in graph mode
212
- op_protos_with_deprecated = get_tensor_op_protos_with_deprecated(tensor_method_protos, op_protos)
213
-
214
- # generate ops python files
215
- generate_ops_py_files(work_path, op_protos, doc_yaml_dict, "gen")
216
- # generate ops c++ files
217
- generate_ops_cc_files(work_path, op_protos, op_protos_with_deprecated)
218
- # generate create prim instance helper file
219
- generate_create_instance_helper_file(work_path, op_protos_with_deprecated)
220
- # generate pyboost code
221
- gen_pyboost_code(work_path, op_protos, doc_yaml_dict, tensor_method_protos, mint_func_protos, alias_api_mapping)
222
- # generate aclnn kernelmod register
223
- generate_aclnn_reg_file(work_path, op_protos)
224
- # generate tensor_py func code
225
- gen_tensor_func_code(work_path, op_protos, tensor_method_protos, alias_api_mapping)
226
- # generate functional map code
227
- gen_functional_map_code(work_path, tensor_method_protos, mint_func_protos, alias_api_mapping)
228
- # generate _tensor_docs.py that attaches docs to tensor func APIs when import mindspore
229
- gen_tensor_docs_code(work_path, tensor_method_doc_yaml_dict)
230
- # generate functional_overload.py which init pybind mint APIs from cpp
231
- gen_functional_overload_py(work_path, mint_func_protos, mint_function_doc_yaml_dict, alias_api_mapping)
232
-
233
-
234
- def delete_auto_gen_files(work_path):
235
- """
236
- Deletes auto-generated files and folders.
237
- """
238
- auto_gen_code_file = get_auto_gen_path_from_gitignore(work_path)
239
-
240
- for name in auto_gen_code_file:
241
- # Recursively delete all single-level folder names
242
- if name.rstrip('/').count('/') == 0:
243
- for dir_path, dir_names, _ in os.walk(work_path, topdown=False):
244
- for dirname in dir_names:
245
- if dirname == name.rstrip('/'):
246
- folder_path = os.path.join(dir_path, dirname)
247
- logging.info("Recursively deleting folder: %s", folder_path)
248
- shutil.rmtree(folder_path)
249
- continue
250
-
251
- # Delete all individual files or folders
252
- tmp_path = os.path.join(work_path, name)
253
- if os.path.exists(tmp_path):
254
- if os.path.isdir(tmp_path):
255
- logging.info("Deleting folder: %s", tmp_path)
256
- shutil.rmtree(tmp_path)
257
- elif os.path.isfile(tmp_path):
258
- logging.info("Deleting file: %s", tmp_path)
259
- os.remove(tmp_path)
260
- else:
261
- logging.info("The path is not exist: %s", tmp_path)
262
-
263
-
264
- def get_auto_gen_path_from_gitignore(work_path):
265
- """
266
- Extracts a list of auto-gen file and folder paths from the "# auto gen code files" section in the .gitignore file.
267
- """
268
- file_path = os.path.join(work_path, ".gitignore")
269
- auto_gen_code_file_started = False
270
- auto_gen_code_file = []
271
- with open(file_path, 'r') as f:
272
- for line in f.readlines():
273
- if line.strip() == "# auto gen code files":
274
- auto_gen_code_file_started = True
275
- continue
276
- if auto_gen_code_file_started:
277
- if line.strip() and not line.strip().startswith("#"):
278
- auto_gen_code_file.append(line.strip())
279
- else:
280
- break
281
- return auto_gen_code_file
282
-
283
-
284
- def load_op_protos_from_ops_yaml(ops_yaml_data):
285
- op_protos = []
286
- for operator_name, operator_data in ops_yaml_data.items():
287
- op_proto = OpProto.load_from_yaml(operator_name, operator_data)
288
- op_protos.append(op_proto)
289
- return op_protos
290
-
291
-
292
- def load_deprecated_op_protos_from_ops_yaml(ops_yaml_data):
293
- op_protos = []
294
- for operator_name, operator_data in ops_yaml_data.items():
295
- op_proto = OpProto.load_from_yaml(operator_name, operator_data)
296
- op_proto.op_name = 'deprecated_' + operator_name
297
- op_protos.append(op_proto)
298
- return op_protos
299
-
300
-
301
- def merge_ops_yaml(work_path):
302
- """
303
- Merges operator YAML files scattered in different directories into a single file.
304
-
305
- Args:
306
- work_path (str): The path to the working directory.
307
-
308
- Returns:
309
- tuple: Paths to the merged documentation and operators YAML files.
310
- """
311
- ops_yaml_dir_path = os.path.join(work_path, K.MS_OP_DEF_YAML_PATH)
312
- ops_yaml_path = os.path.join(work_path, K.PY_OPS_GEN_PATH, 'ops.yaml')
313
- infer_ops_yaml_dir_path = os.path.join(ops_yaml_dir_path, "infer")
314
- merge_files(ops_yaml_dir_path, ops_yaml_path, '*op.yaml')
315
- merge_files_append(infer_ops_yaml_dir_path, ops_yaml_path, '*op.yaml')
316
-
317
- doc_yaml_dir_path = os.path.join(ops_yaml_dir_path, "doc")
318
- doc_yaml_path = os.path.join(work_path, K.PY_OPS_GEN_PATH, 'ops_doc.yaml')
319
- merge_files(doc_yaml_dir_path, doc_yaml_path, '*doc.yaml')
320
-
321
- ops_api_yaml_dir_path = os.path.join(work_path, K.MS_OP_API_YAML_PATH)
322
- ops_api_yaml_path = os.path.join(work_path, K.PY_OPS_GEN_PATH, 'api_def.yaml')
323
- merge_files(ops_api_yaml_dir_path, ops_api_yaml_path, '*.yaml')
324
-
325
- deprecated_ops_yaml_dir_path = os.path.join(work_path, K.MS_OP_DEPRECATED_DEF_YAML_PATH)
326
- deprecated_ops_yaml_path = os.path.join(work_path, K.PY_OPS_GEN_PATH, 'deprecated_ops.yaml')
327
- merge_files(deprecated_ops_yaml_dir_path, deprecated_ops_yaml_path, '*_method.yaml')
328
-
329
- tensor_method_doc_yaml_dir_path = os.path.join(work_path, K.MS_TENSOR_METHOD_DOC_YAML_PATH)
330
- tensor_method_doc_yaml_path = os.path.join(work_path, K.PY_OPS_GEN_PATH, 'tensor_method_doc.yaml')
331
- merge_files(tensor_method_doc_yaml_dir_path, tensor_method_doc_yaml_path, '*doc.yaml')
332
-
333
- mint_func_doc_yaml_dir_path = os.path.join(work_path, K.MS_MINT_FUNC_DOC_YAML_PATH)
334
- mint_func_doc_yaml_path = os.path.join(work_path, K.PY_OPS_GEN_PATH, 'mint_func_doc.yaml')
335
- merge_files(mint_func_doc_yaml_dir_path, mint_func_doc_yaml_path, '*doc.yaml')
336
-
337
- return (doc_yaml_path, ops_yaml_path, deprecated_ops_yaml_path,
338
- ops_api_yaml_path, tensor_method_doc_yaml_path, mint_func_doc_yaml_path)
339
-
340
-
341
- def parse_args():
342
- parser = argparse.ArgumentParser()
343
- parser.add_argument('--clear_auto_gen', default=False, help='clear all auto gen files')
344
- return parser.parse_args()
345
-
45
+ gen_utils.clear_obsolete_auto_gen_files()
346
46
 
347
47
  if __name__ == "__main__":
348
48
  try:
349
- arguments = parse_args()
350
- main(arguments)
351
- # pylint: disable=broad-except
352
- except Exception as e:
49
+ main()
50
+ except Exception as e: # pylint: disable=broad-except
353
51
  logging.critical("Auto generate failed, err info: %s", e)
354
52
  raise e
File without changes
@@ -0,0 +1,90 @@
1
+ # Copyright 2025 Huawei Technologies Co., Ltd
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ # ============================================================================
15
+ """
16
+ Generate operator definitions from ops.yaml
17
+ """
18
+ import copy
19
+
20
+ from common import gen_constants as K
21
+ from resources.resource_list import ResourceType
22
+
23
+ from .ops_def_cc_generator import OpsDefCcGenerator
24
+ from .ops_def_h_generator import OpsDefHGenerator
25
+ from .ops_name_h_generator import OpsNameHGenerator
26
+ from .ops_primitive_h_generator import OpsPrimitiveHGenerator
27
+ from .lite_ops_cpp_generator import LiteOpsCcGenerator, LiteOpsHGenerator
28
+
29
+
30
+ def call_ops_def_cc_generator(work_path, op_protos):
31
+ generator = OpsDefCcGenerator()
32
+ generator.generate(work_path, op_protos)
33
+
34
+
35
+ def call_ops_def_h_generator(work_path, op_protos):
36
+ generator = OpsDefHGenerator()
37
+ generator.generate(work_path, op_protos)
38
+
39
+
40
+ def call_ops_primitive_h_generator(work_path, op_protos):
41
+ generator = OpsPrimitiveHGenerator()
42
+ generator.generate(work_path, op_protos)
43
+
44
+
45
+ def call_lite_ops_h_generator(work_path, op_protos):
46
+ h_generator = LiteOpsHGenerator()
47
+ h_generator.generate(work_path, op_protos)
48
+
49
+
50
+ def call_lite_ops_cc_generator(work_path, op_protos):
51
+ generator = LiteOpsCcGenerator()
52
+ generator.generate(work_path, op_protos)
53
+
54
+
55
+ def call_ops_name_h_generator(work_path, op_protos):
56
+ h_generator = OpsNameHGenerator()
57
+ h_generator.generate(work_path, op_protos)
58
+
59
+
60
+ def get_tensor_op_protos_with_deprecated(func_protos, op_protos):
61
+ """
62
+ Get op_protos with deprecated op_protos from func_protos.
63
+ """
64
+ tensor_op_protos = copy.deepcopy(op_protos)
65
+ for _, item in func_protos.items():
66
+ for func_proto in item:
67
+ op_name = func_proto.op_proto.op_name
68
+ if "deprecated" in func_proto.op_proto.op_name:
69
+ func_proto.op_proto.op_class.name = ''.join(word.capitalize() for word in op_name.split('_'))
70
+ if func_proto.op_proto.op_name[-1] == '_':
71
+ func_proto.op_proto.op_class.name += '_'
72
+ tensor_op_protos.append(func_proto.op_proto)
73
+ return tensor_op_protos
74
+
75
+
76
+ def generate_ops_def_files(resource_mgr):
77
+ """
78
+ Generate ops c++ file from yaml.
79
+ """
80
+ work_path = K.WORK_DIR
81
+ op_protos = resource_mgr.get_resource(ResourceType.OP_PROTO)
82
+ tensor_method_protos = resource_mgr.get_resource(ResourceType.TENSOR_METHOD_PROTOS)
83
+ # for generate tensor method deprecated in graph mode
84
+ op_protos_with_deprecated = get_tensor_op_protos_with_deprecated(tensor_method_protos, op_protos)
85
+ call_ops_def_cc_generator(work_path, op_protos_with_deprecated)
86
+ call_ops_def_h_generator(work_path, op_protos_with_deprecated)
87
+ call_ops_primitive_h_generator(work_path, op_protos)
88
+ call_lite_ops_h_generator(work_path, op_protos)
89
+ call_lite_ops_cc_generator(work_path, op_protos)
90
+ call_ops_name_h_generator(work_path, op_protos)
@@ -18,14 +18,44 @@ Generates C++ header and source files for lite operations based on YAML configur
18
18
 
19
19
  import os
20
20
 
21
- import gen_constants as K
22
- import gen_utils
23
- import pyboost_utils
21
+ import common.gen_constants as K
22
+ import common.gen_utils as gen_utils
23
+ import common.template as template
24
+ from common.base_generator import BaseGenerator
25
+ from pyboost import pyboost_utils
24
26
 
25
- # refactored
26
- import template
27
27
 
28
- from base_generator import BaseGenerator
28
+ LITE_OPS_H = """
29
+ #ifndef MINDSPORE_CORE_OPS_GEN_LITE_OPS_H_
30
+ #define MINDSPORE_CORE_OPS_GEN_LITE_OPS_H_
31
+
32
+ #include <vector>
33
+ #include "ops/base_operator.h"
34
+
35
+ namespace mindspore::ops {
36
+ $ops_namespace_body
37
+
38
+ } // namespace mindspore::ops
39
+ #endif // MINDSPORE_CORE_OPS_GEN_LITE_OPS_H_
40
+ """
41
+
42
+ INC_OPS_HEAD = """
43
+ #include "$auto_gen_path/gen_ops_name_${ch}.h"
44
+ """
45
+
46
+ LITE_OPS_CC = """
47
+ #include "$auto_gen_path/gen_lite_ops.h"
48
+ ${inc_ops_head_str}
49
+ #include "mindapi/helper.h"
50
+ #include "ops/primitive_c.h"
51
+ #include "ops/base_operator.h"
52
+ #include "abstract/abstract_value.h"
53
+
54
+ namespace mindspore::ops {
55
+ $ops_namespace_body
56
+
57
+ } // namespace mindspore::ops
58
+ """
29
59
 
30
60
 
31
61
  class LiteOpsHGenerator(BaseGenerator):
@@ -37,7 +67,7 @@ class LiteOpsHGenerator(BaseGenerator):
37
67
  """
38
68
  Initializes the generator with the necessary templates for generating C++ header files.
39
69
  """
40
- self.lite_ops_h_template = template.Template(K.LITE_OPS_H)
70
+ self.lite_ops_h_template = template.Template(LITE_OPS_H)
41
71
  self.lite_ops_class_template = template.op_cc_template
42
72
  self.arg_prim_init_template = template.Template("\n"
43
73
  " void set_${arg_name}(const ${dtype} &${arg_name});\n"
@@ -89,9 +119,11 @@ class LiteOpsCcGenerator(BaseGenerator):
89
119
  """
90
120
  Initializes the generator with the necessary templates for generating C++ source files.
91
121
  """
92
- self.lite_ops_cc_template = template.Template(K.LITE_OPS_CC)
122
+ self.inc_ops_head_templat = template.Template(INC_OPS_HEAD)
123
+ self.lite_ops_cc_template = template.Template(LITE_OPS_CC)
93
124
  self.op_template = template.op_template
94
- self.register_primitive_c_template = template.Template("REGISTER_PRIMITIVE_C(kName${op_name}, ${op_name});\n"
125
+ self.register_primitive_c_template = template.Template("${op_name}::${op_name}():BaseOperator(kName${op_name}) {}\n"
126
+ "REGISTER_PRIMITIVE_C(kName${op_name}, ${op_name});\n"
95
127
  "MIND_API_OPERATOR_IMPL(${op_name}, BaseOperator);\n\n")
96
128
 
97
129
  def generate(self, work_path, op_protos):
@@ -106,6 +138,7 @@ class LiteOpsCcGenerator(BaseGenerator):
106
138
  None
107
139
  """
108
140
  lite_ops_cc_gen_list = []
141
+ inc_ops_head_list = set()
109
142
  for op_proto in op_protos:
110
143
  arg_prim_init_str = ""
111
144
  op_name = pyboost_utils.get_op_name(op_proto.op_name, op_proto.op_class.name)
@@ -120,9 +153,12 @@ class LiteOpsCcGenerator(BaseGenerator):
120
153
 
121
154
  self.register_primitive_c_template.replace(op_name=op_name)
122
155
  lite_ops_cc_gen_list.append(arg_prim_init_str + self.register_primitive_c_template.replace(op_name=op_name))
123
-
156
+ inc_ops_head_list.add(self.inc_ops_head_templat.replace(auto_gen_path=K.OP_DEF_AUTO_GENERATE_PATH,
157
+ ch=op_name[0].lower()))
158
+ sorted_inc_ops_head_str = sorted(inc_ops_head_list)
124
159
  lite_ops_cc = self.lite_ops_cc_template.replace(auto_gen_path=K.OP_DEF_AUTO_GENERATE_PATH,
125
- ops_namespace_body=lite_ops_cc_gen_list)
160
+ ops_namespace_body=lite_ops_cc_gen_list,
161
+ inc_ops_head_str=sorted_inc_ops_head_str)
126
162
 
127
163
  res_str = template.CC_LICENSE_STR + lite_ops_cc
128
164
  save_path = os.path.join(work_path, K.MS_OP_DEF_AUTO_GENERATE_PATH)
@@ -19,14 +19,25 @@ Module for generating C++ operator definition files.
19
19
  import os
20
20
  import math
21
21
 
22
- import gen_constants as K
23
- import gen_utils
22
+ import common.gen_constants as K
23
+ import common.gen_utils as gen_utils
24
24
 
25
25
  # refactored
26
- from op_proto import OpProto
27
- import template
26
+ from common.op_proto import OpProto
27
+ import common.template as template
28
28
 
29
- from base_generator import BaseGenerator
29
+ from common.base_generator import BaseGenerator
30
+
31
+
32
+ CC_OPS_DEF = """
33
+
34
+ #include "$auto_generate_path/gen_ops_def.h"
35
+ #include "ir/signature.h"
36
+ $gen_include
37
+
38
+ namespace mindspore::ops {$gen_cc_code
39
+ } // namespace mindspore::ops
40
+ """
30
41
 
31
42
 
32
43
  class OpsDefCcGenerator(BaseGenerator):
@@ -43,7 +54,7 @@ class OpsDefCcGenerator(BaseGenerator):
43
54
  self.empty_func_impl_declaration_template = template.Template("static OpFuncImpl g${class_name}FuncImpl;")
44
55
  self.func_impl_define_template = template.Template("g${class_name}FuncImpl")
45
56
  self.OP_PROTO_TEMPLATE = template.OP_PROTO_TEMPLATE
46
- self.CC_OPS_DEF_TEMPLATE = template.Template(K.CC_OPS_DEF)
57
+ self.CC_OPS_DEF_TEMPLATE = template.Template(CC_OPS_DEF)
47
58
 
48
59
  def generate(self, work_path, op_protos):
49
60
  """
@@ -60,7 +71,7 @@ class OpsDefCcGenerator(BaseGenerator):
60
71
  for op_proto in op_protos:
61
72
  operator_name = op_proto.op_name
62
73
  class_name = op_proto.op_class.name
63
- if "deprecated" not in operator_name:
74
+ if not op_proto.func_op:
64
75
  gen_include_list.append(self.include_template.replace(path=K.MS_OPS_FUNC_IMPL_PATH,
65
76
  operator_name=operator_name))
66
77
  func_impl_declaration_str = self.func_impl_declaration_template.replace(class_name=class_name)
@@ -21,11 +21,11 @@ and any additional operators provided. This is useful for managing operator inte
21
21
 
22
22
  import os
23
23
 
24
- import template
25
- from template import Template
26
- from gen_utils import save_file
27
- import gen_constants as K
28
- from base_generator import BaseGenerator
24
+ import common.template as template
25
+ from common.template import Template
26
+ from common.gen_utils import save_file
27
+ import common.gen_constants as K
28
+ from common.base_generator import BaseGenerator
29
29
 
30
30
 
31
31
  class OpsDefHGenerator(BaseGenerator):