mindspore 2.4.10__cp310-cp310-win_amd64.whl → 2.5.0__cp310-cp310-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mindspore might be problematic. Click here for more details.

Files changed (366) hide show
  1. mindspore/.commit_id +1 -1
  2. mindspore/__init__.py +8 -3
  3. mindspore/_c_dataengine.cp310-win_amd64.pyd +0 -0
  4. mindspore/_c_expression.cp310-win_amd64.pyd +0 -0
  5. mindspore/_c_mindrecord.cp310-win_amd64.pyd +0 -0
  6. mindspore/_checkparam.py +0 -5
  7. mindspore/_extends/parallel_compile/akg_compiler/gen_custom_op_files.py +1 -1
  8. mindspore/_extends/parse/compile_config.py +64 -0
  9. mindspore/_extends/parse/deprecated/__init__.py +0 -0
  10. mindspore/_extends/parse/deprecated/deprecated_tensor_method.py +375 -0
  11. mindspore/_extends/parse/parser.py +23 -5
  12. mindspore/_extends/parse/standard_method.py +123 -27
  13. mindspore/_extends/pijit/pijit_func_white_list.py +1 -1
  14. mindspore/amp.py +7 -1
  15. mindspore/avcodec-59.dll +0 -0
  16. mindspore/avdevice-59.dll +0 -0
  17. mindspore/avfilter-8.dll +0 -0
  18. mindspore/avformat-59.dll +0 -0
  19. mindspore/avutil-57.dll +0 -0
  20. mindspore/boost/boost_cell_wrapper.py +136 -41
  21. mindspore/common/__init__.py +3 -1
  22. mindspore/common/_register_for_tensor.py +0 -1
  23. mindspore/common/_stub_tensor.py +25 -4
  24. mindspore/common/_tensor_cpp_method.py +17 -0
  25. mindspore/common/_tensor_docs.py +6132 -0
  26. mindspore/common/api.py +98 -21
  27. mindspore/common/dtype.py +34 -34
  28. mindspore/common/dump.py +2 -1
  29. mindspore/common/file_system.py +8 -3
  30. mindspore/common/generator.py +2 -0
  31. mindspore/common/hook_handle.py +3 -1
  32. mindspore/common/initializer.py +3 -4
  33. mindspore/common/lazy_inline.py +8 -2
  34. mindspore/common/mindir_util.py +10 -2
  35. mindspore/common/parameter.py +31 -15
  36. mindspore/common/tensor.py +713 -1337
  37. mindspore/communication/__init__.py +1 -1
  38. mindspore/communication/_comm_helper.py +5 -0
  39. mindspore/communication/comm_func.py +215 -173
  40. mindspore/communication/management.py +23 -20
  41. mindspore/context.py +285 -191
  42. mindspore/dataset/__init__.py +23 -19
  43. mindspore/dataset/callback/ds_callback.py +2 -1
  44. mindspore/dataset/core/config.py +84 -3
  45. mindspore/dataset/engine/cache_admin.py +3 -3
  46. mindspore/dataset/engine/cache_client.py +5 -4
  47. mindspore/dataset/engine/datasets.py +192 -149
  48. mindspore/dataset/engine/datasets_audio.py +14 -0
  49. mindspore/dataset/engine/datasets_standard_format.py +11 -11
  50. mindspore/dataset/engine/datasets_text.py +38 -1
  51. mindspore/dataset/engine/datasets_user_defined.py +100 -66
  52. mindspore/dataset/engine/datasets_vision.py +81 -8
  53. mindspore/dataset/engine/iterators.py +281 -63
  54. mindspore/dataset/engine/obs/util.py +8 -0
  55. mindspore/dataset/engine/queue.py +40 -0
  56. mindspore/dataset/engine/samplers.py +26 -2
  57. mindspore/dataset/engine/serializer_deserializer.py +1 -1
  58. mindspore/dataset/engine/validators.py +43 -11
  59. mindspore/dataset/transforms/py_transforms_util.py +17 -0
  60. mindspore/dataset/transforms/transforms.py +29 -12
  61. mindspore/dataset/vision/validators.py +1 -2
  62. mindspore/device_context/__init__.py +21 -0
  63. mindspore/device_context/ascend/__init__.py +25 -0
  64. mindspore/device_context/ascend/device.py +72 -0
  65. mindspore/device_context/ascend/op_debug.py +94 -0
  66. mindspore/device_context/ascend/op_precision.py +193 -0
  67. mindspore/device_context/ascend/op_tuning.py +127 -0
  68. mindspore/device_context/cpu/__init__.py +25 -0
  69. mindspore/device_context/cpu/device.py +62 -0
  70. mindspore/device_context/cpu/op_tuning.py +43 -0
  71. mindspore/device_context/gpu/__init__.py +21 -0
  72. mindspore/device_context/gpu/device.py +70 -0
  73. mindspore/device_context/gpu/op_precision.py +67 -0
  74. mindspore/device_context/gpu/op_tuning.py +175 -0
  75. mindspore/device_manager.py +134 -0
  76. mindspore/dnnl.dll +0 -0
  77. mindspore/experimental/llm_boost/__init__.py +1 -0
  78. mindspore/experimental/llm_boost/ascend_native/__init__.py +22 -0
  79. mindspore/experimental/llm_boost/ascend_native/llama_boost_ascend_native.py +211 -0
  80. mindspore/experimental/llm_boost/ascend_native/llm_boost.py +52 -0
  81. mindspore/experimental/llm_boost/atb/boost_base.py +2 -3
  82. mindspore/experimental/llm_boost/atb/llama_boost.py +6 -1
  83. mindspore/experimental/llm_boost/register.py +1 -0
  84. mindspore/experimental/optim/adadelta.py +26 -22
  85. mindspore/experimental/optim/adam.py +3 -0
  86. mindspore/experimental/optim/lr_scheduler.py +33 -24
  87. mindspore/experimental/optim/radam.py +33 -30
  88. mindspore/hal/device.py +28 -0
  89. mindspore/hal/event.py +17 -0
  90. mindspore/hal/memory.py +94 -3
  91. mindspore/hal/stream.py +91 -6
  92. mindspore/include/api/context.h +0 -1
  93. mindspore/jpeg62.dll +0 -0
  94. mindspore/log.py +12 -0
  95. mindspore/mindrecord/__init__.py +1 -1
  96. mindspore/mindrecord/config.py +17 -316
  97. mindspore/mindrecord/filereader.py +1 -9
  98. mindspore/mindrecord/filewriter.py +5 -15
  99. mindspore/mindrecord/mindpage.py +1 -9
  100. mindspore/mindspore_backend.dll +0 -0
  101. mindspore/mindspore_common.dll +0 -0
  102. mindspore/mindspore_core.dll +0 -0
  103. mindspore/mindspore_glog.dll +0 -0
  104. mindspore/mindspore_ops.dll +0 -0
  105. mindspore/mint/__init__.py +824 -218
  106. mindspore/mint/distributed/__init__.py +66 -4
  107. mindspore/mint/distributed/distributed.py +2594 -44
  108. mindspore/mint/linalg/__init__.py +6 -0
  109. mindspore/mint/nn/__init__.py +473 -14
  110. mindspore/mint/nn/functional.py +486 -11
  111. mindspore/mint/nn/layer/__init__.py +17 -4
  112. mindspore/mint/nn/layer/_functions.py +330 -0
  113. mindspore/mint/nn/layer/activation.py +169 -1
  114. mindspore/mint/nn/layer/basic.py +123 -0
  115. mindspore/mint/nn/layer/conv.py +727 -0
  116. mindspore/mint/nn/layer/normalization.py +215 -19
  117. mindspore/mint/nn/layer/padding.py +797 -0
  118. mindspore/mint/nn/layer/pooling.py +170 -0
  119. mindspore/mint/optim/__init__.py +2 -1
  120. mindspore/mint/optim/adam.py +223 -0
  121. mindspore/mint/optim/adamw.py +26 -19
  122. mindspore/mint/special/__init__.py +2 -1
  123. mindspore/multiprocessing/__init__.py +5 -0
  124. mindspore/nn/cell.py +126 -19
  125. mindspore/nn/dynamic_lr.py +2 -1
  126. mindspore/nn/layer/activation.py +6 -6
  127. mindspore/nn/layer/basic.py +35 -25
  128. mindspore/nn/layer/channel_shuffle.py +3 -3
  129. mindspore/nn/layer/embedding.py +3 -3
  130. mindspore/nn/layer/normalization.py +8 -7
  131. mindspore/nn/layer/padding.py +4 -3
  132. mindspore/nn/layer/pooling.py +47 -13
  133. mindspore/nn/layer/rnn_cells.py +1 -1
  134. mindspore/nn/layer/rnns.py +2 -1
  135. mindspore/nn/layer/timedistributed.py +5 -5
  136. mindspore/nn/layer/transformer.py +48 -26
  137. mindspore/nn/learning_rate_schedule.py +5 -3
  138. mindspore/nn/loss/loss.py +31 -36
  139. mindspore/nn/optim/ada_grad.py +1 -0
  140. mindspore/nn/optim/adadelta.py +2 -2
  141. mindspore/nn/optim/adam.py +1 -1
  142. mindspore/nn/optim/lars.py +1 -4
  143. mindspore/nn/optim/optimizer.py +1 -1
  144. mindspore/nn/optim/rprop.py +2 -2
  145. mindspore/nn/optim/thor.py +2 -1
  146. mindspore/nn/utils/init.py +13 -11
  147. mindspore/nn/wrap/cell_wrapper.py +4 -6
  148. mindspore/nn/wrap/loss_scale.py +3 -4
  149. mindspore/numpy/array_creations.py +60 -62
  150. mindspore/numpy/array_ops.py +148 -143
  151. mindspore/numpy/logic_ops.py +41 -42
  152. mindspore/numpy/math_ops.py +361 -359
  153. mindspore/numpy/utils.py +16 -16
  154. mindspore/numpy/utils_const.py +4 -4
  155. mindspore/opencv_core452.dll +0 -0
  156. mindspore/opencv_imgcodecs452.dll +0 -0
  157. mindspore/opencv_imgproc452.dll +0 -0
  158. mindspore/ops/__init__.py +2 -1
  159. mindspore/ops/_grad_experimental/grad_comm_ops.py +94 -13
  160. mindspore/ops/_grad_experimental/grad_debug_ops.py +6 -1
  161. mindspore/ops/_grad_experimental/grad_inner_ops.py +9 -0
  162. mindspore/ops/_grad_experimental/grad_math_ops.py +2 -1
  163. mindspore/ops/_op_impl/cpu/__init__.py +1 -0
  164. mindspore/ops/_op_impl/cpu/raise_op.py +28 -0
  165. mindspore/ops/_vmap/vmap_array_ops.py +20 -19
  166. mindspore/ops/_vmap/vmap_base.py +0 -2
  167. mindspore/ops/_vmap/vmap_grad_nn_ops.py +19 -13
  168. mindspore/ops/_vmap/vmap_math_ops.py +11 -9
  169. mindspore/ops/_vmap/vmap_nn_ops.py +20 -34
  170. mindspore/ops/auto_generate/cpp_create_prim_instance_helper.py +149 -12
  171. mindspore/ops/auto_generate/gen_arg_handler.py +0 -61
  172. mindspore/ops/auto_generate/gen_extend_func.py +554 -60
  173. mindspore/ops/auto_generate/gen_ops_def.py +1621 -115
  174. mindspore/ops/auto_generate/gen_ops_prim.py +8024 -3409
  175. mindspore/ops/auto_generate/pyboost_inner_prim.py +183 -79
  176. mindspore/ops/composite/base.py +1 -1
  177. mindspore/ops/composite/multitype_ops/_compile_utils.py +229 -30
  178. mindspore/ops/composite/multitype_ops/pow_impl.py +0 -29
  179. mindspore/ops/function/__init__.py +12 -0
  180. mindspore/ops/function/array_func.py +561 -159
  181. mindspore/ops/function/clip_func.py +64 -0
  182. mindspore/ops/function/debug_func.py +28 -20
  183. mindspore/ops/function/image_func.py +1 -1
  184. mindspore/ops/function/linalg_func.py +5 -4
  185. mindspore/ops/function/math_func.py +1659 -290
  186. mindspore/ops/function/nn_func.py +988 -317
  187. mindspore/ops/function/parameter_func.py +3 -56
  188. mindspore/ops/function/random_func.py +243 -33
  189. mindspore/ops/function/sparse_unary_func.py +1 -1
  190. mindspore/ops/functional.py +18 -5
  191. mindspore/ops/functional_overload.py +897 -0
  192. mindspore/ops/operations/__init__.py +3 -2
  193. mindspore/ops/operations/_embedding_cache_ops.py +4 -4
  194. mindspore/ops/operations/_grad_ops.py +2 -34
  195. mindspore/ops/operations/_infer_ops.py +2 -1
  196. mindspore/ops/operations/_inner_ops.py +38 -8
  197. mindspore/ops/operations/array_ops.py +45 -303
  198. mindspore/ops/operations/comm_ops.py +19 -16
  199. mindspore/ops/operations/custom_ops.py +11 -55
  200. mindspore/ops/operations/debug_ops.py +42 -47
  201. mindspore/ops/operations/inner_ops.py +6 -4
  202. mindspore/ops/operations/linalg_ops.py +3 -2
  203. mindspore/ops/operations/manually_defined/ops_def.py +185 -104
  204. mindspore/ops/operations/math_ops.py +11 -216
  205. mindspore/ops/operations/nn_ops.py +146 -308
  206. mindspore/ops/primitive.py +23 -21
  207. mindspore/ops/tensor_method.py +1669 -0
  208. mindspore/ops_generate/aclnn_kernel_register_auto_cc_generator.py +110 -0
  209. mindspore/ops_generate/add_tensor_docs_generator.py +54 -0
  210. mindspore/ops_generate/arg_handler.py +0 -61
  211. mindspore/ops_generate/auto_grad_impl_cc_generator.py +135 -0
  212. mindspore/ops_generate/auto_grad_reg_cc_generator.py +93 -0
  213. mindspore/ops_generate/base_generator.py +11 -0
  214. mindspore/ops_generate/cpp_create_prim_instance_helper_generator.py +108 -0
  215. mindspore/ops_generate/functional_map_cpp_generator.py +491 -0
  216. mindspore/ops_generate/functional_overload_py_generator.py +110 -0
  217. mindspore/ops_generate/functions_cc_generator.py +233 -0
  218. mindspore/ops_generate/gen_aclnn_implement.py +110 -114
  219. mindspore/ops_generate/gen_constants.py +157 -3
  220. mindspore/ops_generate/gen_ops.py +245 -990
  221. mindspore/ops_generate/gen_pyboost_func.py +97 -998
  222. mindspore/ops_generate/gen_utils.py +119 -33
  223. mindspore/ops_generate/lite_ops_cpp_generator.py +155 -0
  224. mindspore/ops_generate/op_api_proto.py +206 -0
  225. mindspore/ops_generate/op_def_py_generator.py +131 -0
  226. mindspore/ops_generate/op_prim_py_generator.py +480 -0
  227. mindspore/ops_generate/op_proto.py +373 -108
  228. mindspore/ops_generate/op_template_parser.py +436 -0
  229. mindspore/ops_generate/ops_def_cc_generator.py +288 -0
  230. mindspore/ops_generate/ops_def_h_generator.py +74 -0
  231. mindspore/ops_generate/ops_name_h_generator.py +68 -0
  232. mindspore/ops_generate/ops_primitive_h_generator.py +81 -0
  233. mindspore/ops_generate/pyboost_functions_cpp_generator.py +370 -0
  234. mindspore/ops_generate/pyboost_functions_h_generator.py +68 -0
  235. mindspore/ops_generate/pyboost_functions_py_generator.py +148 -0
  236. mindspore/ops_generate/pyboost_grad_function_cpp_generator.py +154 -0
  237. mindspore/ops_generate/pyboost_inner_prim_generator.py +131 -0
  238. mindspore/ops_generate/pyboost_native_grad_functions_generator.py +268 -0
  239. mindspore/ops_generate/pyboost_op_cpp_code_generator.py +851 -0
  240. mindspore/ops_generate/pyboost_overload_functions_cpp_generator.py +344 -0
  241. mindspore/ops_generate/pyboost_utils.py +92 -33
  242. mindspore/ops_generate/template.py +294 -44
  243. mindspore/ops_generate/tensor_func_reg_cpp_generator.py +422 -0
  244. mindspore/parallel/__init__.py +3 -3
  245. mindspore/parallel/_auto_parallel_context.py +24 -33
  246. mindspore/parallel/_parallel_serialization.py +13 -2
  247. mindspore/parallel/_utils.py +4 -1
  248. mindspore/parallel/algo_parameter_config.py +1 -1
  249. mindspore/parallel/checkpoint_transform.py +44 -0
  250. mindspore/parallel/cluster/process_entity/_api.py +131 -37
  251. mindspore/parallel/cluster/process_entity/_utils.py +41 -6
  252. mindspore/parallel/cluster/run.py +20 -3
  253. mindspore/parallel/parameter_broadcast.py +1 -1
  254. mindspore/parallel/shard.py +3 -0
  255. mindspore/parallel/transform_safetensors.py +119 -253
  256. mindspore/profiler/__init__.py +17 -4
  257. mindspore/profiler/analysis/__init__.py +0 -0
  258. mindspore/profiler/analysis/parser/__init__.py +0 -0
  259. mindspore/profiler/analysis/parser/ascend_cann_parser.py +166 -0
  260. mindspore/profiler/analysis/parser/base_parser.py +158 -0
  261. mindspore/profiler/analysis/parser/framework_cann_relation_parser.py +45 -0
  262. mindspore/profiler/analysis/parser/ms_framework_parser.py +142 -0
  263. mindspore/profiler/analysis/parser/ms_minddata_parser.py +145 -0
  264. mindspore/profiler/analysis/parser/timeline_assembly_factory/__init__.py +0 -0
  265. mindspore/profiler/analysis/parser/timeline_assembly_factory/ascend_timeline_assembler.py +261 -0
  266. mindspore/profiler/analysis/parser/timeline_assembly_factory/base_timeline_assembler.py +40 -0
  267. mindspore/profiler/analysis/parser/timeline_assembly_factory/trace_view_container.py +84 -0
  268. mindspore/profiler/analysis/parser/timeline_creator/__init__.py +0 -0
  269. mindspore/profiler/analysis/parser/timeline_creator/base_timeline_creator.py +44 -0
  270. mindspore/profiler/analysis/parser/timeline_creator/cpu_op_timeline_creator.py +90 -0
  271. mindspore/profiler/analysis/parser/timeline_creator/fwk_timeline_creator.py +76 -0
  272. mindspore/profiler/analysis/parser/timeline_creator/msprof_timeline_creator.py +103 -0
  273. mindspore/profiler/analysis/parser/timeline_creator/scope_layer_timeline_creator.py +134 -0
  274. mindspore/profiler/analysis/parser/timeline_event/__init__.py +0 -0
  275. mindspore/profiler/analysis/parser/timeline_event/base_event.py +233 -0
  276. mindspore/profiler/analysis/parser/timeline_event/cpu_op_event.py +47 -0
  277. mindspore/profiler/analysis/parser/timeline_event/flow_event.py +36 -0
  278. mindspore/profiler/analysis/parser/timeline_event/fwk_event.py +260 -0
  279. mindspore/profiler/analysis/parser/timeline_event/msprof_event.py +73 -0
  280. mindspore/profiler/analysis/parser/timeline_event/scope_layer_event.py +53 -0
  281. mindspore/profiler/analysis/parser/timeline_event/timeline_event_pool.py +146 -0
  282. mindspore/profiler/analysis/task_manager.py +131 -0
  283. mindspore/profiler/analysis/time_converter.py +84 -0
  284. mindspore/profiler/analysis/viewer/__init__.py +0 -0
  285. mindspore/profiler/analysis/viewer/ascend_communication_viewer.py +333 -0
  286. mindspore/profiler/analysis/viewer/ascend_integrate_viewer.py +87 -0
  287. mindspore/profiler/analysis/viewer/ascend_kernel_details_viewer.py +252 -0
  288. mindspore/profiler/analysis/viewer/ascend_memory_viewer.py +313 -0
  289. mindspore/profiler/analysis/viewer/ascend_op_memory_viewer.py +322 -0
  290. mindspore/profiler/analysis/viewer/ascend_step_trace_time_viewer.py +265 -0
  291. mindspore/profiler/analysis/viewer/ascend_timeline_viewer.py +58 -0
  292. mindspore/profiler/analysis/viewer/base_viewer.py +26 -0
  293. mindspore/profiler/analysis/viewer/ms_dataset_viewer.py +97 -0
  294. mindspore/profiler/analysis/viewer/ms_minddata_viewer.py +581 -0
  295. mindspore/profiler/analysis/work_flow.py +73 -0
  296. mindspore/profiler/common/ascend_msprof_exporter.py +138 -0
  297. mindspore/profiler/common/command_executor.py +90 -0
  298. mindspore/profiler/common/constant.py +174 -3
  299. mindspore/profiler/common/file_manager.py +208 -0
  300. mindspore/profiler/common/log.py +130 -0
  301. mindspore/profiler/common/msprof_cmd_tool.py +202 -0
  302. mindspore/profiler/common/path_manager.py +371 -0
  303. mindspore/profiler/common/process_bar.py +168 -0
  304. mindspore/profiler/common/process_pool.py +9 -3
  305. mindspore/profiler/common/profiler_context.py +476 -0
  306. mindspore/profiler/common/profiler_info.py +304 -0
  307. mindspore/profiler/common/profiler_output_path.py +284 -0
  308. mindspore/profiler/common/profiler_parameters.py +210 -0
  309. mindspore/profiler/common/profiler_path_manager.py +120 -0
  310. mindspore/profiler/common/record_function.py +76 -0
  311. mindspore/profiler/common/tlv_decoder.py +76 -0
  312. mindspore/profiler/common/util.py +75 -2
  313. mindspore/profiler/dynamic_profiler.py +270 -37
  314. mindspore/profiler/envprofiler.py +138 -0
  315. mindspore/profiler/mstx.py +199 -0
  316. mindspore/profiler/platform/__init__.py +21 -0
  317. mindspore/profiler/platform/base_profiler.py +40 -0
  318. mindspore/profiler/platform/cpu_profiler.py +124 -0
  319. mindspore/profiler/platform/gpu_profiler.py +74 -0
  320. mindspore/profiler/platform/npu_profiler.py +309 -0
  321. mindspore/profiler/profiler.py +580 -93
  322. mindspore/profiler/profiler_action_controller.py +187 -0
  323. mindspore/profiler/profiler_interface.py +114 -0
  324. mindspore/profiler/schedule.py +208 -0
  325. mindspore/rewrite/api/symbol_tree.py +1 -2
  326. mindspore/run_check/_check_version.py +2 -6
  327. mindspore/runtime/__init__.py +37 -0
  328. mindspore/runtime/device.py +27 -0
  329. mindspore/runtime/event.py +209 -0
  330. mindspore/runtime/executor.py +148 -0
  331. mindspore/runtime/memory.py +392 -0
  332. mindspore/runtime/stream.py +460 -0
  333. mindspore/runtime/thread_bind_core.py +401 -0
  334. mindspore/swresample-4.dll +0 -0
  335. mindspore/swscale-6.dll +0 -0
  336. mindspore/tinyxml2.dll +0 -0
  337. mindspore/train/__init__.py +2 -2
  338. mindspore/train/_utils.py +53 -18
  339. mindspore/train/amp.py +8 -4
  340. mindspore/train/callback/_checkpoint.py +32 -18
  341. mindspore/train/callback/_early_stop.py +1 -1
  342. mindspore/train/callback/_flops_collector.py +105 -69
  343. mindspore/train/callback/_history.py +1 -1
  344. mindspore/train/callback/_summary_collector.py +44 -6
  345. mindspore/train/callback/_tft_register.py +31 -10
  346. mindspore/train/dataset_helper.py +11 -11
  347. mindspore/train/metrics/precision.py +4 -5
  348. mindspore/train/mind_ir_pb2.py +167 -46
  349. mindspore/train/model.py +13 -15
  350. mindspore/train/serialization.py +462 -76
  351. mindspore/train/summary/summary_record.py +1 -2
  352. mindspore/train/train_thor/model_thor.py +1 -1
  353. mindspore/turbojpeg.dll +0 -0
  354. mindspore/utils/__init__.py +4 -2
  355. mindspore/utils/dryrun.py +138 -0
  356. mindspore/utils/runtime_execution_order_check.py +550 -0
  357. mindspore/version.py +1 -1
  358. {mindspore-2.4.10.dist-info → mindspore-2.5.0.dist-info}/METADATA +2 -3
  359. {mindspore-2.4.10.dist-info → mindspore-2.5.0.dist-info}/RECORD +362 -238
  360. {mindspore-2.4.10.dist-info → mindspore-2.5.0.dist-info}/entry_points.txt +1 -1
  361. mindspore/common/_tensor_overload.py +0 -139
  362. mindspore/mindspore_np_dtype.dll +0 -0
  363. mindspore/profiler/envprofiling.py +0 -254
  364. mindspore/profiler/profiling.py +0 -1926
  365. {mindspore-2.4.10.dist-info → mindspore-2.5.0.dist-info}/WHEEL +0 -0
  366. {mindspore-2.4.10.dist-info → mindspore-2.5.0.dist-info}/top_level.txt +0 -0
@@ -16,6 +16,7 @@
16
16
  import os
17
17
  import sys
18
18
  import time
19
+ import stat
19
20
  import json
20
21
  import atexit
21
22
  import struct
@@ -24,11 +25,13 @@ import multiprocessing
24
25
 
25
26
  from mindspore import log as logger
26
27
  from mindspore.train import Callback
27
- from mindspore.profiler import Profiler
28
- from mindspore.profiler import ProfilerLevel
28
+ from mindspore.profiler import Profiler, tensor_board_trace_handler, schedule
29
29
  from mindspore.communication import get_rank
30
30
  from mindspore.profiler.parser.ascend_analysis.file_manager import FileManager
31
31
  from mindspore.profiler.parser.ascend_analysis.path_manager import PathManager
32
+ from mindspore.profiler.profiler_interface import ProfilerInterface
33
+ from mindspore.profiler.common.constant import ProfilerActivity, ProfilerLevel, AicoreMetrics
34
+ from mindspore.profiler.common.util import no_exception_func
32
35
 
33
36
 
34
37
  def get_real_rank():
@@ -48,17 +51,16 @@ class DynamicProfilerArgs:
48
51
  """
49
52
  Data class for dynamic profile config.
50
53
  """
51
- FMT = "iiiiii?????"
54
+ FMT = "iiiiii????"
52
55
  SIZE = struct.calcsize(FMT)
53
56
 
54
57
  def __init__(self,
55
58
  start_step: int = -1,
56
59
  stop_step: int = -1,
57
60
  aicore_metrics: int = -1,
58
- profiler_level: int = -1,
59
- profile_framework: int = -1,
61
+ profiler_level: int = 0,
60
62
  analyse_mode: int = -1,
61
- profile_communication: bool = False,
63
+ activities: int = 0,
62
64
  parallel_strategy: bool = False,
63
65
  with_stack: bool = False,
64
66
  data_simplification: bool = True,
@@ -68,9 +70,8 @@ class DynamicProfilerArgs:
68
70
  self._stop_step = stop_step
69
71
  self._aicore_metrics = aicore_metrics
70
72
  self._profiler_level = profiler_level
71
- self._profile_framework = profile_framework
72
73
  self._analyse_mode = analyse_mode
73
- self._profile_communication = profile_communication
74
+ self._activities = activities
74
75
  self._parallel_strategy = parallel_strategy
75
76
  self._with_stack = with_stack
76
77
  self._data_simplification = data_simplification
@@ -92,24 +93,16 @@ class DynamicProfilerArgs:
92
93
  self._aicore_metrics = -1
93
94
 
94
95
  if not isinstance(self._profiler_level, int):
95
- logger.warning("profiler_level should be int type, profiler_level will be reset to -1.")
96
- self._profiler_level = -1
97
-
98
- if not isinstance(self._profile_framework, int):
99
- logger.warning("profile_framework should be int type, profile_framework will be reset to -1.")
100
- self._profile_framework = -1
96
+ logger.warning("profiler_level should be int type, profiler_level will be reset to 0.")
97
+ self._profiler_level = 0
101
98
 
102
99
  if not isinstance(self._analyse_mode, int):
103
100
  logger.warning("analyse_mode should be int type, analyse_mode will be reset to -1.")
104
101
  self._analyse_mode = -1
105
102
 
106
- if not isinstance(self._profile_communication, bool):
107
- logger.warning("profile_communication should be bool type, profile_communication will be reset to False.")
108
- self._profile_communication = False
109
-
110
- if not isinstance(self._parallel_strategy, bool):
111
- logger.warning("parallel_strategy should be bool type, parallel_strategy will be reset to False.")
112
- self._parallel_strategy = False
103
+ if not isinstance(self._activities, int):
104
+ logger.warning("activities should be int type, activities will be reset to 0.")
105
+ self._activities = 0
113
106
 
114
107
  if not isinstance(self._with_stack, bool):
115
108
  logger.warning("with_stack should be bool type, with_stack will be reset to False.")
@@ -162,7 +155,8 @@ class DynamicProfilerArgs:
162
155
  def args(self):
163
156
  """ get all args in DynamicProfilerArgs."""
164
157
  self._profiler_level = self._convert_profiler_level(self._profiler_level)
165
- self._profile_framework = self._convert_profile_framework(self._profile_framework)
158
+ self._activities = self._convert_activities(self._activities)
159
+ self._aicore_metrics = self._convert_aicore_metrics(self._aicore_metrics)
166
160
  not_supported_args = ['_start_step', '_stop_step', '_analyse_mode', '_is_valid']
167
161
  res = {}
168
162
  for key, value in self.__dict__.items():
@@ -199,16 +193,37 @@ class DynamicProfilerArgs:
199
193
  return ProfilerLevel.Level1
200
194
  if profiler_level == 2:
201
195
  return ProfilerLevel.Level2
202
- return None
203
-
204
- def _convert_profile_framework(self, profile_framework: int) -> str:
205
- """ convert profile_framework to real args in Profiler."""
206
- if profile_framework == 0:
207
- return "time"
208
- if profile_framework == 1:
209
- return "all"
210
- return None
211
-
196
+ return ProfilerLevel.Level0
197
+
198
+ def _convert_activities(self, activities: int) -> ProfilerLevel:
199
+ """ convert activities to real args in Profiler."""
200
+ if activities == 0:
201
+ return [ProfilerActivity.CPU, ProfilerActivity.NPU]
202
+ if activities == 1:
203
+ return [ProfilerActivity.CPU]
204
+ if activities == 2:
205
+ return [ProfilerActivity.NPU]
206
+ return [ProfilerActivity.CPU, ProfilerActivity.NPU]
207
+
208
+ def _convert_aicore_metrics(self, aicore_metrics: int) -> AicoreMetrics:
209
+ """ convert aicore_metrics to real args in Profiler."""
210
+ if aicore_metrics == -1:
211
+ return AicoreMetrics.AiCoreNone
212
+ if aicore_metrics == 0:
213
+ return AicoreMetrics.PipeUtilization
214
+ if aicore_metrics == 1:
215
+ return AicoreMetrics.ArithmeticUtilization
216
+ if aicore_metrics == 2:
217
+ return AicoreMetrics.Memory
218
+ if aicore_metrics == 3:
219
+ return AicoreMetrics.MemoryL0
220
+ if aicore_metrics == 4:
221
+ return AicoreMetrics.MemoryUB
222
+ if aicore_metrics == 5:
223
+ return AicoreMetrics.ResourceConflictRatio
224
+ if aicore_metrics == 6:
225
+ return AicoreMetrics.L2Cache
226
+ return AicoreMetrics.AiCoreNone
212
227
 
213
228
  class DynamicProfilerMonitorBase(Callback):
214
229
  """
@@ -240,12 +255,17 @@ class DynamicProfilerMonitorBase(Callback):
240
255
  self._last_stop_step = None
241
256
  self._is_create_process = None
242
257
  self._is_started = False
258
+ self._start_step = -1
259
+ self._stop_step = -1
260
+ self._step_num = 0
243
261
 
262
+ self._check_shm_for_killed()
244
263
  self._init_cfg_json()
245
264
  self._create_shm()
246
265
  self._create_process()
247
266
  atexit.register(self._clean_resource)
248
267
 
268
+ @no_exception_func()
249
269
  def step_begin(self, run_context):
250
270
  """
251
271
  Start profile at the begin of step.
@@ -287,6 +307,7 @@ class DynamicProfilerMonitorBase(Callback):
287
307
  print_msg(f"Rank {self._rank_id} Dynamic profiler start at step {start_step}, "
288
308
  f"will stop at step {stop_step}")
289
309
 
310
+ @no_exception_func()
290
311
  def step_end(self, run_context):
291
312
  """
292
313
  Stop profile at the end of step.
@@ -312,11 +333,138 @@ class DynamicProfilerMonitorBase(Callback):
312
333
  if prof_args.analyse_mode:
313
334
  self._profiler.analyse(mode=prof_args.analyse_mode)
314
335
  else:
315
- self._profiler._ascend_profiler.finalize()
336
+ ProfilerInterface.finalize()
337
+ ProfilerInterface.clear()
316
338
  self._profiler = None
317
339
  self._is_started = False
318
340
  print_msg(f"Rank {self._rank_id} Dynamic profiler stop at step {step_num}")
319
341
 
342
+ @no_exception_func()
343
+ def step(self):
344
+ """
345
+ Used for Ascend, distinguish step collection and parsing performance data by dynamic profiler.
346
+
347
+ Raises:
348
+ RuntimeError: If the 'start_step' parameter setting is greater than the 'stop_step' parameter setting.
349
+
350
+ Examples:
351
+ >>> import json
352
+ >>> import os
353
+ >>> import numpy as np
354
+ >>>
355
+ >>> import mindspore
356
+ >>> import mindspore.dataset as ds
357
+ >>> from mindspore import context, nn
358
+ >>> from mindspore.profiler import DynamicProfilerMonitor
359
+ >>>
360
+ >>>
361
+ >>> class Net(nn.Cell):
362
+ ... def __init__(self):
363
+ ... super(Net, self).__init__()
364
+ ... self.fc = nn.Dense(2, 2)
365
+ ...
366
+ ... def construct(self, x):
367
+ ... return self.fc(x)
368
+ >>>
369
+ >>> def generator_net():
370
+ ... for _ in range(2):
371
+ ... yield np.ones([2, 2]).astype(np.float32), np.ones([2]).astype(np.int32)
372
+ >>>
373
+ >>> def train(test_net):
374
+ ... optimizer = nn.Momentum(test_net.trainable_params(), 1, 0.9)
375
+ ... loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
376
+ ... data = ds.GeneratorDataset(generator_net(), ["data", "label"])
377
+ ... model = mindspore.train.Model(test_net, loss, optimizer)
378
+ ... model.train(1, data)
379
+ >>>
380
+ >>> def change_cfg_json(json_path):
381
+ ... with open(json_path, 'r', encoding='utf-8') as file:
382
+ ... data = json.load(file)
383
+ ...
384
+ ... data['start_step'] = 6
385
+ ... data['stop_step'] = 7
386
+ ...
387
+ ... with open(json_path, 'w', encoding='utf-8') as file:
388
+ ... json.dump(data, file, ensure_ascii=False, indent=4)
389
+ >>>
390
+ >>> if __name__ == '__main__':
391
+ ... # set json configuration file
392
+ ... cfg_json = {
393
+ ... "start_step": 2,
394
+ ... "stop_step": 5,
395
+ ... "aicore_metrics": -1,
396
+ ... "profiler_level": 0,
397
+ ... "activities": 0,
398
+ ... "analyse_mode": -1,
399
+ ... "parallel_strategy": False,
400
+ ... "with_stack": False,
401
+ ... "data_simplification": True,
402
+ ... }
403
+ ... context.set_context(mode=mindspore.PYNATIVE_MODE)
404
+ ... mindspore.set_device("Ascend")
405
+ ... cfg_path = os.path.join("./cfg_path", "profiler_config.json")
406
+ ... # set cfg file
407
+ ... with open(cfg_path, 'w') as f:
408
+ ... json.dump(cfg_json, f, indent=4)
409
+ ... # Assume the user has correctly configured the environment variable (RANK_ID is not a non-numeric type)
410
+ ... rank_id = int(os.getenv('RANK_ID')) if os.getenv('RANK_ID') else 0
411
+ ... # cfg_path contains the json configuration file path, and output_path is the output path
412
+ ... dp = DynamicProfilerMonitor(cfg_path=cfg_path, output_path=cfg_path)
413
+ ... STEP_NUM = 15
414
+ ... # Define a network of training models
415
+ ... net = Net()
416
+ ... for i in range(STEP_NUM):
417
+ ... print(f"step {i}")
418
+ ... train(net)
419
+ ... # Modify the configuration file after step 7. For example, change start_step to 8 and stop_step to 10
420
+ ... if i == 7:
421
+ ... # Modify parameters in the JSON file
422
+ ... change_cfg_json(os.path.join(cfg_path, "profiler_config.json"))
423
+ ... # Call step collection
424
+ ... dp.step()
425
+ """
426
+
427
+ self._step_num += 1
428
+ prof_args = self._get_prof_args()
429
+
430
+ if not prof_args.is_valid:
431
+ logger.error("Dynamic profile json is not valid, please check the json file.")
432
+ return
433
+
434
+ if prof_args.start_step == -1 or prof_args.stop_step == -1:
435
+ return
436
+
437
+ # Skips the number of steps less than start_step
438
+ if self._step_num < prof_args.start_step:
439
+ return
440
+
441
+ if self._start_step != prof_args.start_step or self._stop_step != prof_args.stop_step:
442
+ # Update new start_step and stop_step
443
+ self._start_step = prof_args.start_step
444
+ self._stop_step = prof_args.stop_step
445
+ if self._start_step >= 0 and 0 <= self._start_step <= self._stop_step:
446
+ prof_path = os.path.join(self._output_path,
447
+ f"rank{self._rank_id}_start{self._start_step}_stop{self._stop_step}")
448
+ print_msg(f"Rank {self._rank_id} create output path {prof_path}")
449
+ print_msg(f"Rank {self._rank_id} Dynamic profile start at step {self._start_step}, "
450
+ f"will stop at step {self._stop_step}")
451
+ self._profiler = Profiler(output_path=prof_path,
452
+ schedule=schedule(wait=0, warmup=0,
453
+ active=self._stop_step - self._start_step + 1,
454
+ repeat=1,
455
+ skip_first=1),
456
+ on_trace_ready=tensor_board_trace_handler,
457
+ **prof_args.args)
458
+ else:
459
+ self._profiler = None
460
+ logger.error("Rank %d Dynamic profile start at step %d and stop at step %d in config_json must be "
461
+ "greater than or equal to 0, and stop step should not be less than start step",
462
+ self._rank_id, self._start_step, self._stop_step)
463
+
464
+ if self._profiler:
465
+ self._profiler.step()
466
+
467
+ @no_exception_func()
320
468
  def on_train_end(self, run_context):
321
469
  """
322
470
  Callback on trian end
@@ -357,6 +505,7 @@ class DynamicProfilerMonitorBase(Callback):
357
505
 
358
506
  return start_step, stop_step
359
507
 
508
+ @no_exception_func()
360
509
  def _init_cfg_json(self):
361
510
  """Init config json file"""
362
511
  if self._rank_id == 0:
@@ -372,6 +521,7 @@ class DynamicProfilerMonitorBase(Callback):
372
521
  """Create a json monitor process based on whether the SharedMemory is successfully created"""
373
522
  logger.error("Dynamic profiler _create_shm is not implemented")
374
523
 
524
+ @no_exception_func()
375
525
  def _create_process(self):
376
526
  """Create json monitor process, one process will be created at one worker"""
377
527
  if self._is_create_process:
@@ -385,18 +535,57 @@ class DynamicProfilerMonitorBase(Callback):
385
535
  self._process = None
386
536
  logger.info("Rank %d no need to create process.", self._rank_id)
387
537
 
538
+ @no_exception_func()
539
+ def _check_shm_for_killed(self):
540
+ """
541
+ User killed process shm can not clean normally, so check this when create shm.
542
+ """
543
+ if sys.version_info >= (3, 8):
544
+ shm_path = os.path.join("/dev/shm", self._shm_name)
545
+ else:
546
+ shm_path = self._shm_path
547
+
548
+ if not os.path.exists(shm_path):
549
+ return
550
+
551
+ MAX_TIME_DIFF = 30 # seconds
552
+ time_shm = os.stat(shm_path).st_ctime
553
+ cur_proc_time = self._get_pid_st_ctime(os.getpid())
554
+
555
+ if cur_proc_time and abs(cur_proc_time - time_shm) > MAX_TIME_DIFF:
556
+ raise RuntimeError("There maybe exist share memory before this task, if you kill last task, "
557
+ "dynamic profiler will not valid, please remove %s, and retry." % shm_path)
558
+
559
+ def _get_pid_st_ctime(self, pid):
560
+ """Get pid st_ctime"""
561
+ try:
562
+ fd = os.open("/proc/" + str(pid), os.O_RDONLY, stat.S_IRUSR | stat.S_IRGRP)
563
+ stat_ino = os.fstat(fd)
564
+ os.close(fd)
565
+ create_time = stat_ino.st_ctime
566
+ return create_time
567
+ except FileNotFoundError:
568
+ logger.error("Process with PID %d does not exist.", pid)
569
+ except PermissionError:
570
+ logger.error("Permission denied when accessing PID %d.", pid)
571
+ except Exception as ex: # pylint: disable=W0703
572
+ logger.error("An error occurred while getting creation time for PID %d: %s", pid, str(ex))
573
+
388
574
 
389
575
  if sys.version_info >= (3, 8):
576
+ @no_exception_func()
390
577
  def write_bytes(shm, byte_data):
391
578
  """Write bytes to shared memory"""
392
579
  shm.buf[:DynamicProfilerArgs.SIZE] = byte_data
393
580
  else:
581
+ @no_exception_func()
394
582
  def write_bytes(shm, byte_data):
395
583
  """Write bytes to shared memory"""
396
584
  shm.seek(0)
397
585
  shm.write(byte_data)
398
586
 
399
587
 
588
+ @no_exception_func()
400
589
  def worker_func(loop_flag, poll_interval, shm, cfg_path):
401
590
  """ Json monitor process worker function python version >= 3.8"""
402
591
  last_file_t = None
@@ -430,13 +619,45 @@ if sys.version_info >= (3, 8):
430
619
  from multiprocessing import shared_memory
431
620
  from unittest.mock import patch
432
621
 
622
+
433
623
  class DynamicProfilerMonitor(DynamicProfilerMonitorBase):
434
624
  r"""
435
625
  This class to enable the dynamic profile monitoring of MindSpore neural networks.
436
626
 
437
627
  Args:
438
628
  cfg_path (str): Dynamic profile json config file directory. The requirement is a shared path
439
- that can be accessed by all nodes.
629
+ that can be accessed by all nodes. The parameters of the json configuration file are as follows:
630
+
631
+ - start_step (int, required) - Sets the step number at which the Profiler starts collecting data.
632
+ It is a relative value, with the first step of training being 1. The default value is -1, indicating
633
+ that data collection will not start during the entire training process.
634
+ - stop_step (int, required) - Sets the step number at which the Profiler stops collecting data. It is
635
+ a relative value, with the first step of training being 1. The stop_step must be greater than or
636
+ equal to start_step. The default value is -1, indicating that data collection will not start during
637
+ the entire training process.
638
+ - aicore_metrics (int, optional) - The range of values corresponds to the Profiler. The default value -1
639
+ indicates that AI Core utilization is not collected, and 0 indicates PipeUtilization, 1 indicates
640
+ ArithmeticUtilization, 2 stands for Memory, 3 stands for MemoryL0, 4 stands for MemoryUB, 5 indicates
641
+ ResourceConflictRatio, 6 indicates L2Cache.
642
+ - profiler_level (int, optional) - Sets the level of performance data collection, where 0 represents
643
+ ProfilerLevel.Level0, 1 represents ProfilerLevel.Level1, and 2 represents ProfilerLevel.Level2. The
644
+ default value is 0, indicating the ProfilerLevel.Level0 collection level.
645
+ - activities (int, optional) - Sets the devices for performance data collection, where 0 represents
646
+ CPU+NPU, 1 represents CPU, and 2 represents NPU. The default value is 0, indicating the collection
647
+ of CPU+NPU performance data.
648
+ - analyse_mode (int, optional) - Sets the mode for online analysis, corresponding to the analyse_mode
649
+ parameter of the mindspore.Profiler.analyse interface, where 0 represents "sync" and 1 represents
650
+ "async". The default value is -1, indicating that online analysis is not used.
651
+ - parallel_strategy (bool, optional) - Sets whether to collect parallel strategy performance data,
652
+ where true means to collect and false means not to collect. The default value is false, indicating
653
+ that parallel strategy performance data is not collected.
654
+ - with_stack (bool, optional) - Sets whether to collect call stack information, where true means to
655
+ collect and false means not to collect. The default value is false, indicating that call stack
656
+ information is not collected.
657
+ - data_simplification (bool, optional) - Sets whether to enable data simplification, where true means
658
+ to enable and false means not to enable. The default value is true, indicating that data
659
+ simplification is enabled.
660
+
440
661
  output_path (str, optional): Output data path. Default: ``"./dyn_profile_data"`` .
441
662
  poll_interval (int, optional): The polling period of the monitoring process, in seconds.
442
663
  Default value: ``2``.
@@ -488,12 +709,14 @@ if sys.version_info >= (3, 8):
488
709
  """ Get prof_args py38"""
489
710
  return DynamicProfilerArgs.from_bytes(self._shm.buf[:DynamicProfilerArgs.SIZE])
490
711
 
712
+ @no_exception_func()
491
713
  def _clean_resource(self):
492
714
  """Clean resource py38"""
493
715
  # stop profiler when stop_step over all train step
494
716
  if self._profiler:
495
717
  self._profiler.stop()
496
- self._profiler._ascend_profiler.finalize()
718
+ ProfilerInterface.finalize()
719
+ ProfilerInterface.clear()
497
720
  self._profiler = None
498
721
  logger.warning("Rank %d Dynamic profiler stop at end of training", self._rank_id)
499
722
 
@@ -514,6 +737,7 @@ if sys.version_info >= (3, 8):
514
737
  logger.warning("Rank %s unlink shm failed, may be removed", self._rank_id)
515
738
  self._shm = None
516
739
 
740
+ @no_exception_func()
517
741
  def _create_shm(self):
518
742
  """Create a json monitor process based on whether the SharedMemory is successfully created py38"""
519
743
  try_times = 10
@@ -542,13 +766,19 @@ if sys.version_info >= (3, 8):
542
766
  logger.warning("Rank %d shared memory create failed, "
543
767
  "retry times = %d.", self._rank_id, try_times)
544
768
  time.sleep(random.uniform(0, 0.02)) # sleep 0 ~ 20 ms
769
+ except Exception as e: # pylint: disable=W0703
770
+ # shm open failed because of other process create shm not finished
771
+ try_times -= 1
772
+ logger.warning("Rank %d shared memory open failed, error: %s, retry times = %d",
773
+ self._rank_id, str(e), try_times)
774
+ time.sleep(random.uniform(0, 0.02)) # sleep 0 ~ 20 ms
545
775
 
546
776
  if try_times <= 0:
547
777
  raise RuntimeError(f"Rank {self._rank_id} failed to create shared memory.")
548
778
 
549
779
  else:
550
780
  import mmap
551
- import stat
781
+
552
782
 
553
783
  class DynamicProfilerMonitor(DynamicProfilerMonitorBase):
554
784
  r"""
@@ -617,12 +847,14 @@ else:
617
847
  self._shm.seek(0)
618
848
  return DynamicProfilerArgs.from_bytes(self._shm.read(DynamicProfilerArgs.SIZE))
619
849
 
850
+ @no_exception_func()
620
851
  def _clean_resource(self):
621
852
  """Clean resource py37"""
622
853
  # stop profiler when stop_step over all train step
623
854
  if self._profiler:
624
855
  self._profiler.stop()
625
- self._profiler._ascend_profiler.finalize()
856
+ ProfilerInterface.finalize()
857
+ ProfilerInterface.clear()
626
858
  self._profiler = None
627
859
  logger.warning("Rank %d Dynamic profiler stop at end of training", self._rank_id)
628
860
 
@@ -647,6 +879,7 @@ else:
647
879
  logger.warning("Rank %s unlink shm failed, may be removed", self._rank_id)
648
880
  self._shm = None
649
881
 
882
+ @no_exception_func()
650
883
  def _create_shm(self):
651
884
  """Create a json monitor process based on whether the SharedMemory is successfully created py37"""
652
885
 
@@ -0,0 +1,138 @@
1
+ # Copyright 2024 Huawei Technologies Co., Ltd
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ # ============================================================================
15
+ """EnvProfiler"""
16
+ import os
17
+ import json
18
+ from mindspore import log as logger
19
+ from mindspore.profiler.profiler import Profiler
20
+ from mindspore.profiler.common.constant import (
21
+ ProfilerLevel,
22
+ AicoreMetrics,
23
+ ProfilerActivity,
24
+ )
25
+ from mindspore.profiler.common.profiler_parameters import ProfilerParameters
26
+
27
+
28
+ class EnvProfiler:
29
+ """Collect and analyze training performance data, support calls during and after training."""
30
+
31
+ NOT_SUPPORTED_PARAMS = ["schedule", "on_trace_ready"]
32
+ profiler = None
33
+
34
+ @classmethod
35
+ def init_profiler(cls):
36
+ """
37
+ Initialize the profiler.
38
+ """
39
+ if not os.getenv("MS_PROFILER_OPTIONS"):
40
+ return
41
+ options = cls._load_options()
42
+ if not options:
43
+ logger.error("Failed to load MS_PROFILER_OPTIONS, json decode error.")
44
+ return
45
+
46
+ params = cls._convert_options_to_profiler_params(options)
47
+ logger.info(f"params: {params}")
48
+ if params["start_profile"]:
49
+ cls.profiler = Profiler(**params)
50
+ logger.info("Profiler init success.")
51
+
52
+ def analyse(self):
53
+ """
54
+ Analyze the collected data.
55
+ """
56
+ logger.info("analyse start")
57
+ if not self.profiler:
58
+ logger.info("Profiler is not initialized, skip analyse.")
59
+ return
60
+ self.profiler.analyse()
61
+ logger.info("analyse end")
62
+
63
+ @classmethod
64
+ def _load_options(cls):
65
+ """
66
+ Load the options from the environment variable.
67
+ """
68
+ try:
69
+ options = json.loads(os.environ.get("MS_PROFILER_OPTIONS", "{}"))
70
+ except json.JSONDecodeError:
71
+ return {}
72
+ return options
73
+
74
+ @classmethod
75
+ def _convert_option_to_enum_value(cls, enum_class, option_value, default_value):
76
+ """
77
+ Convert the option value to the enum value.
78
+ """
79
+ try:
80
+ return enum_class(option_value)
81
+ except ValueError:
82
+ logger.warning(
83
+ f"The value '{option_value}' of parameter '{enum_class.__name__}' is invalid, "
84
+ f"use default value '{default_value}' instead."
85
+ )
86
+ return default_value
87
+
88
+ @classmethod
89
+ def _convert_options_to_profiler_params(cls, options):
90
+ """
91
+ Convert the options to the profiler parameters.
92
+ """
93
+ params = {}
94
+ if not options:
95
+ logger.warning("MS_PROFILER_OPTIONS is empty, use default values.")
96
+ return params
97
+
98
+ if "output_path" in options:
99
+ params["output_path"] = options["output_path"]
100
+
101
+ # if start is not set, default is False
102
+ params["start_profile"] = options.get("start", False)
103
+
104
+ for param, (_, default_value) in ProfilerParameters.PARAMS.items():
105
+ if param in options and param not in cls.NOT_SUPPORTED_PARAMS:
106
+ if param == "activities" and isinstance(options[param], list):
107
+ params[param] = cls._convert_activities_to_list(
108
+ options[param], default_value
109
+ )
110
+ elif param == "aicore_metrics":
111
+ params[param] = cls._convert_option_to_enum_value(
112
+ AicoreMetrics, options[param], default_value
113
+ )
114
+ elif param == "profiler_level":
115
+ params[param] = cls._convert_option_to_enum_value(
116
+ ProfilerLevel, options[param], default_value
117
+ )
118
+ else:
119
+ params[param] = options[param]
120
+ return params
121
+
122
+ @classmethod
123
+ def _convert_activities_to_list(cls, activities, default_value):
124
+ """
125
+ Convert the activities to the list.
126
+ """
127
+ res = []
128
+ for activity in activities:
129
+ res.append(
130
+ cls._convert_option_to_enum_value(
131
+ ProfilerActivity, activity, default_value
132
+ )
133
+ )
134
+ # remove duplicate
135
+ return list(set(default_value if default_value in res else res))
136
+
137
+
138
+ EnvProfiler.init_profiler()