mindspore 2.4.1__cp311-cp311-win_amd64.whl → 2.5.0__cp311-cp311-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mindspore might be problematic. Click here for more details.

Files changed (395) hide show
  1. mindspore/.commit_id +1 -1
  2. mindspore/Microsoft.VisualStudio.Telemetry.dll +0 -0
  3. mindspore/Newtonsoft.Json.dll +0 -0
  4. mindspore/__init__.py +8 -3
  5. mindspore/_c_dataengine.cp311-win_amd64.pyd +0 -0
  6. mindspore/_c_expression.cp311-win_amd64.pyd +0 -0
  7. mindspore/_c_mindrecord.cp311-win_amd64.pyd +0 -0
  8. mindspore/_checkparam.py +0 -5
  9. mindspore/_extends/parallel_compile/akg_compiler/gen_custom_op_files.py +1 -1
  10. mindspore/_extends/parse/compile_config.py +64 -0
  11. mindspore/_extends/parse/deprecated/__init__.py +0 -0
  12. mindspore/_extends/parse/deprecated/deprecated_tensor_method.py +375 -0
  13. mindspore/_extends/parse/parser.py +23 -5
  14. mindspore/_extends/parse/standard_method.py +123 -27
  15. mindspore/_extends/pijit/pijit_func_white_list.py +1 -1
  16. mindspore/amp.py +7 -1
  17. mindspore/atlprov.dll +0 -0
  18. mindspore/avcodec-59.dll +0 -0
  19. mindspore/avdevice-59.dll +0 -0
  20. mindspore/avfilter-8.dll +0 -0
  21. mindspore/avformat-59.dll +0 -0
  22. mindspore/avutil-57.dll +0 -0
  23. mindspore/boost/boost_cell_wrapper.py +136 -41
  24. mindspore/c1.dll +0 -0
  25. mindspore/c1xx.dll +0 -0
  26. mindspore/c2.dll +0 -0
  27. mindspore/common/__init__.py +3 -1
  28. mindspore/common/_register_for_tensor.py +0 -1
  29. mindspore/common/_stub_tensor.py +25 -4
  30. mindspore/common/_tensor_cpp_method.py +17 -0
  31. mindspore/common/_tensor_docs.py +6132 -0
  32. mindspore/common/api.py +99 -25
  33. mindspore/common/dtype.py +34 -34
  34. mindspore/common/dump.py +2 -1
  35. mindspore/common/file_system.py +8 -1
  36. mindspore/common/generator.py +2 -0
  37. mindspore/common/hook_handle.py +3 -1
  38. mindspore/common/initializer.py +3 -4
  39. mindspore/common/lazy_inline.py +8 -2
  40. mindspore/common/mindir_util.py +10 -2
  41. mindspore/common/parameter.py +30 -27
  42. mindspore/common/tensor.py +713 -1337
  43. mindspore/communication/__init__.py +1 -1
  44. mindspore/communication/_comm_helper.py +10 -0
  45. mindspore/communication/comm_func.py +215 -173
  46. mindspore/communication/management.py +23 -20
  47. mindspore/context.py +292 -193
  48. mindspore/dataset/__init__.py +23 -19
  49. mindspore/dataset/callback/ds_callback.py +2 -1
  50. mindspore/dataset/core/config.py +84 -3
  51. mindspore/dataset/engine/cache_admin.py +3 -3
  52. mindspore/dataset/engine/cache_client.py +5 -4
  53. mindspore/dataset/engine/datasets.py +192 -149
  54. mindspore/dataset/engine/datasets_audio.py +14 -0
  55. mindspore/dataset/engine/datasets_standard_format.py +28 -11
  56. mindspore/dataset/engine/datasets_text.py +38 -1
  57. mindspore/dataset/engine/datasets_user_defined.py +125 -65
  58. mindspore/dataset/engine/datasets_vision.py +81 -8
  59. mindspore/dataset/engine/iterators.py +281 -63
  60. mindspore/dataset/engine/obs/util.py +8 -0
  61. mindspore/dataset/engine/queue.py +40 -0
  62. mindspore/dataset/engine/samplers.py +26 -2
  63. mindspore/dataset/engine/serializer_deserializer.py +1 -1
  64. mindspore/dataset/engine/validators.py +43 -11
  65. mindspore/dataset/transforms/py_transforms_util.py +17 -0
  66. mindspore/dataset/transforms/transforms.py +29 -12
  67. mindspore/dataset/vision/validators.py +1 -2
  68. mindspore/device_context/__init__.py +21 -0
  69. mindspore/device_context/ascend/__init__.py +25 -0
  70. mindspore/device_context/ascend/device.py +72 -0
  71. mindspore/device_context/ascend/op_debug.py +94 -0
  72. mindspore/device_context/ascend/op_precision.py +193 -0
  73. mindspore/device_context/ascend/op_tuning.py +127 -0
  74. mindspore/device_context/cpu/__init__.py +25 -0
  75. mindspore/device_context/cpu/device.py +62 -0
  76. mindspore/device_context/cpu/op_tuning.py +43 -0
  77. mindspore/device_context/gpu/__init__.py +21 -0
  78. mindspore/device_context/gpu/device.py +70 -0
  79. mindspore/device_context/gpu/op_precision.py +67 -0
  80. mindspore/device_context/gpu/op_tuning.py +175 -0
  81. mindspore/device_manager.py +134 -0
  82. mindspore/dnnl.dll +0 -0
  83. mindspore/dpcmi.dll +0 -0
  84. mindspore/experimental/llm_boost/__init__.py +3 -2
  85. mindspore/experimental/llm_boost/ascend_native/__init__.py +22 -0
  86. mindspore/experimental/llm_boost/ascend_native/llama_boost_ascend_native.py +211 -0
  87. mindspore/experimental/llm_boost/ascend_native/llm_boost.py +52 -0
  88. mindspore/experimental/llm_boost/atb/boost_base.py +239 -64
  89. mindspore/experimental/llm_boost/atb/llama_boost.py +52 -30
  90. mindspore/experimental/llm_boost/atb/qwen_boost.py +47 -24
  91. mindspore/experimental/llm_boost/register.py +1 -0
  92. mindspore/experimental/optim/adadelta.py +26 -22
  93. mindspore/experimental/optim/adam.py +3 -0
  94. mindspore/experimental/optim/lr_scheduler.py +33 -24
  95. mindspore/experimental/optim/radam.py +33 -30
  96. mindspore/hal/device.py +28 -0
  97. mindspore/hal/event.py +17 -0
  98. mindspore/hal/memory.py +94 -3
  99. mindspore/hal/stream.py +91 -6
  100. mindspore/include/api/context.h +1 -2
  101. mindspore/include/dataset/constants.h +2 -2
  102. mindspore/jpeg62.dll +0 -0
  103. mindspore/log.py +12 -0
  104. mindspore/mindrecord/__init__.py +1 -1
  105. mindspore/mindrecord/config.py +17 -316
  106. mindspore/mindrecord/filereader.py +1 -9
  107. mindspore/mindrecord/filewriter.py +5 -15
  108. mindspore/mindrecord/mindpage.py +1 -9
  109. mindspore/mindspore_backend.dll +0 -0
  110. mindspore/mindspore_common.dll +0 -0
  111. mindspore/mindspore_core.dll +0 -0
  112. mindspore/mindspore_glog.dll +0 -0
  113. mindspore/mindspore_ops.dll +0 -0
  114. mindspore/mint/__init__.py +824 -218
  115. mindspore/mint/distributed/__init__.py +66 -4
  116. mindspore/mint/distributed/distributed.py +2594 -44
  117. mindspore/mint/linalg/__init__.py +6 -0
  118. mindspore/mint/nn/__init__.py +473 -14
  119. mindspore/mint/nn/functional.py +486 -11
  120. mindspore/mint/nn/layer/__init__.py +17 -4
  121. mindspore/mint/nn/layer/_functions.py +330 -0
  122. mindspore/mint/nn/layer/activation.py +169 -1
  123. mindspore/mint/nn/layer/basic.py +123 -0
  124. mindspore/mint/nn/layer/conv.py +727 -0
  125. mindspore/mint/nn/layer/normalization.py +215 -19
  126. mindspore/mint/nn/layer/padding.py +797 -0
  127. mindspore/mint/nn/layer/pooling.py +170 -0
  128. mindspore/mint/optim/__init__.py +2 -1
  129. mindspore/mint/optim/adam.py +223 -0
  130. mindspore/mint/optim/adamw.py +26 -19
  131. mindspore/mint/special/__init__.py +2 -1
  132. mindspore/msobj140.dll +0 -0
  133. mindspore/mspdb140.dll +0 -0
  134. mindspore/mspdbcore.dll +0 -0
  135. mindspore/mspdbst.dll +0 -0
  136. mindspore/mspft140.dll +0 -0
  137. mindspore/msvcdis140.dll +0 -0
  138. mindspore/msvcp140_1.dll +0 -0
  139. mindspore/msvcp140_2.dll +0 -0
  140. mindspore/msvcp140_atomic_wait.dll +0 -0
  141. mindspore/msvcp140_codecvt_ids.dll +0 -0
  142. mindspore/multiprocessing/__init__.py +5 -0
  143. mindspore/nn/__init__.py +2 -0
  144. mindspore/nn/cell.py +142 -21
  145. mindspore/nn/dynamic_lr.py +2 -1
  146. mindspore/nn/layer/activation.py +6 -6
  147. mindspore/nn/layer/basic.py +35 -25
  148. mindspore/nn/layer/channel_shuffle.py +3 -3
  149. mindspore/nn/layer/conv.py +3 -0
  150. mindspore/nn/layer/embedding.py +3 -3
  151. mindspore/nn/layer/normalization.py +8 -7
  152. mindspore/nn/layer/padding.py +4 -3
  153. mindspore/nn/layer/pooling.py +55 -23
  154. mindspore/nn/layer/rnn_cells.py +1 -1
  155. mindspore/nn/layer/rnns.py +2 -1
  156. mindspore/nn/layer/timedistributed.py +5 -5
  157. mindspore/nn/layer/transformer.py +48 -26
  158. mindspore/nn/learning_rate_schedule.py +5 -3
  159. mindspore/nn/loss/loss.py +31 -36
  160. mindspore/nn/optim/ada_grad.py +1 -0
  161. mindspore/nn/optim/adadelta.py +2 -2
  162. mindspore/nn/optim/adam.py +1 -1
  163. mindspore/nn/optim/lars.py +1 -4
  164. mindspore/nn/optim/optimizer.py +1 -1
  165. mindspore/nn/optim/rprop.py +2 -2
  166. mindspore/nn/optim/thor.py +2 -1
  167. mindspore/nn/utils/__init__.py +22 -0
  168. mindspore/nn/utils/init.py +73 -0
  169. mindspore/nn/wrap/cell_wrapper.py +4 -6
  170. mindspore/nn/wrap/loss_scale.py +3 -4
  171. mindspore/numpy/array_creations.py +60 -62
  172. mindspore/numpy/array_ops.py +148 -143
  173. mindspore/numpy/logic_ops.py +41 -42
  174. mindspore/numpy/math_ops.py +361 -359
  175. mindspore/numpy/utils.py +16 -16
  176. mindspore/numpy/utils_const.py +4 -4
  177. mindspore/opencv_core452.dll +0 -0
  178. mindspore/opencv_imgcodecs452.dll +0 -0
  179. mindspore/opencv_imgproc452.dll +0 -0
  180. mindspore/ops/__init__.py +2 -1
  181. mindspore/ops/_grad_experimental/grad_comm_ops.py +107 -8
  182. mindspore/ops/_grad_experimental/grad_debug_ops.py +6 -1
  183. mindspore/ops/_grad_experimental/grad_inner_ops.py +9 -0
  184. mindspore/ops/_grad_experimental/grad_math_ops.py +2 -1
  185. mindspore/ops/_op_impl/cpu/__init__.py +1 -0
  186. mindspore/ops/_op_impl/cpu/raise_op.py +28 -0
  187. mindspore/ops/_vmap/vmap_array_ops.py +20 -19
  188. mindspore/ops/_vmap/vmap_base.py +0 -2
  189. mindspore/ops/_vmap/vmap_grad_nn_ops.py +19 -13
  190. mindspore/ops/_vmap/vmap_math_ops.py +11 -9
  191. mindspore/ops/_vmap/vmap_nn_ops.py +20 -34
  192. mindspore/ops/auto_generate/cpp_create_prim_instance_helper.py +149 -12
  193. mindspore/ops/auto_generate/gen_arg_handler.py +0 -61
  194. mindspore/ops/auto_generate/gen_extend_func.py +554 -60
  195. mindspore/ops/auto_generate/gen_ops_def.py +1621 -115
  196. mindspore/ops/auto_generate/gen_ops_prim.py +8027 -3411
  197. mindspore/ops/auto_generate/pyboost_inner_prim.py +183 -79
  198. mindspore/ops/composite/base.py +1 -1
  199. mindspore/ops/composite/multitype_ops/_compile_utils.py +229 -30
  200. mindspore/ops/composite/multitype_ops/pow_impl.py +0 -29
  201. mindspore/ops/function/__init__.py +12 -0
  202. mindspore/ops/function/array_func.py +561 -159
  203. mindspore/ops/function/clip_func.py +64 -0
  204. mindspore/ops/function/debug_func.py +28 -20
  205. mindspore/ops/function/image_func.py +1 -1
  206. mindspore/ops/function/linalg_func.py +5 -4
  207. mindspore/ops/function/math_func.py +1664 -294
  208. mindspore/ops/function/nn_func.py +988 -317
  209. mindspore/ops/function/parameter_func.py +3 -56
  210. mindspore/ops/function/random_func.py +243 -33
  211. mindspore/ops/function/sparse_unary_func.py +1 -1
  212. mindspore/ops/functional.py +18 -5
  213. mindspore/ops/functional_overload.py +897 -0
  214. mindspore/ops/operations/__init__.py +3 -2
  215. mindspore/ops/operations/_embedding_cache_ops.py +4 -4
  216. mindspore/ops/operations/_grad_ops.py +2 -34
  217. mindspore/ops/operations/_infer_ops.py +2 -1
  218. mindspore/ops/operations/_inner_ops.py +38 -8
  219. mindspore/ops/operations/array_ops.py +45 -303
  220. mindspore/ops/operations/comm_ops.py +23 -17
  221. mindspore/ops/operations/custom_ops.py +7 -49
  222. mindspore/ops/operations/debug_ops.py +42 -47
  223. mindspore/ops/operations/inner_ops.py +6 -4
  224. mindspore/ops/operations/linalg_ops.py +3 -2
  225. mindspore/ops/operations/manually_defined/ops_def.py +185 -104
  226. mindspore/ops/operations/math_ops.py +11 -216
  227. mindspore/ops/operations/nn_ops.py +153 -310
  228. mindspore/ops/primitive.py +23 -21
  229. mindspore/ops/tensor_method.py +1669 -0
  230. mindspore/ops_generate/aclnn_kernel_register_auto_cc_generator.py +110 -0
  231. mindspore/ops_generate/add_tensor_docs_generator.py +54 -0
  232. mindspore/ops_generate/arg_handler.py +0 -61
  233. mindspore/ops_generate/auto_grad_impl_cc_generator.py +135 -0
  234. mindspore/ops_generate/auto_grad_reg_cc_generator.py +93 -0
  235. mindspore/ops_generate/base_generator.py +11 -0
  236. mindspore/ops_generate/cpp_create_prim_instance_helper_generator.py +108 -0
  237. mindspore/ops_generate/functional_map_cpp_generator.py +491 -0
  238. mindspore/ops_generate/functional_overload_py_generator.py +110 -0
  239. mindspore/ops_generate/functions_cc_generator.py +233 -0
  240. mindspore/ops_generate/gen_aclnn_implement.py +110 -114
  241. mindspore/ops_generate/gen_constants.py +157 -3
  242. mindspore/ops_generate/gen_ops.py +245 -990
  243. mindspore/ops_generate/gen_pyboost_func.py +97 -998
  244. mindspore/ops_generate/gen_utils.py +119 -33
  245. mindspore/ops_generate/lite_ops_cpp_generator.py +155 -0
  246. mindspore/ops_generate/op_api_proto.py +206 -0
  247. mindspore/ops_generate/op_def_py_generator.py +131 -0
  248. mindspore/ops_generate/op_prim_py_generator.py +480 -0
  249. mindspore/ops_generate/op_proto.py +373 -108
  250. mindspore/ops_generate/op_template_parser.py +436 -0
  251. mindspore/ops_generate/ops_def_cc_generator.py +288 -0
  252. mindspore/ops_generate/ops_def_h_generator.py +74 -0
  253. mindspore/ops_generate/ops_name_h_generator.py +68 -0
  254. mindspore/ops_generate/ops_primitive_h_generator.py +81 -0
  255. mindspore/ops_generate/pyboost_functions_cpp_generator.py +370 -0
  256. mindspore/ops_generate/pyboost_functions_h_generator.py +68 -0
  257. mindspore/ops_generate/pyboost_functions_py_generator.py +148 -0
  258. mindspore/ops_generate/pyboost_grad_function_cpp_generator.py +154 -0
  259. mindspore/ops_generate/pyboost_inner_prim_generator.py +131 -0
  260. mindspore/ops_generate/pyboost_native_grad_functions_generator.py +268 -0
  261. mindspore/ops_generate/pyboost_op_cpp_code_generator.py +851 -0
  262. mindspore/ops_generate/pyboost_overload_functions_cpp_generator.py +344 -0
  263. mindspore/ops_generate/pyboost_utils.py +92 -33
  264. mindspore/ops_generate/template.py +294 -44
  265. mindspore/ops_generate/tensor_func_reg_cpp_generator.py +422 -0
  266. mindspore/parallel/__init__.py +3 -3
  267. mindspore/parallel/_auto_parallel_context.py +44 -34
  268. mindspore/parallel/_cell_wrapper.py +22 -3
  269. mindspore/parallel/_parallel_serialization.py +13 -2
  270. mindspore/parallel/_utils.py +4 -2
  271. mindspore/parallel/algo_parameter_config.py +1 -1
  272. mindspore/parallel/checkpoint_transform.py +44 -0
  273. mindspore/parallel/cluster/process_entity/_api.py +131 -37
  274. mindspore/parallel/cluster/process_entity/_utils.py +41 -6
  275. mindspore/parallel/cluster/run.py +20 -3
  276. mindspore/parallel/parameter_broadcast.py +1 -1
  277. mindspore/parallel/shard.py +3 -0
  278. mindspore/parallel/transform_safetensors.py +119 -253
  279. mindspore/pgodb140.dll +0 -0
  280. mindspore/pgort140.dll +0 -0
  281. mindspore/profiler/__init__.py +17 -4
  282. mindspore/profiler/analysis/__init__.py +0 -0
  283. mindspore/profiler/analysis/parser/__init__.py +0 -0
  284. mindspore/profiler/analysis/parser/ascend_cann_parser.py +166 -0
  285. mindspore/profiler/analysis/parser/base_parser.py +158 -0
  286. mindspore/profiler/analysis/parser/framework_cann_relation_parser.py +45 -0
  287. mindspore/profiler/analysis/parser/ms_framework_parser.py +142 -0
  288. mindspore/profiler/analysis/parser/ms_minddata_parser.py +145 -0
  289. mindspore/profiler/analysis/parser/timeline_assembly_factory/__init__.py +0 -0
  290. mindspore/profiler/analysis/parser/timeline_assembly_factory/ascend_timeline_assembler.py +261 -0
  291. mindspore/profiler/analysis/parser/timeline_assembly_factory/base_timeline_assembler.py +40 -0
  292. mindspore/profiler/analysis/parser/timeline_assembly_factory/trace_view_container.py +84 -0
  293. mindspore/profiler/analysis/parser/timeline_creator/__init__.py +0 -0
  294. mindspore/profiler/analysis/parser/timeline_creator/base_timeline_creator.py +44 -0
  295. mindspore/profiler/analysis/parser/timeline_creator/cpu_op_timeline_creator.py +90 -0
  296. mindspore/profiler/analysis/parser/timeline_creator/fwk_timeline_creator.py +76 -0
  297. mindspore/profiler/analysis/parser/timeline_creator/msprof_timeline_creator.py +103 -0
  298. mindspore/profiler/analysis/parser/timeline_creator/scope_layer_timeline_creator.py +134 -0
  299. mindspore/profiler/analysis/parser/timeline_event/__init__.py +0 -0
  300. mindspore/profiler/analysis/parser/timeline_event/base_event.py +233 -0
  301. mindspore/profiler/analysis/parser/timeline_event/cpu_op_event.py +47 -0
  302. mindspore/profiler/analysis/parser/timeline_event/flow_event.py +36 -0
  303. mindspore/profiler/analysis/parser/timeline_event/fwk_event.py +260 -0
  304. mindspore/profiler/analysis/parser/timeline_event/msprof_event.py +73 -0
  305. mindspore/profiler/analysis/parser/timeline_event/scope_layer_event.py +53 -0
  306. mindspore/profiler/analysis/parser/timeline_event/timeline_event_pool.py +146 -0
  307. mindspore/profiler/analysis/task_manager.py +131 -0
  308. mindspore/profiler/analysis/time_converter.py +84 -0
  309. mindspore/profiler/analysis/viewer/__init__.py +0 -0
  310. mindspore/profiler/analysis/viewer/ascend_communication_viewer.py +333 -0
  311. mindspore/profiler/analysis/viewer/ascend_integrate_viewer.py +87 -0
  312. mindspore/profiler/analysis/viewer/ascend_kernel_details_viewer.py +252 -0
  313. mindspore/profiler/analysis/viewer/ascend_memory_viewer.py +313 -0
  314. mindspore/profiler/analysis/viewer/ascend_op_memory_viewer.py +322 -0
  315. mindspore/profiler/analysis/viewer/ascend_step_trace_time_viewer.py +265 -0
  316. mindspore/profiler/analysis/viewer/ascend_timeline_viewer.py +58 -0
  317. mindspore/profiler/analysis/viewer/base_viewer.py +26 -0
  318. mindspore/profiler/analysis/viewer/ms_dataset_viewer.py +97 -0
  319. mindspore/profiler/analysis/viewer/ms_minddata_viewer.py +581 -0
  320. mindspore/profiler/analysis/work_flow.py +73 -0
  321. mindspore/profiler/common/ascend_msprof_exporter.py +138 -0
  322. mindspore/profiler/common/command_executor.py +90 -0
  323. mindspore/profiler/common/constant.py +174 -3
  324. mindspore/profiler/common/file_manager.py +208 -0
  325. mindspore/profiler/common/log.py +130 -0
  326. mindspore/profiler/common/msprof_cmd_tool.py +202 -0
  327. mindspore/profiler/common/path_manager.py +371 -0
  328. mindspore/profiler/common/process_bar.py +168 -0
  329. mindspore/profiler/common/process_pool.py +9 -3
  330. mindspore/profiler/common/profiler_context.py +476 -0
  331. mindspore/profiler/common/profiler_info.py +304 -0
  332. mindspore/profiler/common/profiler_output_path.py +284 -0
  333. mindspore/profiler/common/profiler_parameters.py +210 -0
  334. mindspore/profiler/common/profiler_path_manager.py +120 -0
  335. mindspore/profiler/common/record_function.py +76 -0
  336. mindspore/profiler/common/tlv_decoder.py +76 -0
  337. mindspore/profiler/common/util.py +75 -2
  338. mindspore/profiler/dynamic_profiler.py +270 -37
  339. mindspore/profiler/envprofiler.py +138 -0
  340. mindspore/profiler/mstx.py +199 -0
  341. mindspore/profiler/platform/__init__.py +21 -0
  342. mindspore/profiler/platform/base_profiler.py +40 -0
  343. mindspore/profiler/platform/cpu_profiler.py +124 -0
  344. mindspore/profiler/platform/gpu_profiler.py +74 -0
  345. mindspore/profiler/platform/npu_profiler.py +309 -0
  346. mindspore/profiler/profiler.py +580 -93
  347. mindspore/profiler/profiler_action_controller.py +187 -0
  348. mindspore/profiler/profiler_interface.py +114 -0
  349. mindspore/profiler/schedule.py +208 -0
  350. mindspore/rewrite/api/symbol_tree.py +1 -2
  351. mindspore/run_check/_check_version.py +18 -13
  352. mindspore/runtime/__init__.py +37 -0
  353. mindspore/runtime/device.py +27 -0
  354. mindspore/runtime/event.py +209 -0
  355. mindspore/runtime/executor.py +148 -0
  356. mindspore/runtime/memory.py +392 -0
  357. mindspore/runtime/stream.py +460 -0
  358. mindspore/runtime/thread_bind_core.py +401 -0
  359. mindspore/swresample-4.dll +0 -0
  360. mindspore/swscale-6.dll +0 -0
  361. mindspore/tbbmalloc.dll +0 -0
  362. mindspore/tinyxml2.dll +0 -0
  363. mindspore/train/__init__.py +2 -2
  364. mindspore/train/_utils.py +53 -18
  365. mindspore/train/amp.py +8 -4
  366. mindspore/train/callback/_checkpoint.py +32 -18
  367. mindspore/train/callback/_early_stop.py +1 -1
  368. mindspore/train/callback/_flops_collector.py +105 -69
  369. mindspore/train/callback/_history.py +1 -1
  370. mindspore/train/callback/_summary_collector.py +44 -6
  371. mindspore/train/callback/_tft_register.py +37 -15
  372. mindspore/train/dataset_helper.py +11 -11
  373. mindspore/train/metrics/precision.py +4 -5
  374. mindspore/train/mind_ir_pb2.py +167 -46
  375. mindspore/train/model.py +13 -14
  376. mindspore/train/serialization.py +461 -72
  377. mindspore/train/summary/summary_record.py +1 -2
  378. mindspore/train/train_thor/model_thor.py +1 -1
  379. mindspore/turbojpeg.dll +0 -0
  380. mindspore/utils/__init__.py +4 -2
  381. mindspore/utils/dryrun.py +138 -0
  382. mindspore/utils/runtime_execution_order_check.py +550 -0
  383. mindspore/vcmeta.dll +0 -0
  384. mindspore/vcruntime140.dll +0 -0
  385. mindspore/vcruntime140_1.dll +0 -0
  386. mindspore/version.py +1 -1
  387. {mindspore-2.4.1.dist-info → mindspore-2.5.0.dist-info}/METADATA +3 -4
  388. {mindspore-2.4.1.dist-info → mindspore-2.5.0.dist-info}/RECORD +391 -265
  389. {mindspore-2.4.1.dist-info → mindspore-2.5.0.dist-info}/entry_points.txt +1 -1
  390. mindspore/common/_tensor_overload.py +0 -139
  391. mindspore/mindspore_np_dtype.dll +0 -0
  392. mindspore/profiler/envprofiling.py +0 -254
  393. mindspore/profiler/profiling.py +0 -1926
  394. {mindspore-2.4.1.dist-info → mindspore-2.5.0.dist-info}/WHEEL +0 -0
  395. {mindspore-2.4.1.dist-info → mindspore-2.5.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,166 @@
1
+ # Copyright 2024 Huawei Technologies Co., Ltd
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ # ============================================================================
15
+ """Parser for Ascend CANN profiling data."""
16
+ import os
17
+ import glob
18
+ from typing import Dict, Optional
19
+
20
+ import numpy as np
21
+
22
+ from mindspore import log as logger
23
+ from mindspore.profiler.analysis.parser.base_parser import BaseParser
24
+ from mindspore.profiler.common.file_manager import FileManager
25
+ from mindspore.profiler.common.path_manager import PathManager
26
+ from mindspore.profiler.common.constant import ProfilerLevel
27
+ from mindspore.profiler.common.ascend_msprof_exporter import AscendMsprofExporter
28
+ from mindspore.profiler.common.log import ProfilerLogger
29
+
30
+
31
+ class AscendMsprofParser(BaseParser):
32
+ """Parser for MindSpore profiling data on Ascend platform."""
33
+
34
+ _MSPROF_TIMELINE_FILE_PATTERN = ["msprof_[0-9]*.json", "msprof_slice_*.json"]
35
+ _OP_SUMMARY_FILE_PATTERN = "op_summary_*.csv"
36
+ OVERSIZE_MB = 1024
37
+
38
+ def __init__(self, next_parser: Optional[BaseParser] = None, **kwargs):
39
+ """Initialize AscendMsprofParser."""
40
+ super().__init__(next_parser)
41
+ self._kwargs = kwargs
42
+ self._msprof_profile_output_path = self._kwargs.get(
43
+ "msprof_profile_output_path"
44
+ )
45
+ self._msprof_profile_host_path = self._kwargs.get(
46
+ "msprof_profile_host_path"
47
+ )
48
+ self._msprof_profile_device_path = self._kwargs.get(
49
+ "msprof_profile_device_path"
50
+ )
51
+ self._ascend_ms_dir = self._kwargs.get("ascend_ms_dir")
52
+ self._profiler_level = kwargs.get("profiler_level")
53
+ ProfilerLogger.init(self._ascend_ms_dir)
54
+ self._logger = ProfilerLogger.get_instance()
55
+ self.op_summary = None
56
+ self.op_summary_headers = None
57
+ self.msprof_timeline = []
58
+
59
+ def _parse(self, data=None) -> Dict:
60
+ """Parse profiling data and update the input dictionary.
61
+
62
+ Args:
63
+ data (Dict, optional): Input data dictionary. Defaults to None.
64
+
65
+ Returns:
66
+ Dict: Updated data with op_summary, headers and timeline information.
67
+ """
68
+ if data is None:
69
+ data = {}
70
+ self._check_msprof_data_size()
71
+ AscendMsprofExporter(**self._kwargs).export()
72
+ self._logger.info("AscendMsprofExporter export done.")
73
+ self._parse_op_summary()
74
+ self._logger.info("AscendMsprofParser parse op summary done.")
75
+ self._parse_msprof_timeline()
76
+ self._logger.info("AscendMsprofParser parse msprof timeline done.")
77
+ data.update(
78
+ {
79
+ "op_summary": self.op_summary,
80
+ "op_summary_headers": self.op_summary_headers,
81
+ "msprof_timeline": self.msprof_timeline,
82
+ }
83
+ )
84
+ return data
85
+
86
+ def _parse_op_summary(self):
87
+ """Parse operation summary data from CSV files.
88
+
89
+ Raises:
90
+ RuntimeError: If no op summary files are found or read file failed.
91
+ """
92
+ if self._profiler_level == ProfilerLevel.LevelNone.value:
93
+ return
94
+ file_path_list = glob.glob(
95
+ os.path.join(
96
+ self._msprof_profile_output_path, self._OP_SUMMARY_FILE_PATTERN
97
+ )
98
+ )
99
+ if not file_path_list:
100
+ logger.error(
101
+ f"Failed to find op_summary_*.csv in directory: {self._msprof_profile_output_path}"
102
+ )
103
+ return
104
+
105
+ for file_path in file_path_list:
106
+ csv_data_np, self.op_summary_headers = FileManager.read_csv_file_as_numpy(
107
+ file_path=file_path, extern_headers=["Step ID"]
108
+ )
109
+ if self.op_summary is None:
110
+ self.op_summary = csv_data_np
111
+ else:
112
+ self.op_summary = np.concatenate([self.op_summary, csv_data_np], axis=0)
113
+
114
+ def _parse_msprof_timeline(self) -> None:
115
+ """Parse msprof timeline data from JSON files.
116
+
117
+ Raises:
118
+ RuntimeError: If no msprof JSON files are found in the specified directory.
119
+ """
120
+ file_path_list = []
121
+ for pattern in self._MSPROF_TIMELINE_FILE_PATTERN:
122
+ file_path_list.extend(glob.glob(os.path.join(self._msprof_profile_output_path, pattern)))
123
+
124
+ if not file_path_list:
125
+ raise RuntimeError(
126
+ f"Failed to find msprof JSON files in directory: {self._msprof_profile_output_path}"
127
+ )
128
+ for file_path in file_path_list:
129
+ self._process_timeline_file(file_path)
130
+ if not self.msprof_timeline:
131
+ logger.error(
132
+ "Failed to collect msprof timeline data"
133
+ )
134
+
135
+ def _process_timeline_file(self, file_path: str) -> None:
136
+ """Process a single timeline JSON file.
137
+
138
+ Args:
139
+ file_path (str): Path to the JSON file to process.
140
+ """
141
+ try:
142
+ timeline_data = FileManager.read_json_file(file_path)
143
+ if timeline_data:
144
+ self.msprof_timeline.extend(timeline_data)
145
+ else:
146
+ logger.warning(
147
+ f"Msporf timeline data in {file_path} is empty."
148
+ )
149
+ except RuntimeError as err:
150
+ logger.error(
151
+ f"Failed to read file: {file_path}. Error: {str(err)}"
152
+ )
153
+
154
+ def _check_msprof_data_size(self) -> None:
155
+ """Check the size of profiling data."""
156
+ host_data_size = PathManager.get_directory_size(self._msprof_profile_host_path)
157
+ device_data_size = PathManager.get_directory_size(self._msprof_profile_device_path)
158
+ total_size = host_data_size + device_data_size
159
+
160
+ if total_size >= self.OVERSIZE_MB:
161
+ msg = (
162
+ f"The size of profiling data is too large: {total_size} MB, "
163
+ "which may spend a lot of time on analysing, you can consider "
164
+ "reducing the number of collection steps"
165
+ )
166
+ logger.warning(msg)
@@ -0,0 +1,158 @@
1
+ # Copyright 2024 Huawei Technologies Co., Ltd
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ # ============================================================================
15
+ """Base parser"""
16
+ from typing import (
17
+ Any,
18
+ List,
19
+ Callable,
20
+ Optional,
21
+ )
22
+ from multiprocessing import Process
23
+ from mindspore import log as logger
24
+ from mindspore.profiler.common.log import ProfilerLogger
25
+
26
+
27
+ class BaseParser:
28
+ """
29
+ Base class for all parsers in the workflow.
30
+ """
31
+ EXEC_HOOK_TIMEOUT = 30 * 60
32
+
33
+ def __init__(self, next_parser: Optional["BaseParser"] = None):
34
+ """
35
+ Initialize the BaseParser.
36
+
37
+ Args:
38
+ next_parser (Optional[BaseParser]): The next parser in the chain.
39
+ """
40
+ self.next_parser: Optional["BaseParser"] = next_parser
41
+ self._post_hooks: List[Callable[[Any], None]] = []
42
+ self._logger = ProfilerLogger.get_instance()
43
+
44
+ def set_next(self, next_parser: "BaseParser") -> "BaseParser":
45
+ """
46
+ Set the next parser in the chain.
47
+
48
+ Args:
49
+ next_parser (BaseParser): The next parser to be set.
50
+
51
+ Returns:
52
+ BaseParser: The next parser that was set.
53
+ """
54
+ if not isinstance(next_parser, BaseParser):
55
+ raise ValueError(
56
+ f"next_parser {next_parser.__class__.__name__} must be a BaseParser"
57
+ )
58
+ self.next_parser = next_parser
59
+ return self
60
+
61
+ def parse(self, data: Any) -> Any:
62
+ """
63
+ Parse the input data and execute post-hooks.
64
+
65
+ Args:
66
+ data (Any): The input data to be parsed.
67
+
68
+ Returns:
69
+ Any: The parsed result.
70
+ """
71
+ try:
72
+ result = self._parse(data)
73
+ self._execute_post_hooks(result)
74
+ except Exception as e: # pylint: disable=W0703
75
+ logger.error("Parser [%s] error: %s", self.__class__.__name__, str(e))
76
+ self._logger.error("Parser [%s] error: %s", self.__class__.__name__, str(e), exc_info=True)
77
+ return data
78
+ return result
79
+
80
+ def register_post_hook(self, hook: Callable[[Any], None]) -> "BaseParser":
81
+ """
82
+ Register a post-hook to be executed after parsing.
83
+
84
+ Args:
85
+ hook (Callable[[Any], None]): The hook function to be registered.
86
+
87
+ Returns:
88
+ BaseParser: The current parser instance.
89
+
90
+ Raises:
91
+ ValueError: If the hook is not callable.
92
+ """
93
+ if callable(hook):
94
+ self._post_hooks.append(hook)
95
+ else:
96
+ raise ValueError("Hook must be callable")
97
+ return self
98
+
99
+ def _execute_post_hooks(self, res: Any) -> None:
100
+ """
101
+ Execute all registered post-hooks asynchronously.
102
+
103
+ Args:
104
+ res (Any): The result to be passed to the post-hooks.
105
+ """
106
+ if not self._post_hooks:
107
+ return
108
+
109
+ processes = []
110
+ for hook in self._post_hooks:
111
+ p = Process(target=hook, args=(res,))
112
+ p.start()
113
+ hook_class = hook.__self__.__class__.__name__ if hasattr(hook, '__self__') else 'Unknown'
114
+ hook_name = f"{hook_class}.{hook.__name__}"
115
+ processes.append((p, hook_name))
116
+ self._logger.info("Parser [%s] post hook [%s] start", self.__class__.__name__, hook_name)
117
+
118
+ for p, hook_name in processes:
119
+ try:
120
+ p.join(timeout=self.EXEC_HOOK_TIMEOUT)
121
+ if p.is_alive():
122
+ logger.error(
123
+ "Parser [%s] post hook [%s] timeout after %s seconds, terminating",
124
+ self.__class__.__name__, hook_name, self.EXEC_HOOK_TIMEOUT
125
+ )
126
+ p.terminate()
127
+ p.join()
128
+ else:
129
+ self._logger.info(
130
+ "Parser [%s] post hook [%s] completed",
131
+ self.__class__.__name__, hook_name
132
+ )
133
+ except Exception as e: # pylint: disable=W0703
134
+ self._logger.error(
135
+ "Parser [%s] post hook [%s] failed: %s",
136
+ self.__class__.__name__, hook_name, str(e), exc_info=True
137
+ )
138
+ if p.is_alive():
139
+ p.terminate()
140
+
141
+ def _parse(self, data: Any) -> Any:
142
+ """
143
+ Abstract method to be implemented by subclasses for parsing logic.
144
+
145
+ Args:
146
+ data (Any): The input data to be parsed.
147
+
148
+ Raises:
149
+ NotImplementedError: If not implemented by subclass.
150
+ """
151
+ raise NotImplementedError("Subclasses should implement this!")
152
+
153
+
154
+ class DummyParser(BaseParser):
155
+ """Dummy parser"""
156
+ def _parse(self, data: Any) -> Any:
157
+ """Dummy parse"""
158
+ return data
@@ -0,0 +1,45 @@
1
+ # Copyright 2024 Huawei Technologies Co., Ltd
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ # ============================================================================
15
+ """Parser for analyzing the relationship between framework and CANN data."""
16
+ from typing import Dict, Any
17
+
18
+ from mindspore.profiler.analysis.parser.base_parser import BaseParser
19
+ from mindspore.profiler.analysis.parser.timeline_assembly_factory.ascend_timeline_assembler import (
20
+ AscendTimelineAssembler
21
+ )
22
+ from mindspore.profiler.common.log import ProfilerLogger
23
+
24
+
25
+ class FrameworkCannRelationParser(BaseParser):
26
+ """FrameworkCannRelationParser"""
27
+
28
+ def __init__(self, **kwargs):
29
+ """Initialize the AscendTraceAnalyser."""
30
+ super().__init__()
31
+ self.assembler = AscendTimelineAssembler(**kwargs)
32
+ self._ascend_ms_dir = kwargs.get("ascend_ms_dir")
33
+ ProfilerLogger.init(kwargs.get("ascend_ms_dir"))
34
+ self._logger = ProfilerLogger.get_instance()
35
+
36
+ def _parse(self, data: Dict[str, Any]) -> Dict[str, Any]:
37
+ """Parse the relation of framework and cann data."""
38
+ self.assembler.assemble(data)
39
+ self._logger.info("FrameworkCannRelationParser assemble done")
40
+ trace_view_container = self.assembler.get_trace_view_container()
41
+ self._logger.info("FrameworkCannRelationParser get trace view container done")
42
+ data.update({
43
+ "trace_view_container": trace_view_container,
44
+ })
45
+ return data
@@ -0,0 +1,142 @@
1
+ # Copyright 2024 Huawei Technologies Co., Ltd
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ # ============================================================================
15
+ """Parser for MindSpore framework profiling data."""
16
+ import os
17
+ import struct
18
+ from typing import List, Dict, Any, Optional
19
+
20
+ from mindspore import log as logger
21
+ from mindspore.profiler.common.tlv_decoder import TLVDecoder
22
+ from mindspore.profiler.common.file_manager import FileManager
23
+ from mindspore.profiler.common.constant import ProfilerActivity, FileConstant, DeviceTarget
24
+ from mindspore.profiler.analysis.parser.base_parser import BaseParser
25
+ from mindspore.profiler.analysis.parser.timeline_event.fwk_event import FwkFixSizeFormat, OpRangeStructField
26
+ from mindspore.profiler.common.log import ProfilerLogger
27
+
28
+
29
+ class FrameworkParser(BaseParser):
30
+ """Parser for MindSpore framework profiling data."""
31
+
32
+ _OP_RANGE_FILE_NAME = "op_range_{}"
33
+ _CPU_OP_TIMESTAMP_FILE_NAME = "cpu_op_execute_timestamp_{}.txt"
34
+
35
+ def __init__(self, next_parser: Optional[BaseParser] = None, **kwargs):
36
+ """Initialize FrameworkParser."""
37
+ super().__init__(next_parser)
38
+ self._rank_id = kwargs.get("rank_id")
39
+ self._activities = kwargs.get("activities")
40
+ self._step_list = kwargs.get("step_list")
41
+ self._framework_path = kwargs.get("framework_path")
42
+ self._device_target = kwargs.get("device_target")
43
+ self._ascend_ms_dir = kwargs.get("ascend_ms_dir")
44
+ self._op_range_path = os.path.join(
45
+ self._framework_path,
46
+ self._OP_RANGE_FILE_NAME.format(self._rank_id)
47
+ )
48
+ self._cpu_op_path = os.path.join(
49
+ self._framework_path,
50
+ self._CPU_OP_TIMESTAMP_FILE_NAME.format(self._rank_id),
51
+ )
52
+ ProfilerLogger.init(self._ascend_ms_dir)
53
+ self._logger = ProfilerLogger.get_instance()
54
+
55
+ def _parse(self, data: Dict[str, Any]) -> Dict[str, Any]:
56
+ """Parse MindSpore framework profiling data.
57
+
58
+ Args:
59
+ data (Dict[str, Any]): Input data dictionary.
60
+
61
+ Returns:
62
+ Dict[str, Any]: Updated data with:
63
+ - mindspore_op_list: List of parsed operation events
64
+ - cpu_op_lines: List of CPU operation timestamp data
65
+ """
66
+ if ProfilerActivity.CPU.value not in self._activities:
67
+ return data
68
+ mindspore_op_list = self._parse_op_range_data()
69
+ self._logger.info("FrameworkParser parse op range done.")
70
+ cpu_op_lines = self._parse_cpu_op_data()
71
+ self._logger.info("FrameworkParser parse cpu op done.")
72
+ data.update(
73
+ {
74
+ "mindspore_op_list": mindspore_op_list,
75
+ "cpu_op_lines": cpu_op_lines,
76
+ }
77
+ )
78
+ return data
79
+
80
+ def _parse_op_range_data(self) -> List[Dict]:
81
+ """Read and decode MindSpore op_range data.
82
+
83
+ Returns:
84
+ List[Dict]: List of parsed MindSpore operation events.
85
+ """
86
+ if self._device_target == DeviceTarget.CPU.value:
87
+ return []
88
+
89
+ if not os.path.exists(self._op_range_path):
90
+ logger.error("Failed to find op_range data. Skipping parse host profiler data.")
91
+ return []
92
+
93
+ try:
94
+ op_range_bytes = FileManager.read_file_content(self._op_range_path, "rb")
95
+ op_range_list = TLVDecoder.decode(
96
+ op_range_bytes, FwkFixSizeFormat.OpRangeStruct, struct.calcsize(FwkFixSizeFormat.OpRangeStruct)
97
+ )
98
+ self._logger.info("FrameworkParser parse op range done, op_range_list length: %d", len(op_range_list))
99
+ if not op_range_list:
100
+ logger.error(
101
+ f"Failed to decode op_range data: empty result from file {self._op_range_path}"
102
+ )
103
+ return self._filter_op_range_list(op_range_list)
104
+ except RuntimeError as err:
105
+ logger.error(f"Failed to read file: {self._op_range_path}. Error: {str(err)}")
106
+ return []
107
+
108
+ def _filter_op_range_list(self, op_range_list: List[Dict]) -> List[Dict]:
109
+ """Filter op_range_list based on step_list.
110
+
111
+ Args:
112
+ op_range_list (List[Dict]): List of operation events to filter.
113
+
114
+ Returns:
115
+ List[Dict]: Filtered list of operation events.
116
+ """
117
+ if not self._step_list or not isinstance(self._step_list, list):
118
+ return op_range_list
119
+
120
+ first_step = min(
121
+ op[FileConstant.FIX_SIZE_DATA][OpRangeStructField.STEP_ID.value]
122
+ for op in op_range_list
123
+ )
124
+ adjusted_step_list = [step - 1 + first_step for step in self._step_list]
125
+ return [
126
+ op for op in op_range_list
127
+ if op[FileConstant.FIX_SIZE_DATA][OpRangeStructField.STEP_ID.value] in adjusted_step_list
128
+ ]
129
+
130
+ def _parse_cpu_op_data(self) -> List[str]:
131
+ """Parse CPU op timestamp data.
132
+
133
+ Returns:
134
+ List[str]: List of CPU operation timestamp data lines.
135
+ """
136
+ if not os.path.exists(self._cpu_op_path):
137
+ return []
138
+ try:
139
+ return FileManager.read_txt_file(self._cpu_op_path)
140
+ except RuntimeError as err:
141
+ logger.warning(f"Failed to read file: {self._cpu_op_path}. Error: {str(err)}")
142
+ return []
@@ -0,0 +1,145 @@
1
+ # Copyright 2024 Huawei Technologies Co., Ltd
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ # ============================================================================
15
+ """mind data parser"""
16
+ import os
17
+ from typing import Dict, Tuple, Optional, Any, List
18
+ from mindspore import log as logger
19
+ from mindspore.profiler.analysis.parser.base_parser import BaseParser
20
+ from mindspore.profiler.common.file_manager import FileManager
21
+ from mindspore.profiler.common.constant import ProfilerActivity
22
+ from mindspore.profiler.common.exceptions.exceptions import (
23
+ ProfilerPathErrorException,
24
+ ProfilerRawFileException
25
+ )
26
+ from mindspore.profiler.common.log import ProfilerLogger
27
+
28
+
29
+ class MindDataParser(BaseParser):
30
+ """
31
+ Parser for MindData profiling information.
32
+ """
33
+ _FILE_NAMES = {
34
+ 'pipeline': 'pipeline_profiling_{}.json',
35
+ 'cpu_utilization': 'minddata_cpu_utilization_{}.json',
36
+ 'device_queue': 'device_queue_profiling_{}.txt',
37
+ 'device_iterator': 'dataset_iterator_profiling_{}.txt'
38
+ }
39
+
40
+ def __init__(self, next_parser: Optional[BaseParser] = None, **kwargs):
41
+ super().__init__(next_parser)
42
+ self._device_id = kwargs.get("rank_id") if (ProfilerActivity.NPU.value in
43
+ kwargs.get("activities")) else kwargs.get("device_id")
44
+ self._output_path = kwargs.get("framework_path")
45
+ self._ascend_ms_dir = kwargs.get("ascend_ms_dir")
46
+ ProfilerLogger.init(self._ascend_ms_dir)
47
+ self._logger = ProfilerLogger.get_instance()
48
+ self._file_paths = self._setup_file_paths()
49
+
50
+ def _setup_file_paths(self) -> Dict[str, str]:
51
+ return {
52
+ key: os.path.join(self._output_path, file_name.format(self._device_id))
53
+ for key, file_name in self._FILE_NAMES.items()
54
+ }
55
+
56
+ def _parse(self, data=None) -> Dict[str, Any]:
57
+ if data is None:
58
+ data = {}
59
+ if not os.path.exists(self._file_paths['pipeline']):
60
+ logger.info(
61
+ "pipeline profiling file %s does not exist. Please check whether minddata profiling data should exist.",
62
+ self._file_paths['pipeline'])
63
+ return data
64
+ op_id_info, sample_interval = self._parse_pipeline_info_dict()
65
+ cpu_util_info = self._parse_cpu_util_info()
66
+ device_trace_info = self._parse_device_trace()
67
+ self._logger.info("MindDataParser parse done")
68
+ data.update({
69
+ "pipeline_info": (op_id_info, sample_interval),
70
+ "cpu_util_info": cpu_util_info,
71
+ "device_trace_info": device_trace_info,
72
+ "device_queue_file_found": self._device_queue_file_found
73
+ })
74
+ return data
75
+
76
+ def _parse_pipeline_info_dict(self) -> Tuple[Dict[str, Any], float]:
77
+ """
78
+ Parse the pipeline information into a dictionary.
79
+ Returns:
80
+ Tuple[Dict[str, Any], float]: A tuple containing the parsed op_info dictionary
81
+ and sampling interval.
82
+ Raises:
83
+ ProfilerRawFileException: If the format of the pipeline raw file is incorrect.
84
+ """
85
+ pipeline_info = FileManager.read_json_file(self._file_paths['pipeline'])
86
+ if not pipeline_info:
87
+ raise ProfilerRawFileException('The minddata pipeline file is empty.')
88
+
89
+ sample_interval = pipeline_info.get('sampling_interval')
90
+ op_info = pipeline_info.get('op_info')
91
+
92
+ if sample_interval is None or not op_info:
93
+ raise ProfilerRawFileException('The format of minddata pipeline raw file is wrong.')
94
+
95
+ dict_op_id_info = {
96
+ item['op_id']: item for item in op_info if item
97
+ }
98
+
99
+ if len(dict_op_id_info) != len(op_info):
100
+ raise ProfilerRawFileException('The content of minddata pipeline raw file is wrong.')
101
+
102
+ return dict_op_id_info, sample_interval
103
+
104
+ def _parse_cpu_util_info(self) -> Dict[str, Any]:
105
+ try:
106
+ cpu_util_info = FileManager.read_json_file(self._file_paths['cpu_utilization'])
107
+ if not cpu_util_info:
108
+ msg = f'The MindData CPU utilization file {self._file_paths["cpu_utilization"]} is empty.'
109
+ raise RuntimeError(msg)
110
+ except RuntimeError as err:
111
+ cpu_util_info = {}
112
+ logger.warning(f'Failed to read the MindData CPU utilization data. ERROR:{err}')
113
+ return cpu_util_info
114
+
115
+ def _parse_device_trace(self) -> List[Any]:
116
+ """parse the device trace data"""
117
+ try:
118
+ self._device_trace_path, self._device_queue_file_found = self._setup_device_trace()
119
+ device_trace = FileManager.read_txt_file(self._device_trace_path)
120
+ if not device_trace:
121
+ msg = f"The MindData trace profiling file {self._device_trace_path} is empty."
122
+ raise RuntimeError(msg)
123
+ except RuntimeError as err:
124
+ device_trace = []
125
+ logger.warning(f'Failed to read the MindData device trace data. ERROR:{err}')
126
+ return device_trace
127
+
128
+ def _setup_device_trace(self) -> Tuple[str, bool]:
129
+ """
130
+ Set up the device trace file path.
131
+ Returns:
132
+ Tuple[str, bool]: A tuple containing the device trace file path and a boolean
133
+ indicating whether the device queue file was found.
134
+ Raises:
135
+ ProfilerPathErrorException: If no device trace file is found.
136
+ """
137
+ queue_path = self._file_paths['device_queue']
138
+ iterator_path = self._file_paths['device_iterator']
139
+
140
+ if os.path.exists(queue_path):
141
+ return queue_path, True
142
+ if os.path.exists(iterator_path):
143
+ return iterator_path, False
144
+
145
+ raise ProfilerPathErrorException('A MindData device trace profiling file cannot be found.')