mindspore 2.6.0rc1__cp39-cp39-win_amd64.whl → 2.7.0rc1__cp39-cp39-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mindspore might be problematic. Click here for more details.

Files changed (384) hide show
  1. mindspore/.commit_id +1 -1
  2. mindspore/__init__.py +1 -1
  3. mindspore/_c_dataengine.cp39-win_amd64.pyd +0 -0
  4. mindspore/_c_expression.cp39-win_amd64.pyd +0 -0
  5. mindspore/_c_mindrecord.cp39-win_amd64.pyd +0 -0
  6. mindspore/_checkparam.py +40 -9
  7. mindspore/{_deprecated → _extends/optimize}/__init__.py +9 -3
  8. mindspore/_extends/optimize/cell_utils.py +96 -0
  9. mindspore/_extends/parse/__init__.py +2 -2
  10. mindspore/_extends/parse/compile_config.py +44 -22
  11. mindspore/_extends/parse/deprecated/deprecated_tensor_method.py +1 -1
  12. mindspore/_extends/parse/parser.py +37 -62
  13. mindspore/_extends/parse/resources.py +39 -0
  14. mindspore/_extends/parse/standard_method.py +43 -13
  15. mindspore/_extends/parse/trope.py +8 -1
  16. mindspore/_extends/pijit/__init__.py +1 -2
  17. mindspore/amp.py +4 -4
  18. mindspore/avcodec-59.dll +0 -0
  19. mindspore/avdevice-59.dll +0 -0
  20. mindspore/avfilter-8.dll +0 -0
  21. mindspore/avformat-59.dll +0 -0
  22. mindspore/avutil-57.dll +0 -0
  23. mindspore/boost/adasum.py +1 -1
  24. mindspore/boost/boost_cell_wrapper.py +4 -4
  25. mindspore/common/__init__.py +27 -2
  26. mindspore/common/_grad_function.py +2 -1
  27. mindspore/common/_pijit_context.py +28 -7
  28. mindspore/common/_stub_tensor.py +1 -209
  29. mindspore/common/_tensor_cpp_method.py +1 -1
  30. mindspore/common/_tensor_docs.py +77 -16
  31. mindspore/common/api.py +238 -113
  32. mindspore/common/dtype.py +21 -11
  33. mindspore/common/dump.py +10 -15
  34. mindspore/common/generator.py +5 -3
  35. mindspore/common/hook_handle.py +11 -2
  36. mindspore/common/jit_config.py +1 -1
  37. mindspore/common/jit_trace.py +84 -105
  38. mindspore/common/parameter.py +26 -12
  39. mindspore/common/recompute.py +3 -3
  40. mindspore/common/sparse_tensor.py +0 -3
  41. mindspore/common/symbol.py +0 -1
  42. mindspore/common/tensor.py +81 -81
  43. mindspore/communication/_comm_helper.py +46 -4
  44. mindspore/communication/management.py +79 -7
  45. mindspore/context.py +58 -40
  46. mindspore/dataset/core/config.py +3 -3
  47. mindspore/dataset/engine/datasets.py +20 -7
  48. mindspore/dataset/engine/datasets_user_defined.py +33 -3
  49. mindspore/dataset/engine/iterators.py +2 -2
  50. mindspore/dataset/engine/obs/config_loader.py +2 -2
  51. mindspore/dataset/engine/obs/obs_mindrecord_dataset.py +8 -0
  52. mindspore/dataset/transforms/py_transforms.py +7 -3
  53. mindspore/dataset/transforms/transforms.py +7 -3
  54. mindspore/dataset/vision/validators.py +1 -0
  55. mindspore/device_context/ascend/device.py +1 -1
  56. mindspore/device_context/gpu/__init__.py +2 -2
  57. mindspore/device_context/gpu/device.py +1 -1
  58. mindspore/device_context/gpu/op_precision.py +4 -2
  59. mindspore/device_context/gpu/op_tuning.py +6 -3
  60. mindspore/device_manager.py +16 -9
  61. mindspore/dnnl.dll +0 -0
  62. mindspore/experimental/llm_boost/ascend_native/llama_boost_ascend_native.py +3 -7
  63. mindspore/experimental/llm_boost/atb/boost_base.py +2 -3
  64. mindspore/experimental/optim/adadelta.py +13 -20
  65. mindspore/experimental/optim/adagrad.py +15 -22
  66. mindspore/experimental/optim/adam.py +17 -24
  67. mindspore/experimental/optim/adamax.py +14 -22
  68. mindspore/experimental/optim/adamw.py +28 -34
  69. mindspore/experimental/optim/asgd.py +15 -25
  70. mindspore/experimental/optim/lr_scheduler.py +27 -45
  71. mindspore/experimental/optim/nadam.py +14 -24
  72. mindspore/experimental/optim/optimizer.py +13 -23
  73. mindspore/experimental/optim/radam.py +18 -24
  74. mindspore/experimental/optim/rmsprop.py +14 -25
  75. mindspore/experimental/optim/rprop.py +15 -26
  76. mindspore/experimental/optim/sgd.py +9 -19
  77. mindspore/hal/__init__.py +4 -4
  78. mindspore/hal/contiguous_tensors_handle.py +2 -2
  79. mindspore/hal/memory.py +27 -7
  80. mindspore/include/api/cell.h +37 -1
  81. mindspore/include/api/delegate.h +10 -0
  82. mindspore/include/api/model.h +3 -0
  83. mindspore/include/api/types.h +2 -2
  84. mindspore/include/c_api/model_c.h +0 -58
  85. mindspore/include/c_api/tensor_c.h +0 -26
  86. mindspore/include/dataset/vision_ascend.h +1 -1
  87. mindspore/jpeg62.dll +0 -0
  88. mindspore/mindrecord/tools/cifar10.py +60 -11
  89. mindspore/mindrecord/tools/cifar10_to_mr.py +5 -0
  90. mindspore/mindspore_backend_common.dll +0 -0
  91. mindspore/mindspore_backend_manager.dll +0 -0
  92. mindspore/mindspore_common.dll +0 -0
  93. mindspore/mindspore_core.dll +0 -0
  94. mindspore/mindspore_cpu_res_manager.dll +0 -0
  95. mindspore/mindspore_dump.dll +0 -0
  96. mindspore/mindspore_frontend.dll +0 -0
  97. mindspore/mindspore_glog.dll +0 -0
  98. mindspore/mindspore_memory_pool.dll +0 -0
  99. mindspore/mindspore_ms_backend.dll +0 -0
  100. mindspore/mindspore_ops.dll +0 -0
  101. mindspore/mindspore_ops_host.dll +0 -0
  102. mindspore/mindspore_ops_kernel_common.dll +0 -0
  103. mindspore/mindspore_profiler.dll +0 -0
  104. mindspore/mindspore_pyboost.dll +0 -0
  105. mindspore/mindspore_pynative.dll +0 -0
  106. mindspore/mindspore_res_manager.dll +0 -0
  107. mindspore/mindspore_runtime_pipeline.dll +0 -0
  108. mindspore/mint/__init__.py +6 -46
  109. mindspore/mint/distributed/__init__.py +1 -0
  110. mindspore/mint/distributed/distributed.py +212 -9
  111. mindspore/mint/nn/__init__.py +1 -1
  112. mindspore/mint/nn/functional.py +53 -6
  113. mindspore/mint/nn/layer/_functions.py +164 -294
  114. mindspore/mint/nn/layer/activation.py +8 -6
  115. mindspore/mint/nn/layer/conv.py +137 -101
  116. mindspore/mint/nn/layer/normalization.py +8 -22
  117. mindspore/mint/optim/adam.py +19 -18
  118. mindspore/mint/optim/adamw.py +14 -8
  119. mindspore/mint/optim/sgd.py +5 -5
  120. mindspore/nn/cell.py +328 -502
  121. mindspore/nn/grad/cell_grad.py +11 -12
  122. mindspore/nn/layer/activation.py +32 -34
  123. mindspore/nn/layer/basic.py +67 -64
  124. mindspore/nn/layer/channel_shuffle.py +4 -4
  125. mindspore/nn/layer/combined.py +4 -2
  126. mindspore/nn/layer/conv.py +117 -110
  127. mindspore/nn/layer/dense.py +9 -7
  128. mindspore/nn/layer/embedding.py +50 -52
  129. mindspore/nn/layer/image.py +37 -39
  130. mindspore/nn/layer/math.py +111 -112
  131. mindspore/nn/layer/normalization.py +56 -44
  132. mindspore/nn/layer/pooling.py +58 -63
  133. mindspore/nn/layer/rnn_cells.py +33 -33
  134. mindspore/nn/layer/rnns.py +56 -56
  135. mindspore/nn/layer/thor_layer.py +74 -73
  136. mindspore/nn/layer/transformer.py +11 -1
  137. mindspore/nn/learning_rate_schedule.py +20 -20
  138. mindspore/nn/loss/loss.py +79 -81
  139. mindspore/nn/optim/adam.py +3 -3
  140. mindspore/nn/optim/adasum.py +2 -2
  141. mindspore/nn/optim/asgd.py +2 -0
  142. mindspore/nn/optim/optimizer.py +1 -1
  143. mindspore/nn/optim/thor.py +2 -2
  144. mindspore/nn/probability/distribution/exponential.py +2 -1
  145. mindspore/nn/probability/distribution/poisson.py +2 -1
  146. mindspore/nn/sparse/sparse.py +3 -3
  147. mindspore/nn/wrap/cell_wrapper.py +34 -37
  148. mindspore/nn/wrap/grad_reducer.py +37 -37
  149. mindspore/nn/wrap/loss_scale.py +72 -74
  150. mindspore/numpy/array_creations.py +5 -5
  151. mindspore/numpy/fft.py +1 -1
  152. mindspore/numpy/math_ops.py +5 -5
  153. mindspore/opencv_core452.dll +0 -0
  154. mindspore/opencv_imgcodecs452.dll +0 -0
  155. mindspore/opencv_imgproc452.dll +0 -0
  156. mindspore/ops/_grad_experimental/grad_comm_ops.py +51 -13
  157. mindspore/ops/_grad_experimental/grad_debug_ops.py +14 -0
  158. mindspore/ops/_vmap/vmap_array_ops.py +31 -13
  159. mindspore/ops/_vmap/vmap_nn_ops.py +8 -16
  160. mindspore/ops/auto_generate/cpp_create_prim_instance_helper.py +42 -11
  161. mindspore/ops/auto_generate/gen_extend_func.py +23 -141
  162. mindspore/ops/auto_generate/gen_ops_def.py +727 -321
  163. mindspore/ops/auto_generate/gen_ops_prim.py +1721 -984
  164. mindspore/ops/auto_generate/pyboost_inner_prim.py +31 -1
  165. mindspore/ops/composite/__init__.py +10 -0
  166. mindspore/ops/composite/base.py +8 -4
  167. mindspore/ops/composite/multitype_ops/__init__.py +12 -1
  168. mindspore/ops/composite/multitype_ops/_compile_utils.py +133 -109
  169. mindspore/ops/composite/multitype_ops/add_impl.py +70 -2
  170. mindspore/ops/composite/multitype_ops/div_impl.py +49 -0
  171. mindspore/ops/composite/multitype_ops/floordiv_impl.py +29 -0
  172. mindspore/ops/composite/multitype_ops/getitem_impl.py +11 -0
  173. mindspore/ops/composite/multitype_ops/mod_impl.py +5 -3
  174. mindspore/ops/composite/multitype_ops/mul_impl.py +49 -0
  175. mindspore/ops/composite/multitype_ops/setitem_impl.py +57 -0
  176. mindspore/ops/composite/multitype_ops/sub_impl.py +34 -0
  177. mindspore/ops/composite/multitype_ops/zeros_like_impl.py +14 -0
  178. mindspore/ops/function/__init__.py +3 -1
  179. mindspore/ops/function/_add_attr_func.py +11 -6
  180. mindspore/ops/function/array_func.py +9 -96
  181. mindspore/ops/function/debug_func.py +4 -3
  182. mindspore/ops/function/grad/grad_func.py +1 -1
  183. mindspore/ops/function/math_func.py +33 -540
  184. mindspore/ops/function/nn_func.py +28 -74
  185. mindspore/ops/function/other_func.py +4 -1
  186. mindspore/ops/function/random_func.py +44 -5
  187. mindspore/ops/function/vmap_func.py +2 -1
  188. mindspore/ops/functional.py +2 -3
  189. mindspore/ops/functional_overload.py +571 -6
  190. mindspore/ops/op_info_register.py +21 -0
  191. mindspore/ops/operations/__init__.py +16 -11
  192. mindspore/ops/operations/_custom_ops_utils.py +689 -34
  193. mindspore/ops/operations/_inner_ops.py +3 -6
  194. mindspore/ops/operations/_sequence_ops.py +1 -1
  195. mindspore/ops/operations/array_ops.py +2 -2
  196. mindspore/ops/operations/comm_ops.py +185 -26
  197. mindspore/ops/operations/custom_ops.py +294 -174
  198. mindspore/ops/operations/debug_ops.py +59 -4
  199. mindspore/ops/operations/image_ops.py +13 -13
  200. mindspore/ops/operations/manually_defined/ops_def.py +15 -16
  201. mindspore/ops/operations/math_ops.py +3 -4
  202. mindspore/ops/operations/nn_ops.py +7 -39
  203. mindspore/ops/primitive.py +6 -10
  204. mindspore/ops/tensor_method.py +47 -8
  205. mindspore/ops_generate/api/cpp_create_prim_instance_helper_generator.py +1 -1
  206. mindspore/ops_generate/api/functional_map_cpp_generator.py +10 -9
  207. mindspore/ops_generate/api/functions_cc_generator.py +58 -10
  208. mindspore/ops_generate/api/tensor_func_reg_cpp_generator.py +1 -1
  209. mindspore/ops_generate/common/base_generator.py +14 -0
  210. mindspore/ops_generate/common/gen_constants.py +8 -3
  211. mindspore/ops_generate/common/gen_utils.py +0 -19
  212. mindspore/ops_generate/common/op_proto.py +11 -4
  213. mindspore/ops_generate/common/template.py +88 -11
  214. mindspore/ops_generate/gen_ops.py +1 -1
  215. mindspore/ops_generate/op_def/lite_ops_cpp_generator.py +4 -4
  216. mindspore/ops_generate/op_def/ops_def_cc_generator.py +0 -3
  217. mindspore/ops_generate/op_def/ops_name_h_generator.py +0 -3
  218. mindspore/ops_generate/op_def/ops_primitive_h_generator.py +0 -4
  219. mindspore/ops_generate/op_def_py/op_prim_py_generator.py +5 -2
  220. mindspore/ops_generate/pyboost/auto_grad_impl_cc_generator.py +49 -8
  221. mindspore/ops_generate/pyboost/auto_grad_reg_cc_generator.py +2 -2
  222. mindspore/ops_generate/pyboost/gen_pyboost_func.py +31 -0
  223. mindspore/ops_generate/pyboost/op_template_parser.py +98 -72
  224. mindspore/ops_generate/pyboost/pyboost_functions_cpp_generator.py +70 -273
  225. mindspore/ops_generate/pyboost/pyboost_functions_h_generator.py +14 -6
  226. mindspore/ops_generate/pyboost/pyboost_functions_impl_cpp_generator.py +316 -0
  227. mindspore/ops_generate/pyboost/pyboost_functions_py_generator.py +1 -1
  228. mindspore/ops_generate/pyboost/pyboost_grad_function_cpp_generator.py +5 -3
  229. mindspore/ops_generate/pyboost/pyboost_inner_prim_generator.py +1 -1
  230. mindspore/ops_generate/pyboost/pyboost_internal_functions_cpp_generator.py +76 -0
  231. mindspore/ops_generate/pyboost/pyboost_internal_functions_h_generator.py +76 -0
  232. mindspore/ops_generate/pyboost/pyboost_internal_kernel_info_adapter_generator.py +125 -0
  233. mindspore/ops_generate/pyboost/pyboost_native_grad_functions_generator.py +4 -3
  234. mindspore/ops_generate/pyboost/pyboost_op_cpp_code_generator.py +348 -61
  235. mindspore/ops_generate/pyboost/pyboost_overload_functions_cpp_generator.py +1 -1
  236. mindspore/ops_generate/pyboost/pyboost_utils.py +118 -9
  237. mindspore/ops_generate/tensor_py_cc_generator.py +1 -24
  238. mindspore/parallel/_auto_parallel_context.py +11 -8
  239. mindspore/parallel/_cell_wrapper.py +113 -45
  240. mindspore/parallel/_parallel_serialization.py +1 -1
  241. mindspore/parallel/_ps_context.py +4 -6
  242. mindspore/parallel/_tensor.py +167 -12
  243. mindspore/parallel/_transformer/moe.py +1 -1
  244. mindspore/parallel/_transformer/transformer.py +13 -8
  245. mindspore/parallel/auto_parallel.py +14 -7
  246. mindspore/parallel/checkpoint_convert.py +3 -3
  247. mindspore/parallel/checkpoint_transform.py +11 -7
  248. mindspore/parallel/cluster/process_entity/_api.py +84 -48
  249. mindspore/parallel/cluster/process_entity/_utils.py +95 -7
  250. mindspore/parallel/cluster/run.py +43 -4
  251. mindspore/parallel/function/__init__.py +8 -1
  252. mindspore/parallel/function/reshard_func.py +6 -7
  253. mindspore/parallel/nn/__init__.py +15 -2
  254. mindspore/parallel/nn/parallel_cell_wrapper.py +9 -10
  255. mindspore/parallel/nn/parallel_grad_reducer.py +7 -6
  256. mindspore/parallel/shard.py +3 -4
  257. mindspore/parallel/transform_safetensors.py +463 -174
  258. mindspore/profiler/__init__.py +2 -1
  259. mindspore/profiler/analysis/parser/timeline_assembly_factory/ascend_timeline_assembler.py +7 -7
  260. mindspore/profiler/analysis/parser/timeline_assembly_factory/base_timeline_assembler.py +3 -0
  261. mindspore/profiler/analysis/parser/timeline_assembly_factory/trace_view_container.py +12 -6
  262. mindspore/profiler/analysis/parser/timeline_creator/cpu_op_timeline_creator.py +3 -3
  263. mindspore/profiler/analysis/parser/timeline_creator/fwk_timeline_creator.py +3 -3
  264. mindspore/profiler/analysis/parser/timeline_creator/msprof_timeline_creator.py +4 -4
  265. mindspore/profiler/analysis/parser/timeline_creator/scope_layer_timeline_creator.py +3 -3
  266. mindspore/profiler/analysis/parser/timeline_event/fwk_event.py +4 -1
  267. mindspore/profiler/analysis/parser/timeline_event/timeline_event_pool.py +2 -1
  268. mindspore/profiler/analysis/task_manager.py +1 -1
  269. mindspore/profiler/analysis/viewer/ascend_communication_viewer.py +5 -1
  270. mindspore/profiler/analysis/viewer/ascend_integrate_viewer.py +2 -1
  271. mindspore/profiler/analysis/viewer/ascend_op_memory_viewer.py +42 -22
  272. mindspore/profiler/analysis/viewer/ascend_step_trace_time_viewer.py +3 -2
  273. mindspore/profiler/analysis/viewer/ms_minddata_viewer.py +9 -5
  274. mindspore/profiler/analysis/viewer/ms_operator_details_viewer.py +132 -0
  275. mindspore/profiler/common/constant.py +16 -0
  276. mindspore/profiler/common/profiler_context.py +25 -27
  277. mindspore/profiler/common/profiler_info.py +0 -16
  278. mindspore/profiler/common/profiler_op_analyse.py +235 -0
  279. mindspore/profiler/common/profiler_output_path.py +23 -8
  280. mindspore/profiler/common/profiler_parameters.py +128 -35
  281. mindspore/profiler/dynamic_profile/__init__.py +0 -0
  282. mindspore/profiler/dynamic_profile/dynamic_monitor_proxy.py +39 -0
  283. mindspore/profiler/dynamic_profile/dynamic_profiler_config_context.py +666 -0
  284. mindspore/profiler/dynamic_profile/dynamic_profiler_utils.py +62 -0
  285. mindspore/profiler/dynamic_profiler.py +305 -314
  286. mindspore/profiler/envprofiler.py +12 -7
  287. mindspore/profiler/experimental_config.py +96 -6
  288. mindspore/profiler/mstx.py +33 -12
  289. mindspore/profiler/platform/__init__.py +2 -3
  290. mindspore/profiler/platform/npu_profiler.py +29 -19
  291. mindspore/profiler/profiler.py +35 -19
  292. mindspore/profiler/profiler_action_controller.py +64 -76
  293. mindspore/profiler/schedule.py +10 -4
  294. mindspore/rewrite/common/config.py +1 -0
  295. mindspore/rewrite/common/namer.py +1 -0
  296. mindspore/rewrite/common/namespace.py +1 -0
  297. mindspore/rewrite/node/node.py +31 -11
  298. mindspore/rewrite/parsers/assign_parser.py +1 -1
  299. mindspore/rewrite/symbol_tree/symbol_tree.py +1 -1
  300. mindspore/run_check/_check_version.py +7 -10
  301. mindspore/runtime/__init__.py +5 -5
  302. mindspore/runtime/event.py +10 -4
  303. mindspore/runtime/executor.py +60 -45
  304. mindspore/runtime/memory.py +30 -32
  305. mindspore/runtime/thread_bind_core.py +298 -164
  306. mindspore/safeguard/rewrite_obfuscation.py +12 -13
  307. mindspore/swresample-4.dll +0 -0
  308. mindspore/swscale-6.dll +0 -0
  309. mindspore/tinyxml2.dll +0 -0
  310. mindspore/train/_utils.py +14 -4
  311. mindspore/train/amp.py +43 -20
  312. mindspore/train/callback/__init__.py +5 -5
  313. mindspore/train/callback/_checkpoint.py +3 -6
  314. mindspore/train/callback/_flops_collector.py +1 -1
  315. mindspore/train/callback/_landscape.py +0 -1
  316. mindspore/train/callback/_train_fault_tolerance.py +97 -16
  317. mindspore/train/data_sink.py +11 -2
  318. mindspore/train/dataset_helper.py +9 -0
  319. mindspore/train/model.py +135 -55
  320. mindspore/train/serialization.py +133 -111
  321. mindspore/train/summary/summary_record.py +13 -2
  322. mindspore/turbojpeg.dll +0 -0
  323. mindspore/utils/__init__.py +3 -2
  324. mindspore/utils/dryrun.py +0 -6
  325. mindspore/utils/runtime_execution_order_check.py +163 -77
  326. mindspore/utils/sdc_detect.py +68 -0
  327. mindspore/utils/utils.py +6 -9
  328. mindspore/version.py +1 -1
  329. {mindspore-2.6.0rc1.dist-info → mindspore-2.7.0rc1.dist-info}/METADATA +5 -4
  330. {mindspore-2.6.0rc1.dist-info → mindspore-2.7.0rc1.dist-info}/RECORD +333 -371
  331. mindspore/_deprecated/jit.py +0 -198
  332. mindspore/experimental/es/__init__.py +0 -22
  333. mindspore/experimental/es/embedding_service.py +0 -891
  334. mindspore/experimental/es/embedding_service_layer.py +0 -581
  335. mindspore/profiler/parser/__init__.py +0 -14
  336. mindspore/profiler/parser/aicpu_data_parser.py +0 -272
  337. mindspore/profiler/parser/ascend_analysis/__init__.py +0 -14
  338. mindspore/profiler/parser/ascend_analysis/constant.py +0 -71
  339. mindspore/profiler/parser/ascend_analysis/file_manager.py +0 -180
  340. mindspore/profiler/parser/ascend_analysis/function_event.py +0 -185
  341. mindspore/profiler/parser/ascend_analysis/fwk_cann_parser.py +0 -136
  342. mindspore/profiler/parser/ascend_analysis/fwk_file_parser.py +0 -131
  343. mindspore/profiler/parser/ascend_analysis/msprof_timeline_parser.py +0 -104
  344. mindspore/profiler/parser/ascend_analysis/path_manager.py +0 -313
  345. mindspore/profiler/parser/ascend_analysis/profiler_info_parser.py +0 -123
  346. mindspore/profiler/parser/ascend_analysis/tlv_decoder.py +0 -86
  347. mindspore/profiler/parser/ascend_analysis/trace_event_manager.py +0 -75
  348. mindspore/profiler/parser/ascend_cluster_generator.py +0 -116
  349. mindspore/profiler/parser/ascend_communicate_generator.py +0 -314
  350. mindspore/profiler/parser/ascend_flops_generator.py +0 -116
  351. mindspore/profiler/parser/ascend_fpbp_generator.py +0 -82
  352. mindspore/profiler/parser/ascend_hccl_generator.py +0 -271
  353. mindspore/profiler/parser/ascend_integrate_generator.py +0 -42
  354. mindspore/profiler/parser/ascend_memory_generator.py +0 -185
  355. mindspore/profiler/parser/ascend_msprof_exporter.py +0 -282
  356. mindspore/profiler/parser/ascend_msprof_generator.py +0 -187
  357. mindspore/profiler/parser/ascend_op_generator.py +0 -334
  358. mindspore/profiler/parser/ascend_steptrace_generator.py +0 -94
  359. mindspore/profiler/parser/ascend_timeline_generator.py +0 -545
  360. mindspore/profiler/parser/base_timeline_generator.py +0 -483
  361. mindspore/profiler/parser/container.py +0 -229
  362. mindspore/profiler/parser/cpu_gpu_timeline_generator.py +0 -697
  363. mindspore/profiler/parser/flops_parser.py +0 -531
  364. mindspore/profiler/parser/framework_enum.py +0 -111
  365. mindspore/profiler/parser/framework_parser.py +0 -464
  366. mindspore/profiler/parser/framework_struct.py +0 -61
  367. mindspore/profiler/parser/gpu_analysis/__init__.py +0 -14
  368. mindspore/profiler/parser/gpu_analysis/function_event.py +0 -44
  369. mindspore/profiler/parser/gpu_analysis/fwk_file_parser.py +0 -89
  370. mindspore/profiler/parser/gpu_analysis/profiler_info_parser.py +0 -72
  371. mindspore/profiler/parser/hccl_parser.py +0 -573
  372. mindspore/profiler/parser/hwts_log_parser.py +0 -122
  373. mindspore/profiler/parser/integrator.py +0 -526
  374. mindspore/profiler/parser/memory_usage_parser.py +0 -277
  375. mindspore/profiler/parser/minddata_analyzer.py +0 -800
  376. mindspore/profiler/parser/minddata_parser.py +0 -186
  377. mindspore/profiler/parser/minddata_pipeline_parser.py +0 -299
  378. mindspore/profiler/parser/op_intermediate_parser.py +0 -149
  379. mindspore/profiler/parser/optime_parser.py +0 -250
  380. mindspore/profiler/parser/profiler_info.py +0 -213
  381. mindspore/profiler/parser/step_trace_parser.py +0 -666
  382. {mindspore-2.6.0rc1.dist-info → mindspore-2.7.0rc1.dist-info}/WHEEL +0 -0
  383. {mindspore-2.6.0rc1.dist-info → mindspore-2.7.0rc1.dist-info}/entry_points.txt +0 -0
  384. {mindspore-2.6.0rc1.dist-info → mindspore-2.7.0rc1.dist-info}/top_level.txt +0 -0
@@ -1,185 +0,0 @@
1
- # Copyright 2023 Huawei Technologies Co., Ltd
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
- # ============================================================================
15
- """Function event data struct."""
16
- from typing import Dict, Optional
17
- from enum import Enum
18
- from decimal import Decimal
19
- import struct
20
- from abc import ABC, abstractmethod
21
-
22
- from mindspore.profiler.parser.ascend_analysis.constant import Constant
23
- from mindspore.profiler.parser.ascend_analysis.profiler_info_parser import ProfilerInfoParser
24
-
25
-
26
- class BaseEvent(ABC):
27
- """Base class of CANNEvent and MindSporeOpEvent"""
28
-
29
- def __init__(self, data: Dict):
30
- if not isinstance(data, dict):
31
- raise TypeError("Input data must be dict.")
32
- # common attributes
33
- self._orig_data = data
34
- self.name: str = ""
35
- self.pid: int = 0
36
- self.tid: int = 0
37
- self.ts: Decimal = Decimal(0)
38
- self.te: Decimal = Decimal(0)
39
- self.dur: float = 0.0
40
- self.args: Dict = {}
41
- self.parent: Optional[BaseEvent] = None
42
- self._init_params()
43
-
44
- @abstractmethod
45
- def _init_params(self):
46
- err_msg = "Protected function _init_params need to be implemented."
47
- raise NotImplementedError(err_msg)
48
-
49
-
50
- class CANNEvent(BaseEvent):
51
- """Function event collected on the CANN side"""
52
-
53
- def _init_params(self):
54
- """Initialize the attribute value of CANNEvent."""
55
- self.ts = Decimal(str(self._orig_data.get("ts", 0)))
56
- self.pid = self._orig_data.get("pid", 0)
57
- self.tid = self._orig_data.get("tid", 0)
58
- self.dur = self._orig_data.get("dur", 0.0)
59
- self.te = self.ts + Decimal(str(self.dur))
60
- self.name = self._orig_data.get("name", "")
61
- self.id = self._orig_data.get("id", 0)
62
- self.args = self._orig_data.get("args", {})
63
- self.ph = self._orig_data.get("ph")
64
- self.cat = self._orig_data.get("cat")
65
-
66
- def is_flow_start_event(self) -> bool:
67
- """Determine whether the event is flow start event or not."""
68
- return self._orig_data.get("cat") == Constant.HOST_TO_DEVICE and \
69
- self._orig_data.get("ph") == Constant.START_FLOW
70
-
71
- def is_flow_end_event(self) -> bool:
72
- """Determine whether the event is flow end event or not."""
73
- return self._orig_data.get("cat") == Constant.HOST_TO_DEVICE and \
74
- self._orig_data.get("ph") == Constant.END_FLOW
75
-
76
- def is_x_event(self) -> bool:
77
- """Determine whether the event x event or not."""
78
- return self._orig_data.get("ph") == Constant.COMPLETE_EVENT
79
-
80
- def get_sort_index(self):
81
- """get the process sort index"""
82
- if self.args.get('name', '') == 'Ascend Hardware':
83
- return Constant.ASCEND_HARDWARE
84
- if self.args.get('name', '') == 'CANN':
85
- return Constant.CANN
86
- if self.args.get('name', '') == 'HCCL':
87
- return Constant.HCCL
88
- if self.args.get('name', '') == 'Overlap Analysis':
89
- return Constant.OVERLAP
90
- return Constant.OTHERWISE
91
-
92
- def to_json(self):
93
- """Cast to trace event."""
94
- if self.ph == Constant.META_EVENT:
95
- if self.name == Constant.PROCESS_NAME:
96
- return [{'name': self.name, 'pid': self.pid, 'tid': self.tid,
97
- 'args': self.args, 'ph': self.ph},
98
- {'name': Constant.PROCESS_SORT, 'pid': self.pid, 'tid': self.tid,
99
- 'args': {'sort_index': self.get_sort_index()}, 'ph': self.ph}]
100
- if self.name == Constant.PROCESS_SORT:
101
- return None
102
- return {'name': self.name, 'pid': self.pid, 'tid': self.tid,
103
- 'args': self.args, 'ph': self.ph}
104
-
105
- if self.ph == Constant.COMPLETE_EVENT:
106
- if self.parent is not None:
107
- self.args.update({'mindspore_op': self.parent.name})
108
- res = {'name': self.name, 'pid': self.pid, 'tid': self.tid,
109
- 'ts': str(self.ts), 'dur': self.dur, 'args': self.args, 'ph': self.ph}
110
- if self.cat:
111
- res.update({'cat': self.cat})
112
- return res
113
-
114
- if self.ph == Constant.START_FLOW:
115
- return {"ph": self.ph, "name": self.name, "id": self.id, "pid": self.pid,
116
- "tid": self.tid, "ts": str(self.ts), "cat": self.cat}
117
-
118
- if self.ph == Constant.END_FLOW:
119
- return {"ph": self.ph, "name": self.name, "id": self.id, "pid": self.pid,
120
- "tid": self.tid, "ts": str(self.ts), "cat": self.cat, 'bp': "e"}
121
- return None
122
-
123
-
124
- class MindSporeOpEnum(Enum):
125
- """MindSporeOp index enum"""
126
- START_NS = 0
127
- END_NS = 1
128
- SEQUENCE_UNMBER = 2
129
- PROCESS_ID = 3
130
- START_THREAD_ID = 4
131
- END_THREAD_ID = 5
132
- FORWORD_THREAD_ID = 6
133
- FLOW_ID = 7
134
- STEP_ID = 8
135
- LEVEL = 9
136
- IS_ASYNC = 10
137
-
138
-
139
- class MindSporeOpEvent(BaseEvent):
140
- """
141
- Function event collected on the mindspore frame side.
142
-
143
- Args:
144
- data(Dict): The mindspore frame side data decoded by TLVDecoder.
145
- """
146
- _tlv_type_dict = {
147
- Constant.OP_NAME: 3, Constant.INPUT_SHAPES: 5, Constant.INPUT_DTYPES: 4,
148
- Constant.CALL_STACK: 6, Constant.MODULE_HIERARCHY: 7, Constant.FLOPS: 8,
149
- Constant.CUSTOM_INFO: 9
150
- }
151
- _fix_data_format = "<3q6Qb?"
152
-
153
- def _init_params(self):
154
- """Initialize the attribute value of MindSporeOpEvent."""
155
- fix_size_data = struct.unpack(self._fix_data_format, self._orig_data.get(Constant.FIX_SIZE_BYTES))
156
- self.pid = Constant.MINDSPORE
157
- self.tid = int(fix_size_data[MindSporeOpEnum.START_THREAD_ID.value])
158
- self.name = str(self._orig_data.get(self._tlv_type_dict.get(Constant.OP_NAME), ""))
159
- self.ts = ProfilerInfoParser.get_local_time(fix_size_data[MindSporeOpEnum.START_NS.value]) # unit is us
160
- self.te = ProfilerInfoParser.get_local_time(fix_size_data[MindSporeOpEnum.END_NS.value]) # unit is us
161
- self.dur = self.te - self.ts
162
- self.flow_id = int(fix_size_data[MindSporeOpEnum.FLOW_ID.value])
163
- self.step = int(fix_size_data[MindSporeOpEnum.STEP_ID.value])
164
- self.level = int(fix_size_data[MindSporeOpEnum.LEVEL.value])
165
- self.custom_info = ""
166
- self.args = self._get_args(fix_size_data)
167
-
168
- def _get_args(self, fix_size_data) -> Dict:
169
- """Get the rest information saved in args"""
170
- args = {
171
- Constant.SEQUENCE_UNMBER: int(fix_size_data[MindSporeOpEnum.SEQUENCE_UNMBER.value]),
172
- Constant.FORWORD_THREAD_ID: int(fix_size_data[MindSporeOpEnum.FORWORD_THREAD_ID.value])}
173
- for type_name, type_id in self._tlv_type_dict.items():
174
- if type_name == Constant.OP_NAME or type_id not in self._orig_data.keys():
175
- continue
176
- if type_name in set([Constant.INPUT_SHAPES, Constant.INPUT_DTYPES, Constant.CALL_STACK]):
177
- args[type_name] = self._orig_data.get(type_id).replace("|", "\r\n")
178
- elif type_name == Constant.CUSTOM_INFO and self._orig_data.get(type_id):
179
- pairs = self._orig_data.get(type_id).split(';')
180
- custom_info = {pair.split(':')[0]: pair.split(':')[1] for pair in pairs}
181
- args[type_name] = custom_info
182
- self.custom_info = custom_info.__str__()
183
- else:
184
- args[type_name] = self._orig_data.get(type_id)
185
- return args
@@ -1,136 +0,0 @@
1
- # Copyright 2023 Huawei Technologies Co., Ltd
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
- # ============================================================================
15
- """The parser for integrating data from the framework side and CANN side."""
16
- from collections import defaultdict
17
- from typing import List, Tuple, Optional
18
- from decimal import Decimal
19
-
20
- from mindspore import log as logger
21
- from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
22
- from mindspore.profiler.parser.ascend_analysis.function_event import CANNEvent, MindSporeOpEvent
23
- from mindspore.profiler.parser.ascend_analysis.fwk_file_parser import FwkFileParser
24
- from mindspore.profiler.parser.ascend_analysis.trace_event_manager import TraceEventManager
25
- from mindspore.profiler.parser.ascend_analysis.msprof_timeline_parser import MsprofTimelineParser
26
- from mindspore.profiler.parser.ascend_analysis.profiler_info_parser import ProfilerInfoParser
27
- from mindspore.profiler.parser.profiler_info import ProfilerInfo
28
-
29
-
30
- class FwkCANNParser:
31
- """The top-level trace view parser."""
32
-
33
- def __init__(self, source_path: str, msprof_data: List, rank_id: int, step_list=None):
34
- source_path = validate_and_normalize_path(source_path)
35
- ProfilerInfoParser.init_source_path(source_path)
36
- ProfilerInfoParser.init_rank_id(rank_id)
37
-
38
- fwk_parser = FwkFileParser(source_path, rank_id)
39
- self._fwk_data = fwk_parser.get_op_range_data(step_list)
40
- self._fwk_json, self._link_event_dict, self._fwk_launch_op = fwk_parser.get_fwk_trace_data(self._fwk_data)
41
-
42
- msprof_timeline_parser = MsprofTimelineParser(msprof_data)
43
- self._start_flow_to_npu_dict, self._device_mostly_json, self.scope_data_without_flow \
44
- = msprof_timeline_parser.get_acl_to_npu_data()
45
- self.scope_data_with_flow = []
46
- self.rank_id: int = rank_id
47
- self.kernels: List[CANNEvent] = []
48
-
49
- def generate_trace_data(self):
50
- """
51
- Generate trace view json timeline file which contains framework side and
52
- device side data.
53
- """
54
- fwk_flow_json = self.__link_msop_self()
55
- device_flow_and_x_json = self.__link_msop_kernel()
56
- return self._fwk_json + fwk_flow_json + self._device_mostly_json + device_flow_and_x_json
57
-
58
- def __link_msop_self(self):
59
- """Create flow between framework-side multi-level pipeline task."""
60
- flow_json = []
61
- for op_data_list in self._link_event_dict.values():
62
- if len(op_data_list) != 2:
63
- logger.info('Only alow 2 op_data have the same flow_id. but got %s', len(op_data_list))
64
- continue
65
- flow_json.extend(TraceEventManager.create_mindspore_to_self_flow(op_data_list[0],
66
- op_data_list[1]))
67
- return flow_json
68
-
69
- def __link_msop_kernel(self):
70
- """Associate the frame-side operator with the device-side kernel"""
71
- trace_data_json = []
72
- acl_to_npu_by_tid = {}
73
- for (cann_op_tid, cann_op_ts), event_list in self._start_flow_to_npu_dict.items():
74
- acl_to_npu_by_tid.setdefault(cann_op_tid, defaultdict(list))[cann_op_ts].extend(event_list)
75
-
76
- if self._fwk_launch_op and acl_to_npu_by_tid and self._fwk_launch_op.keys() != acl_to_npu_by_tid.keys():
77
- logger.warning("Failed to create link between mindspore operator and kernels.")
78
- is_not_O2 = bool(ProfilerInfo.get_profiler_info().get(ProfilerInfo.JIT_LEVEL, "") != "O2")
79
- for device_tid in acl_to_npu_by_tid:
80
- host_data_sorted = sorted(self._fwk_launch_op.get(device_tid, []), key=lambda x: x.ts)
81
- op_idx = 0
82
-
83
- for ts, device_data_list in sorted(acl_to_npu_by_tid.get(device_tid).items(), key=lambda x: x[0]):
84
- op_idx, status = FwkCANNParser.__find_launch_op(ts, host_data_sorted, op_idx)
85
- if not status and is_not_O2:
86
- logger.warning("There are %s device ops have no flows were found. The CANN ts is %s.",
87
- len(device_data_list), ts)
88
-
89
- for device_data in device_data_list:
90
- if status:
91
- device_data.parent = host_data_sorted[op_idx]
92
- trace_data_json.extend(TraceEventManager.create_mindspore_to_npu_flow(device_data.parent,
93
- device_data))
94
- self.kernels.append(device_data)
95
-
96
- scope_layer = MsprofTimelineParser.parse_ascend_hardware_scope(device_data)
97
-
98
- if scope_layer is not None:
99
- self.scope_data_with_flow.append(scope_layer)
100
-
101
- event_json = device_data.to_json()
102
- if event_json and isinstance(event_json, list):
103
- trace_data_json.extend(event_json)
104
- elif event_json and isinstance(event_json, dict):
105
- trace_data_json.append(event_json)
106
-
107
- return trace_data_json
108
-
109
- @staticmethod
110
- def __find_launch_op(ts: Decimal, op_list: List[MindSporeOpEvent],
111
- left: Optional[int] = None, right: Optional[int] = None
112
- ) -> Tuple[int, bool]:
113
- """
114
- Searching the op_list in [left, right) range and find the operator
115
- whose start time is larger than ts and end time is less than ts.
116
-
117
- Args:
118
- ts(Decimal): kernel start time
119
- op_list(List): MindSporeOpEvent list
120
- left & right(int): the searching index range is [left, right)
121
-
122
- Return:
123
- Tuple[int, bool]: the first element is the searched index, the second element
124
- marks where the operator index is found or not.
125
- """
126
- left = 0 if (left is None or left < 0) else left
127
- right = len(op_list) if (right is None or right < 0) else right
128
- # The data in range [left, right) is considered.
129
- while right > left:
130
- if op_list[left].ts > ts:
131
- return left, False
132
- if op_list[left].te < ts:
133
- left += 1
134
- else:
135
- return left, True
136
- return left, False
@@ -1,131 +0,0 @@
1
- # Copyright 2023 Huawei Technologies Co., Ltd
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
- # ============================================================================
15
- """MindSpore framework oprange file parser"""
16
-
17
- import os
18
- import stat
19
- import csv
20
- from typing import List
21
- from collections import defaultdict
22
-
23
- from mindspore import log as logger
24
- from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
25
- from mindspore.profiler.parser.ascend_analysis.file_manager import FileManager
26
- from mindspore.profiler.parser.ascend_analysis.tlv_decoder import TLVDecoder
27
- from mindspore.profiler.parser.ascend_analysis.function_event import MindSporeOpEvent
28
- from mindspore.profiler.parser.ascend_analysis.trace_event_manager import TraceEventManager
29
- from mindspore.profiler.parser.ascend_analysis.constant import Constant
30
-
31
-
32
- class FwkFileParser:
33
- """Framework-side operator file parser."""
34
-
35
- _op_range = "FRAMEWORK/op_range_{}"
36
- _op_range_struct_size = 74
37
-
38
- def __init__(self, source_path: str, rank_id: int):
39
- """
40
- source_path: The path of PROF_* directory
41
- """
42
- self.rank_id = rank_id
43
- self._prof_root = None
44
- self._op_range_path = None
45
- self._init_framework_path(source_path)
46
-
47
- def get_op_range_data(self, step_list=None) -> List[MindSporeOpEvent]:
48
- """Read and decode all the mindspore oprange data."""
49
- op_range_list = []
50
- if os.path.exists(self._op_range_path):
51
- op_range_bytes = FileManager.read_file_content(self._op_range_path, "rb")
52
- op_range_list = TLVDecoder.decode(op_range_bytes, MindSporeOpEvent, self._op_range_struct_size)
53
- else:
54
- logger.warning("Failed to find op_range data. skip parse host profiler data.")
55
- if step_list and isinstance(step_list, list):
56
- first_step = min(op.step for op in op_range_list)
57
- step_list = [step - 1 + first_step for step in step_list]
58
- op_range_list = list(filter(lambda op: op.step in step_list, op_range_list))
59
- return op_range_list
60
-
61
- def get_fwk_trace_data(self, mindspore_op_data: List[MindSporeOpEvent] = None):
62
- """Generate chrome trace format json data from decoded oprange data."""
63
- if not mindspore_op_data:
64
- mindspore_op_data = self.get_op_range_data()
65
- tid_map = defaultdict(set)
66
- fwk_x_event_list = []
67
- link_event_dict = defaultdict(list)
68
- fwk_launch_op = defaultdict(list)
69
- dataset_op_data = []
70
-
71
- for mindspore_op in mindspore_op_data:
72
- if mindspore_op.name == Constant.FLOW_OP:
73
- link_event_dict[mindspore_op.flow_id].insert(0, mindspore_op)
74
- continue
75
- if mindspore_op.flow_id != Constant.INVALID_FLOW_ID:
76
- link_event_dict[mindspore_op.flow_id].append(mindspore_op)
77
-
78
- if 'KernelLaunch' in mindspore_op.name or 'LaunchTask' in mindspore_op.name:
79
- fwk_launch_op[mindspore_op.tid].append(mindspore_op)
80
-
81
- if mindspore_op.name.split('::')[0] == 'Dataset':
82
- dataset_op_data.append(mindspore_op)
83
-
84
- tid_map[mindspore_op.pid].add(mindspore_op.tid)
85
- if mindspore_op.dur > 0:
86
- fwk_x_event_list.append(TraceEventManager.create_x_event(mindspore_op, "cpu_op"))
87
- else:
88
- fwk_x_event_list.append(TraceEventManager.create_i_event(mindspore_op))
89
-
90
- fwk_m_event_list = []
91
- for pid, tid_set in tid_map.items():
92
- fwk_m_event_list.extend(TraceEventManager.create_m_event(pid, tid_set, pid))
93
-
94
- self.calculate_dataset_item(dataset_op_data)
95
-
96
- return fwk_x_event_list + fwk_m_event_list, link_event_dict, fwk_launch_op
97
-
98
- def calculate_dataset_item(self, dataset_op_data: List[MindSporeOpEvent]):
99
- """
100
- Get the summary data of dataset op by parsing the dataset_op_data.
101
- """
102
- self._prof_root = validate_and_normalize_path(self._prof_root)
103
- dataset_file = os.path.join(self._prof_root, f'dataset_{self.rank_id}.csv')
104
- summary_data = defaultdict(list)
105
- for op_data in dataset_op_data:
106
- _, event, stage = op_data.name.split('::')
107
- key = event + '::' + stage + '::' + op_data.custom_info
108
- summary_data[key].append(op_data.dur)
109
-
110
- flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
111
- modes = stat.S_IWUSR | stat.S_IRUSR
112
- with os.fdopen(os.open(dataset_file, flags, modes), 'w', newline='') as fw:
113
- csv_writer = csv.writer(fw)
114
- csv_writer.writerow(['Operation', 'Stage', 'Occurrences', 'Avg. time (us)', 'Custom Info'])
115
- for k, v in summary_data.items():
116
- event, stage, custom_info = k.split('::')
117
- count = len(v)
118
- average_execution = round(float(sum(v) / count), 2)
119
- csv_writer.writerow([event, stage, count, average_execution, custom_info])
120
- os.chmod(dataset_file, modes)
121
-
122
- def _init_framework_path(self, source_path: str):
123
- """Init the oprange data path."""
124
- source_path = validate_and_normalize_path(source_path)
125
- if not os.path.exists(source_path):
126
- raise FileNotFoundError("Input source_path does not exist!")
127
- device_name = os.path.basename(source_path)
128
- if not device_name.startswith("device") and not os.path.isdir(source_path):
129
- raise RuntimeError("Input source_path is invalid!")
130
- self._prof_root = os.path.dirname(os.path.dirname(source_path))
131
- self._op_range_path = os.path.join(self._prof_root, self._op_range.format(self.rank_id))
@@ -1,104 +0,0 @@
1
- # Copyright 2023 Huawei Technologies Co., Ltd
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
- # ============================================================================
15
- """msprof timeline file parser"""
16
- from collections import defaultdict
17
- from typing import List
18
-
19
- from mindspore.profiler.parser.ascend_analysis.constant import Constant
20
- from mindspore.profiler.parser.ascend_analysis.function_event import CANNEvent, MindSporeOpEvent, BaseEvent
21
-
22
-
23
- class MsprofTimelineParser:
24
- """Msprof timeline file parser."""
25
-
26
- def __init__(self, msprof_data: List):
27
- self.timeline_data = msprof_data
28
-
29
- def get_acl_to_npu_data(self):
30
- """Get all the acl to npu flow events."""
31
- flow_start_dict, flow_end_dict = {}, {}
32
- x_event_list, other_event_json = [], []
33
- for data in self.timeline_data:
34
- cann_event = CANNEvent(data)
35
- if cann_event.is_x_event():
36
- x_event_list.append(cann_event)
37
- else:
38
- event_json = cann_event.to_json()
39
- if event_json and isinstance(event_json, list):
40
- other_event_json.extend(event_json)
41
- elif event_json and isinstance(event_json, dict):
42
- other_event_json.append(event_json)
43
-
44
- if cann_event.is_flow_start_event():
45
- flow_start_dict[cann_event.id] = (cann_event.tid, cann_event.ts)
46
- elif cann_event.is_flow_end_event():
47
- flow_end_dict[(cann_event.pid, cann_event.tid, cann_event.ts)] = cann_event.id
48
-
49
- start_flow_to_npu_dict = defaultdict(list)
50
- device_data_without_flow_json = []
51
- scope_data_without_flow = []
52
- for cann_event in x_event_list:
53
- flow_id = flow_end_dict.get((cann_event.pid, cann_event.tid, cann_event.ts))
54
- start_flow_info = flow_start_dict.get(flow_id)
55
- if flow_id is not None and start_flow_info is not None:
56
- start_flow_to_npu_dict[start_flow_info].append(cann_event)
57
- else:
58
- event_json = cann_event.to_json()
59
- if event_json and isinstance(event_json, list):
60
- device_data_without_flow_json.extend(event_json)
61
- elif event_json and isinstance(event_json, dict):
62
- device_data_without_flow_json.append(event_json)
63
-
64
- scope_layer = self.parse_ascend_hardware_scope(cann_event)
65
- if scope_layer is not None:
66
- scope_data_without_flow.append(scope_layer)
67
-
68
- return start_flow_to_npu_dict, device_data_without_flow_json + other_event_json, scope_data_without_flow
69
-
70
- @staticmethod
71
- def parse_ascend_hardware_scope(cann_event: BaseEvent):
72
- """
73
- parse ascend hardware scope
74
- """
75
- scope_from_flow_start = MsprofTimelineParser.get_scope_layer(cann_event.parent)
76
- scope = MsprofTimelineParser.get_scope_layer(cann_event)
77
-
78
- if scope and scope_from_flow_start:
79
- scope = scope_from_flow_start if len(scope_from_flow_start) > len(scope) else scope
80
- elif not scope and not scope_from_flow_start:
81
- return None
82
- else:
83
- scope = scope_from_flow_start if scope_from_flow_start else scope
84
-
85
- return (scope, cann_event.ts, cann_event.te)
86
-
87
- @staticmethod
88
- def get_scope_layer(event: BaseEvent):
89
- """
90
- get the scope of device op
91
- """
92
-
93
- if event is None:
94
- return None
95
-
96
- full_scope_name = None
97
- if isinstance(event, MindSporeOpEvent):
98
- full_scope_name = event.name.split('::')[-1]
99
- elif isinstance(event, CANNEvent):
100
- full_scope_name = event.name
101
-
102
- if full_scope_name and full_scope_name.startswith(Constant.TOP_SCOPE_NAMES):
103
- return full_scope_name.split('/')[:-1]
104
- return None