mindspore 2.6.0rc1__cp39-cp39-win_amd64.whl → 2.7.0rc1__cp39-cp39-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mindspore might be problematic. Click here for more details.

Files changed (384) hide show
  1. mindspore/.commit_id +1 -1
  2. mindspore/__init__.py +1 -1
  3. mindspore/_c_dataengine.cp39-win_amd64.pyd +0 -0
  4. mindspore/_c_expression.cp39-win_amd64.pyd +0 -0
  5. mindspore/_c_mindrecord.cp39-win_amd64.pyd +0 -0
  6. mindspore/_checkparam.py +40 -9
  7. mindspore/{_deprecated → _extends/optimize}/__init__.py +9 -3
  8. mindspore/_extends/optimize/cell_utils.py +96 -0
  9. mindspore/_extends/parse/__init__.py +2 -2
  10. mindspore/_extends/parse/compile_config.py +44 -22
  11. mindspore/_extends/parse/deprecated/deprecated_tensor_method.py +1 -1
  12. mindspore/_extends/parse/parser.py +37 -62
  13. mindspore/_extends/parse/resources.py +39 -0
  14. mindspore/_extends/parse/standard_method.py +43 -13
  15. mindspore/_extends/parse/trope.py +8 -1
  16. mindspore/_extends/pijit/__init__.py +1 -2
  17. mindspore/amp.py +4 -4
  18. mindspore/avcodec-59.dll +0 -0
  19. mindspore/avdevice-59.dll +0 -0
  20. mindspore/avfilter-8.dll +0 -0
  21. mindspore/avformat-59.dll +0 -0
  22. mindspore/avutil-57.dll +0 -0
  23. mindspore/boost/adasum.py +1 -1
  24. mindspore/boost/boost_cell_wrapper.py +4 -4
  25. mindspore/common/__init__.py +27 -2
  26. mindspore/common/_grad_function.py +2 -1
  27. mindspore/common/_pijit_context.py +28 -7
  28. mindspore/common/_stub_tensor.py +1 -209
  29. mindspore/common/_tensor_cpp_method.py +1 -1
  30. mindspore/common/_tensor_docs.py +77 -16
  31. mindspore/common/api.py +238 -113
  32. mindspore/common/dtype.py +21 -11
  33. mindspore/common/dump.py +10 -15
  34. mindspore/common/generator.py +5 -3
  35. mindspore/common/hook_handle.py +11 -2
  36. mindspore/common/jit_config.py +1 -1
  37. mindspore/common/jit_trace.py +84 -105
  38. mindspore/common/parameter.py +26 -12
  39. mindspore/common/recompute.py +3 -3
  40. mindspore/common/sparse_tensor.py +0 -3
  41. mindspore/common/symbol.py +0 -1
  42. mindspore/common/tensor.py +81 -81
  43. mindspore/communication/_comm_helper.py +46 -4
  44. mindspore/communication/management.py +79 -7
  45. mindspore/context.py +58 -40
  46. mindspore/dataset/core/config.py +3 -3
  47. mindspore/dataset/engine/datasets.py +20 -7
  48. mindspore/dataset/engine/datasets_user_defined.py +33 -3
  49. mindspore/dataset/engine/iterators.py +2 -2
  50. mindspore/dataset/engine/obs/config_loader.py +2 -2
  51. mindspore/dataset/engine/obs/obs_mindrecord_dataset.py +8 -0
  52. mindspore/dataset/transforms/py_transforms.py +7 -3
  53. mindspore/dataset/transforms/transforms.py +7 -3
  54. mindspore/dataset/vision/validators.py +1 -0
  55. mindspore/device_context/ascend/device.py +1 -1
  56. mindspore/device_context/gpu/__init__.py +2 -2
  57. mindspore/device_context/gpu/device.py +1 -1
  58. mindspore/device_context/gpu/op_precision.py +4 -2
  59. mindspore/device_context/gpu/op_tuning.py +6 -3
  60. mindspore/device_manager.py +16 -9
  61. mindspore/dnnl.dll +0 -0
  62. mindspore/experimental/llm_boost/ascend_native/llama_boost_ascend_native.py +3 -7
  63. mindspore/experimental/llm_boost/atb/boost_base.py +2 -3
  64. mindspore/experimental/optim/adadelta.py +13 -20
  65. mindspore/experimental/optim/adagrad.py +15 -22
  66. mindspore/experimental/optim/adam.py +17 -24
  67. mindspore/experimental/optim/adamax.py +14 -22
  68. mindspore/experimental/optim/adamw.py +28 -34
  69. mindspore/experimental/optim/asgd.py +15 -25
  70. mindspore/experimental/optim/lr_scheduler.py +27 -45
  71. mindspore/experimental/optim/nadam.py +14 -24
  72. mindspore/experimental/optim/optimizer.py +13 -23
  73. mindspore/experimental/optim/radam.py +18 -24
  74. mindspore/experimental/optim/rmsprop.py +14 -25
  75. mindspore/experimental/optim/rprop.py +15 -26
  76. mindspore/experimental/optim/sgd.py +9 -19
  77. mindspore/hal/__init__.py +4 -4
  78. mindspore/hal/contiguous_tensors_handle.py +2 -2
  79. mindspore/hal/memory.py +27 -7
  80. mindspore/include/api/cell.h +37 -1
  81. mindspore/include/api/delegate.h +10 -0
  82. mindspore/include/api/model.h +3 -0
  83. mindspore/include/api/types.h +2 -2
  84. mindspore/include/c_api/model_c.h +0 -58
  85. mindspore/include/c_api/tensor_c.h +0 -26
  86. mindspore/include/dataset/vision_ascend.h +1 -1
  87. mindspore/jpeg62.dll +0 -0
  88. mindspore/mindrecord/tools/cifar10.py +60 -11
  89. mindspore/mindrecord/tools/cifar10_to_mr.py +5 -0
  90. mindspore/mindspore_backend_common.dll +0 -0
  91. mindspore/mindspore_backend_manager.dll +0 -0
  92. mindspore/mindspore_common.dll +0 -0
  93. mindspore/mindspore_core.dll +0 -0
  94. mindspore/mindspore_cpu_res_manager.dll +0 -0
  95. mindspore/mindspore_dump.dll +0 -0
  96. mindspore/mindspore_frontend.dll +0 -0
  97. mindspore/mindspore_glog.dll +0 -0
  98. mindspore/mindspore_memory_pool.dll +0 -0
  99. mindspore/mindspore_ms_backend.dll +0 -0
  100. mindspore/mindspore_ops.dll +0 -0
  101. mindspore/mindspore_ops_host.dll +0 -0
  102. mindspore/mindspore_ops_kernel_common.dll +0 -0
  103. mindspore/mindspore_profiler.dll +0 -0
  104. mindspore/mindspore_pyboost.dll +0 -0
  105. mindspore/mindspore_pynative.dll +0 -0
  106. mindspore/mindspore_res_manager.dll +0 -0
  107. mindspore/mindspore_runtime_pipeline.dll +0 -0
  108. mindspore/mint/__init__.py +6 -46
  109. mindspore/mint/distributed/__init__.py +1 -0
  110. mindspore/mint/distributed/distributed.py +212 -9
  111. mindspore/mint/nn/__init__.py +1 -1
  112. mindspore/mint/nn/functional.py +53 -6
  113. mindspore/mint/nn/layer/_functions.py +164 -294
  114. mindspore/mint/nn/layer/activation.py +8 -6
  115. mindspore/mint/nn/layer/conv.py +137 -101
  116. mindspore/mint/nn/layer/normalization.py +8 -22
  117. mindspore/mint/optim/adam.py +19 -18
  118. mindspore/mint/optim/adamw.py +14 -8
  119. mindspore/mint/optim/sgd.py +5 -5
  120. mindspore/nn/cell.py +328 -502
  121. mindspore/nn/grad/cell_grad.py +11 -12
  122. mindspore/nn/layer/activation.py +32 -34
  123. mindspore/nn/layer/basic.py +67 -64
  124. mindspore/nn/layer/channel_shuffle.py +4 -4
  125. mindspore/nn/layer/combined.py +4 -2
  126. mindspore/nn/layer/conv.py +117 -110
  127. mindspore/nn/layer/dense.py +9 -7
  128. mindspore/nn/layer/embedding.py +50 -52
  129. mindspore/nn/layer/image.py +37 -39
  130. mindspore/nn/layer/math.py +111 -112
  131. mindspore/nn/layer/normalization.py +56 -44
  132. mindspore/nn/layer/pooling.py +58 -63
  133. mindspore/nn/layer/rnn_cells.py +33 -33
  134. mindspore/nn/layer/rnns.py +56 -56
  135. mindspore/nn/layer/thor_layer.py +74 -73
  136. mindspore/nn/layer/transformer.py +11 -1
  137. mindspore/nn/learning_rate_schedule.py +20 -20
  138. mindspore/nn/loss/loss.py +79 -81
  139. mindspore/nn/optim/adam.py +3 -3
  140. mindspore/nn/optim/adasum.py +2 -2
  141. mindspore/nn/optim/asgd.py +2 -0
  142. mindspore/nn/optim/optimizer.py +1 -1
  143. mindspore/nn/optim/thor.py +2 -2
  144. mindspore/nn/probability/distribution/exponential.py +2 -1
  145. mindspore/nn/probability/distribution/poisson.py +2 -1
  146. mindspore/nn/sparse/sparse.py +3 -3
  147. mindspore/nn/wrap/cell_wrapper.py +34 -37
  148. mindspore/nn/wrap/grad_reducer.py +37 -37
  149. mindspore/nn/wrap/loss_scale.py +72 -74
  150. mindspore/numpy/array_creations.py +5 -5
  151. mindspore/numpy/fft.py +1 -1
  152. mindspore/numpy/math_ops.py +5 -5
  153. mindspore/opencv_core452.dll +0 -0
  154. mindspore/opencv_imgcodecs452.dll +0 -0
  155. mindspore/opencv_imgproc452.dll +0 -0
  156. mindspore/ops/_grad_experimental/grad_comm_ops.py +51 -13
  157. mindspore/ops/_grad_experimental/grad_debug_ops.py +14 -0
  158. mindspore/ops/_vmap/vmap_array_ops.py +31 -13
  159. mindspore/ops/_vmap/vmap_nn_ops.py +8 -16
  160. mindspore/ops/auto_generate/cpp_create_prim_instance_helper.py +42 -11
  161. mindspore/ops/auto_generate/gen_extend_func.py +23 -141
  162. mindspore/ops/auto_generate/gen_ops_def.py +727 -321
  163. mindspore/ops/auto_generate/gen_ops_prim.py +1721 -984
  164. mindspore/ops/auto_generate/pyboost_inner_prim.py +31 -1
  165. mindspore/ops/composite/__init__.py +10 -0
  166. mindspore/ops/composite/base.py +8 -4
  167. mindspore/ops/composite/multitype_ops/__init__.py +12 -1
  168. mindspore/ops/composite/multitype_ops/_compile_utils.py +133 -109
  169. mindspore/ops/composite/multitype_ops/add_impl.py +70 -2
  170. mindspore/ops/composite/multitype_ops/div_impl.py +49 -0
  171. mindspore/ops/composite/multitype_ops/floordiv_impl.py +29 -0
  172. mindspore/ops/composite/multitype_ops/getitem_impl.py +11 -0
  173. mindspore/ops/composite/multitype_ops/mod_impl.py +5 -3
  174. mindspore/ops/composite/multitype_ops/mul_impl.py +49 -0
  175. mindspore/ops/composite/multitype_ops/setitem_impl.py +57 -0
  176. mindspore/ops/composite/multitype_ops/sub_impl.py +34 -0
  177. mindspore/ops/composite/multitype_ops/zeros_like_impl.py +14 -0
  178. mindspore/ops/function/__init__.py +3 -1
  179. mindspore/ops/function/_add_attr_func.py +11 -6
  180. mindspore/ops/function/array_func.py +9 -96
  181. mindspore/ops/function/debug_func.py +4 -3
  182. mindspore/ops/function/grad/grad_func.py +1 -1
  183. mindspore/ops/function/math_func.py +33 -540
  184. mindspore/ops/function/nn_func.py +28 -74
  185. mindspore/ops/function/other_func.py +4 -1
  186. mindspore/ops/function/random_func.py +44 -5
  187. mindspore/ops/function/vmap_func.py +2 -1
  188. mindspore/ops/functional.py +2 -3
  189. mindspore/ops/functional_overload.py +571 -6
  190. mindspore/ops/op_info_register.py +21 -0
  191. mindspore/ops/operations/__init__.py +16 -11
  192. mindspore/ops/operations/_custom_ops_utils.py +689 -34
  193. mindspore/ops/operations/_inner_ops.py +3 -6
  194. mindspore/ops/operations/_sequence_ops.py +1 -1
  195. mindspore/ops/operations/array_ops.py +2 -2
  196. mindspore/ops/operations/comm_ops.py +185 -26
  197. mindspore/ops/operations/custom_ops.py +294 -174
  198. mindspore/ops/operations/debug_ops.py +59 -4
  199. mindspore/ops/operations/image_ops.py +13 -13
  200. mindspore/ops/operations/manually_defined/ops_def.py +15 -16
  201. mindspore/ops/operations/math_ops.py +3 -4
  202. mindspore/ops/operations/nn_ops.py +7 -39
  203. mindspore/ops/primitive.py +6 -10
  204. mindspore/ops/tensor_method.py +47 -8
  205. mindspore/ops_generate/api/cpp_create_prim_instance_helper_generator.py +1 -1
  206. mindspore/ops_generate/api/functional_map_cpp_generator.py +10 -9
  207. mindspore/ops_generate/api/functions_cc_generator.py +58 -10
  208. mindspore/ops_generate/api/tensor_func_reg_cpp_generator.py +1 -1
  209. mindspore/ops_generate/common/base_generator.py +14 -0
  210. mindspore/ops_generate/common/gen_constants.py +8 -3
  211. mindspore/ops_generate/common/gen_utils.py +0 -19
  212. mindspore/ops_generate/common/op_proto.py +11 -4
  213. mindspore/ops_generate/common/template.py +88 -11
  214. mindspore/ops_generate/gen_ops.py +1 -1
  215. mindspore/ops_generate/op_def/lite_ops_cpp_generator.py +4 -4
  216. mindspore/ops_generate/op_def/ops_def_cc_generator.py +0 -3
  217. mindspore/ops_generate/op_def/ops_name_h_generator.py +0 -3
  218. mindspore/ops_generate/op_def/ops_primitive_h_generator.py +0 -4
  219. mindspore/ops_generate/op_def_py/op_prim_py_generator.py +5 -2
  220. mindspore/ops_generate/pyboost/auto_grad_impl_cc_generator.py +49 -8
  221. mindspore/ops_generate/pyboost/auto_grad_reg_cc_generator.py +2 -2
  222. mindspore/ops_generate/pyboost/gen_pyboost_func.py +31 -0
  223. mindspore/ops_generate/pyboost/op_template_parser.py +98 -72
  224. mindspore/ops_generate/pyboost/pyboost_functions_cpp_generator.py +70 -273
  225. mindspore/ops_generate/pyboost/pyboost_functions_h_generator.py +14 -6
  226. mindspore/ops_generate/pyboost/pyboost_functions_impl_cpp_generator.py +316 -0
  227. mindspore/ops_generate/pyboost/pyboost_functions_py_generator.py +1 -1
  228. mindspore/ops_generate/pyboost/pyboost_grad_function_cpp_generator.py +5 -3
  229. mindspore/ops_generate/pyboost/pyboost_inner_prim_generator.py +1 -1
  230. mindspore/ops_generate/pyboost/pyboost_internal_functions_cpp_generator.py +76 -0
  231. mindspore/ops_generate/pyboost/pyboost_internal_functions_h_generator.py +76 -0
  232. mindspore/ops_generate/pyboost/pyboost_internal_kernel_info_adapter_generator.py +125 -0
  233. mindspore/ops_generate/pyboost/pyboost_native_grad_functions_generator.py +4 -3
  234. mindspore/ops_generate/pyboost/pyboost_op_cpp_code_generator.py +348 -61
  235. mindspore/ops_generate/pyboost/pyboost_overload_functions_cpp_generator.py +1 -1
  236. mindspore/ops_generate/pyboost/pyboost_utils.py +118 -9
  237. mindspore/ops_generate/tensor_py_cc_generator.py +1 -24
  238. mindspore/parallel/_auto_parallel_context.py +11 -8
  239. mindspore/parallel/_cell_wrapper.py +113 -45
  240. mindspore/parallel/_parallel_serialization.py +1 -1
  241. mindspore/parallel/_ps_context.py +4 -6
  242. mindspore/parallel/_tensor.py +167 -12
  243. mindspore/parallel/_transformer/moe.py +1 -1
  244. mindspore/parallel/_transformer/transformer.py +13 -8
  245. mindspore/parallel/auto_parallel.py +14 -7
  246. mindspore/parallel/checkpoint_convert.py +3 -3
  247. mindspore/parallel/checkpoint_transform.py +11 -7
  248. mindspore/parallel/cluster/process_entity/_api.py +84 -48
  249. mindspore/parallel/cluster/process_entity/_utils.py +95 -7
  250. mindspore/parallel/cluster/run.py +43 -4
  251. mindspore/parallel/function/__init__.py +8 -1
  252. mindspore/parallel/function/reshard_func.py +6 -7
  253. mindspore/parallel/nn/__init__.py +15 -2
  254. mindspore/parallel/nn/parallel_cell_wrapper.py +9 -10
  255. mindspore/parallel/nn/parallel_grad_reducer.py +7 -6
  256. mindspore/parallel/shard.py +3 -4
  257. mindspore/parallel/transform_safetensors.py +463 -174
  258. mindspore/profiler/__init__.py +2 -1
  259. mindspore/profiler/analysis/parser/timeline_assembly_factory/ascend_timeline_assembler.py +7 -7
  260. mindspore/profiler/analysis/parser/timeline_assembly_factory/base_timeline_assembler.py +3 -0
  261. mindspore/profiler/analysis/parser/timeline_assembly_factory/trace_view_container.py +12 -6
  262. mindspore/profiler/analysis/parser/timeline_creator/cpu_op_timeline_creator.py +3 -3
  263. mindspore/profiler/analysis/parser/timeline_creator/fwk_timeline_creator.py +3 -3
  264. mindspore/profiler/analysis/parser/timeline_creator/msprof_timeline_creator.py +4 -4
  265. mindspore/profiler/analysis/parser/timeline_creator/scope_layer_timeline_creator.py +3 -3
  266. mindspore/profiler/analysis/parser/timeline_event/fwk_event.py +4 -1
  267. mindspore/profiler/analysis/parser/timeline_event/timeline_event_pool.py +2 -1
  268. mindspore/profiler/analysis/task_manager.py +1 -1
  269. mindspore/profiler/analysis/viewer/ascend_communication_viewer.py +5 -1
  270. mindspore/profiler/analysis/viewer/ascend_integrate_viewer.py +2 -1
  271. mindspore/profiler/analysis/viewer/ascend_op_memory_viewer.py +42 -22
  272. mindspore/profiler/analysis/viewer/ascend_step_trace_time_viewer.py +3 -2
  273. mindspore/profiler/analysis/viewer/ms_minddata_viewer.py +9 -5
  274. mindspore/profiler/analysis/viewer/ms_operator_details_viewer.py +132 -0
  275. mindspore/profiler/common/constant.py +16 -0
  276. mindspore/profiler/common/profiler_context.py +25 -27
  277. mindspore/profiler/common/profiler_info.py +0 -16
  278. mindspore/profiler/common/profiler_op_analyse.py +235 -0
  279. mindspore/profiler/common/profiler_output_path.py +23 -8
  280. mindspore/profiler/common/profiler_parameters.py +128 -35
  281. mindspore/profiler/dynamic_profile/__init__.py +0 -0
  282. mindspore/profiler/dynamic_profile/dynamic_monitor_proxy.py +39 -0
  283. mindspore/profiler/dynamic_profile/dynamic_profiler_config_context.py +666 -0
  284. mindspore/profiler/dynamic_profile/dynamic_profiler_utils.py +62 -0
  285. mindspore/profiler/dynamic_profiler.py +305 -314
  286. mindspore/profiler/envprofiler.py +12 -7
  287. mindspore/profiler/experimental_config.py +96 -6
  288. mindspore/profiler/mstx.py +33 -12
  289. mindspore/profiler/platform/__init__.py +2 -3
  290. mindspore/profiler/platform/npu_profiler.py +29 -19
  291. mindspore/profiler/profiler.py +35 -19
  292. mindspore/profiler/profiler_action_controller.py +64 -76
  293. mindspore/profiler/schedule.py +10 -4
  294. mindspore/rewrite/common/config.py +1 -0
  295. mindspore/rewrite/common/namer.py +1 -0
  296. mindspore/rewrite/common/namespace.py +1 -0
  297. mindspore/rewrite/node/node.py +31 -11
  298. mindspore/rewrite/parsers/assign_parser.py +1 -1
  299. mindspore/rewrite/symbol_tree/symbol_tree.py +1 -1
  300. mindspore/run_check/_check_version.py +7 -10
  301. mindspore/runtime/__init__.py +5 -5
  302. mindspore/runtime/event.py +10 -4
  303. mindspore/runtime/executor.py +60 -45
  304. mindspore/runtime/memory.py +30 -32
  305. mindspore/runtime/thread_bind_core.py +298 -164
  306. mindspore/safeguard/rewrite_obfuscation.py +12 -13
  307. mindspore/swresample-4.dll +0 -0
  308. mindspore/swscale-6.dll +0 -0
  309. mindspore/tinyxml2.dll +0 -0
  310. mindspore/train/_utils.py +14 -4
  311. mindspore/train/amp.py +43 -20
  312. mindspore/train/callback/__init__.py +5 -5
  313. mindspore/train/callback/_checkpoint.py +3 -6
  314. mindspore/train/callback/_flops_collector.py +1 -1
  315. mindspore/train/callback/_landscape.py +0 -1
  316. mindspore/train/callback/_train_fault_tolerance.py +97 -16
  317. mindspore/train/data_sink.py +11 -2
  318. mindspore/train/dataset_helper.py +9 -0
  319. mindspore/train/model.py +135 -55
  320. mindspore/train/serialization.py +133 -111
  321. mindspore/train/summary/summary_record.py +13 -2
  322. mindspore/turbojpeg.dll +0 -0
  323. mindspore/utils/__init__.py +3 -2
  324. mindspore/utils/dryrun.py +0 -6
  325. mindspore/utils/runtime_execution_order_check.py +163 -77
  326. mindspore/utils/sdc_detect.py +68 -0
  327. mindspore/utils/utils.py +6 -9
  328. mindspore/version.py +1 -1
  329. {mindspore-2.6.0rc1.dist-info → mindspore-2.7.0rc1.dist-info}/METADATA +5 -4
  330. {mindspore-2.6.0rc1.dist-info → mindspore-2.7.0rc1.dist-info}/RECORD +333 -371
  331. mindspore/_deprecated/jit.py +0 -198
  332. mindspore/experimental/es/__init__.py +0 -22
  333. mindspore/experimental/es/embedding_service.py +0 -891
  334. mindspore/experimental/es/embedding_service_layer.py +0 -581
  335. mindspore/profiler/parser/__init__.py +0 -14
  336. mindspore/profiler/parser/aicpu_data_parser.py +0 -272
  337. mindspore/profiler/parser/ascend_analysis/__init__.py +0 -14
  338. mindspore/profiler/parser/ascend_analysis/constant.py +0 -71
  339. mindspore/profiler/parser/ascend_analysis/file_manager.py +0 -180
  340. mindspore/profiler/parser/ascend_analysis/function_event.py +0 -185
  341. mindspore/profiler/parser/ascend_analysis/fwk_cann_parser.py +0 -136
  342. mindspore/profiler/parser/ascend_analysis/fwk_file_parser.py +0 -131
  343. mindspore/profiler/parser/ascend_analysis/msprof_timeline_parser.py +0 -104
  344. mindspore/profiler/parser/ascend_analysis/path_manager.py +0 -313
  345. mindspore/profiler/parser/ascend_analysis/profiler_info_parser.py +0 -123
  346. mindspore/profiler/parser/ascend_analysis/tlv_decoder.py +0 -86
  347. mindspore/profiler/parser/ascend_analysis/trace_event_manager.py +0 -75
  348. mindspore/profiler/parser/ascend_cluster_generator.py +0 -116
  349. mindspore/profiler/parser/ascend_communicate_generator.py +0 -314
  350. mindspore/profiler/parser/ascend_flops_generator.py +0 -116
  351. mindspore/profiler/parser/ascend_fpbp_generator.py +0 -82
  352. mindspore/profiler/parser/ascend_hccl_generator.py +0 -271
  353. mindspore/profiler/parser/ascend_integrate_generator.py +0 -42
  354. mindspore/profiler/parser/ascend_memory_generator.py +0 -185
  355. mindspore/profiler/parser/ascend_msprof_exporter.py +0 -282
  356. mindspore/profiler/parser/ascend_msprof_generator.py +0 -187
  357. mindspore/profiler/parser/ascend_op_generator.py +0 -334
  358. mindspore/profiler/parser/ascend_steptrace_generator.py +0 -94
  359. mindspore/profiler/parser/ascend_timeline_generator.py +0 -545
  360. mindspore/profiler/parser/base_timeline_generator.py +0 -483
  361. mindspore/profiler/parser/container.py +0 -229
  362. mindspore/profiler/parser/cpu_gpu_timeline_generator.py +0 -697
  363. mindspore/profiler/parser/flops_parser.py +0 -531
  364. mindspore/profiler/parser/framework_enum.py +0 -111
  365. mindspore/profiler/parser/framework_parser.py +0 -464
  366. mindspore/profiler/parser/framework_struct.py +0 -61
  367. mindspore/profiler/parser/gpu_analysis/__init__.py +0 -14
  368. mindspore/profiler/parser/gpu_analysis/function_event.py +0 -44
  369. mindspore/profiler/parser/gpu_analysis/fwk_file_parser.py +0 -89
  370. mindspore/profiler/parser/gpu_analysis/profiler_info_parser.py +0 -72
  371. mindspore/profiler/parser/hccl_parser.py +0 -573
  372. mindspore/profiler/parser/hwts_log_parser.py +0 -122
  373. mindspore/profiler/parser/integrator.py +0 -526
  374. mindspore/profiler/parser/memory_usage_parser.py +0 -277
  375. mindspore/profiler/parser/minddata_analyzer.py +0 -800
  376. mindspore/profiler/parser/minddata_parser.py +0 -186
  377. mindspore/profiler/parser/minddata_pipeline_parser.py +0 -299
  378. mindspore/profiler/parser/op_intermediate_parser.py +0 -149
  379. mindspore/profiler/parser/optime_parser.py +0 -250
  380. mindspore/profiler/parser/profiler_info.py +0 -213
  381. mindspore/profiler/parser/step_trace_parser.py +0 -666
  382. {mindspore-2.6.0rc1.dist-info → mindspore-2.7.0rc1.dist-info}/WHEEL +0 -0
  383. {mindspore-2.6.0rc1.dist-info → mindspore-2.7.0rc1.dist-info}/entry_points.txt +0 -0
  384. {mindspore-2.6.0rc1.dist-info → mindspore-2.7.0rc1.dist-info}/top_level.txt +0 -0
@@ -28,6 +28,7 @@ __all__ = [
28
28
  "ProfilerActivity",
29
29
  "AicoreMetrics",
30
30
  "ExportType",
31
+ "HostSystem",
31
32
  "DynamicProfilerMonitor",
32
33
  "mstx",
33
34
  "_ExperimentalConfig",
@@ -39,7 +40,7 @@ from mindspore.profiler.profiler import Profiler
39
40
  from mindspore.profiler.profiler import tensorboard_trace_handler
40
41
  from mindspore.profiler.schedule import Schedule as schedule
41
42
  from mindspore.profiler.envprofiler import EnvProfiler
42
- from mindspore.profiler.common.constant import ProfilerLevel, ProfilerActivity, AicoreMetrics, ExportType
43
+ from mindspore.profiler.common.constant import ProfilerLevel, ProfilerActivity, AicoreMetrics, ExportType, HostSystem
43
44
  from mindspore.profiler.dynamic_profiler import DynamicProfilerMonitor
44
45
  from mindspore.profiler.experimental_config import _ExperimentalConfig
45
46
  from mindspore.profiler.profiler import Profile as profile
@@ -17,8 +17,8 @@ from typing import List, Dict, Any
17
17
  from decimal import Decimal
18
18
  from collections import defaultdict
19
19
 
20
- from mindspore import context
21
20
  from mindspore import log as logger
21
+ from mindspore.profiler.common.log import ProfilerLogger
22
22
  from mindspore.profiler.common.constant import EventConstant, TimelineLayerName, ProfilerLevel, JitLevel
23
23
  from mindspore.profiler.analysis.parser.timeline_event.base_event import BaseEvent
24
24
  from mindspore.profiler.analysis.parser.timeline_event.timeline_event_pool import TimelineEventPool
@@ -39,9 +39,10 @@ class AscendTimelineAssembler(BaseTimelineAssembler):
39
39
  def __init__(self, **kwargs):
40
40
  super().__init__()
41
41
  self._profiler_level = kwargs.get("profiler_level")
42
- self._context_mode = kwargs.get("context_mode")
43
42
  self._jit_level = kwargs.get("jit_level")
44
43
  self._init_creators()
44
+ ProfilerLogger.init(kwargs.get("ascend_ms_dir"))
45
+ self._logger = ProfilerLogger.get_instance()
45
46
 
46
47
  def _init_creators(self):
47
48
  """Initialize trace creators."""
@@ -128,17 +129,16 @@ class AscendTimelineAssembler(BaseTimelineAssembler):
128
129
  """Create flow events between framework and hardware events."""
129
130
  acl_to_npu_flow_dict = self._msprof_creator.get_acl_to_npu_flow_dict()
130
131
  fwk_launch_op_list = self.trace_view_container.kernel_launch_op_event
131
- # The graph mode O2 does not have the flow from CANN to hardware at each step
132
+ # The GE backend does not have the flow from CANN to hardware at each step
132
133
  if not acl_to_npu_flow_dict and self._jit_level != JitLevel.GRAPH_LEVEL:
133
134
  logger.error("Cannot find connection between CANN layer and Ascend Hardware layer.")
134
135
  return []
135
- # The graph model O2 does not have "KernelLaunch" or "LaunchTask" keywords
136
+ # The GE backend does not have "KernelLaunch" or "LaunchTask" keywords
136
137
  if not fwk_launch_op_list and self._jit_level != JitLevel.GRAPH_LEVEL:
137
138
  logger.warning("Cannot find launch op in MindSpore framework.")
138
139
  return []
139
- if (set(acl_to_npu_flow_dict.keys()) != set(fwk_launch_op_list.keys()) and
140
- self._context_mode == context.PYNATIVE_MODE):
141
- logger.warning(
140
+ if set(acl_to_npu_flow_dict.keys()) != set(fwk_launch_op_list.keys()):
141
+ self._logger.warning(
142
142
  "The number of launch op threads in MindSpore framework is inconsistent with the CANN layer.")
143
143
 
144
144
  fwk_to_npu_flows = []
@@ -1,3 +1,6 @@
1
+ # Copyright 2025 Huawei Technologies Co., Ltd
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
1
4
  # you may not use this file except in compliance with the License.
2
5
  # You may obtain a copy of the License at
3
6
  #
@@ -1,3 +1,6 @@
1
+ # Copyright 2025 Huawei Technologies Co., Ltd
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
1
4
  # you may not use this file except in compliance with the License.
2
5
  # You may obtain a copy of the License at
3
6
  #
@@ -95,12 +98,15 @@ class TraceViewContainer:
95
98
  events = mindspore_pool.get_all_events()
96
99
 
97
100
  # Filter events that contain "ProfilerStep" and create a dictionary mapping (start_ts, end_ts) to step ID
98
- step_id_to_time_dict = dict(sorted(
99
- (
100
- event.name.split("#")[-1], (event.ts, event.dur + event.ts)
101
+ step_id_to_time_dict = dict(
102
+ sorted(
103
+ (
104
+ (event.name.split("#")[-1], (event.ts, event.dur + event.ts))
105
+ for event in events
106
+ if ProfilerStepNameConstant.PROFILER_STEP in event.name
107
+ ),
108
+ key=lambda item: item[1][0]
101
109
  )
102
- for event in events
103
- if ProfilerStepNameConstant.PROFILER_STEP in event.name
104
- ))
110
+ )
105
111
 
106
112
  return step_id_to_time_dict
@@ -31,15 +31,15 @@ class CpuOpTimelineCreator(BaseTimelineCreator):
31
31
  super().__init__()
32
32
  self.scope_data: List[CpuOpCompleteEvent] = []
33
33
 
34
- def create(self, cpu_info_lines: List[str]) -> None:
34
+ def create(self, data: List[str]) -> None:
35
35
  """Create timeline event pools from CPU info lines."""
36
- if not cpu_info_lines:
36
+ if not data:
37
37
  return
38
38
 
39
39
  pool = TimelineEventPool(EventConstant.CPU_OP_PID)
40
40
  self.event_pools[EventConstant.CPU_OP_PID] = pool
41
41
 
42
- self._create_base_events(pool, cpu_info_lines)
42
+ self._create_base_events(pool, data)
43
43
  self._create_meta_event(pool)
44
44
 
45
45
  def _create_base_events(self, pool: TimelineEventPool, cpu_info_lines: List[str]) -> None:
@@ -29,15 +29,15 @@ from mindspore.profiler.analysis.parser.timeline_event.fwk_event import (
29
29
  class FwkTimelineCreator(BaseTimelineCreator):
30
30
  """Create timeline event pools for framework operations."""
31
31
 
32
- def create(self, fwk_tlv_data: List[Dict]) -> None:
32
+ def create(self, data: List[Dict]) -> None:
33
33
  """Create timeline event pools from framework TLV data."""
34
- if not fwk_tlv_data:
34
+ if not data:
35
35
  return
36
36
 
37
37
  pool = TimelineEventPool(EventConstant.MINDSPORE_PID)
38
38
  self.event_pools[EventConstant.MINDSPORE_PID] = pool
39
39
 
40
- self._create_base_events(pool, fwk_tlv_data)
40
+ self._create_base_events(pool, data)
41
41
  self._create_meta_event(pool)
42
42
 
43
43
  def _create_base_events(self, pool: TimelineEventPool, fwk_tlv_data: List[Dict]) -> None:
@@ -35,12 +35,12 @@ class MsprofTimelineCreator(BaseTimelineCreator):
35
35
  self.msprof_timeline_raw_data = []
36
36
  self.acl_to_npu_flow_dict: Dict[int, Dict[str, List[MsprofCompleteEvent]]] = {}
37
37
 
38
- def create(self, msprof_timeline_data: List[Dict]) -> None:
38
+ def create(self, data: List[Dict]) -> None:
39
39
  """Create timeline event pools from MsProf timeline data."""
40
- if not msprof_timeline_data:
40
+ if not data:
41
41
  return
42
- self.msprof_timeline_raw_data = msprof_timeline_data
43
- flow_dict, complete_event_map = self._create_base_events(msprof_timeline_data)
42
+ self.msprof_timeline_raw_data = data
43
+ flow_dict, complete_event_map = self._create_base_events(data)
44
44
  self._create_acl_to_npu_flow_dict(flow_dict, complete_event_map)
45
45
 
46
46
  def _create_base_events(self, msprof_timeline_data: List[Dict]) -> Tuple[Dict, Dict]:
@@ -32,15 +32,15 @@ from mindspore.profiler.analysis.parser.timeline_event.scope_layer_event import
32
32
  class ScopeLayerTimelineCreator(BaseTimelineCreator):
33
33
  """Create timeline event pools for scope layer operations."""
34
34
 
35
- def create(self, event_list: List[BaseEvent]) -> None:
35
+ def create(self, data: List[BaseEvent]) -> None:
36
36
  """Create timeline event pools from scope layer events."""
37
- if not event_list:
37
+ if not data:
38
38
  return
39
39
 
40
40
  pool = TimelineEventPool(EventConstant.SCOPE_LAYER_PID)
41
41
  self.event_pools[EventConstant.SCOPE_LAYER_PID] = pool
42
42
 
43
- self._create_base_events(pool, event_list)
43
+ self._create_base_events(pool, data)
44
44
  self._create_meta_event(pool)
45
45
 
46
46
  def _create_base_events(self, pool: TimelineEventPool, event_list: List[BaseEvent]) -> None:
@@ -110,6 +110,7 @@ class ProfilerEvent(Enum):
110
110
  PYBOOST_MALLOC_INPUT = "PyBoostMallocInput"
111
111
  PYBOOST_MALLOC_OUTPUT = "PyBoostMallocOutput"
112
112
  PYBOOST_LAUNCH_ACLLNN = "PyBoostLaunchAclnn"
113
+ PYBOOST_LAUNCH_ATB = "PyBoostLaunchAtb"
113
114
  # pybind api
114
115
  PYNATIVE_NEW_GRAPH = "PyNativeNewGraph"
115
116
  PYNATIVE_END_GRAPH = "PyNativeEndGraph"
@@ -179,7 +180,9 @@ class FwkProfileDataField:
179
180
  The enum value string.
180
181
  """
181
182
  try:
182
- return list(enum_class)[index].value
183
+ # pylint: disable=protected-access
184
+ name = enum_class._member_names_[index]
185
+ return enum_class[name].value
183
186
  except IndexError:
184
187
  logger.warning(f"Invalid {enum_type} index: {index}")
185
188
  return enum_class.DEFAULT.value
@@ -86,7 +86,8 @@ class TimelineEventPool:
86
86
  self.start_to_end_events_pairs[flow_key] = {"start": [], "end": []}
87
87
  self.start_to_end_events_pairs[flow_key]["end"].append(event)
88
88
 
89
- def _get_events(self, event_dict: dict) -> List[BaseEvent]:
89
+ @staticmethod
90
+ def _get_events(event_dict: dict) -> List[BaseEvent]:
90
91
  """Helper function to get events from a dictionary."""
91
92
  events = []
92
93
  for event_list in event_dict.values():
@@ -110,8 +110,8 @@ class TaskManager:
110
110
  else workflow
111
111
  )
112
112
 
113
+ data = {}
113
114
  try:
114
- data = {}
115
115
  for parser in parsers:
116
116
  parser_start_time = time.perf_counter()
117
117
  data = parser.parse(data)
@@ -352,7 +352,11 @@ class AscendCommunicationViewer(BaseViewer):
352
352
  if bandwidth_msg in add_list:
353
353
  total_bandwidth_info_dict[transport_type][bandwidth_msg] += value
354
354
  if bandwidth_msg in dict_list:
355
- self._combine_size_distribution(value, total_bandwidth_info_dict[transport_type][bandwidth_msg])
355
+ self._combine_size_distribution(
356
+ value,
357
+ total_bandwidth_info_dict.get(transport_type, {})
358
+ .get(bandwidth_msg, defaultdict(lambda: [0, 0]))
359
+ )
356
360
 
357
361
  def _compute_time_ratio(self, total_time_info_dict: dict):
358
362
  """compute time ratio"""
@@ -25,7 +25,8 @@ from mindspore.profiler.common.log import ProfilerLogger
25
25
  class AscendIntegrateViewer(BaseViewer):
26
26
  """Ascend integrate viewer"""
27
27
 
28
- CSV_PREFIX_NAME = ["l2_cache", "api_statistic", "op_statistic", "static_op_mem"]
28
+ CSV_PREFIX_NAME = ["l2_cache", "api_statistic", "op_statistic", "static_op_mem",
29
+ "hbm", "nic", "roce", "pcie", "hccs"]
29
30
  AI_CPU_CSV_PATTERN = "aicpu_[0-9]*.csv"
30
31
 
31
32
  def __init__(self, **kwargs):
@@ -20,10 +20,10 @@ from enum import Enum
20
20
  from typing import Any, Dict, List
21
21
  from abc import ABC
22
22
 
23
- from mindspore.profiler.parser.ascend_analysis.tlv_decoder import TLVDecoder
23
+ from mindspore.profiler.common.tlv_decoder import TLVDecoder
24
24
  from mindspore.profiler.common.file_manager import FileManager
25
25
  from mindspore.profiler.common.log import ProfilerLogger
26
- from mindspore.profiler.common.constant import ProfilerActivity
26
+ from mindspore.profiler.common.constant import ProfilerActivity, FileConstant
27
27
 
28
28
 
29
29
  class OpMemoryIndexEnum(Enum):
@@ -68,9 +68,7 @@ class OpMemoryEvent(BaseEvent):
68
68
 
69
69
  def __init__(self, data: Dict):
70
70
  super().__init__(data)
71
- self.fix_size_data = struct.unpack(
72
- self.FIX_DATA_FORMAT, self._origin_data.get("fix_size_bytes")
73
- )
71
+ self.fix_size_data = self._origin_data[FileConstant.FIX_SIZE_DATA]
74
72
 
75
73
  @property
76
74
  def device_id(self):
@@ -237,9 +235,10 @@ class AscendOpMemoryViewer:
237
235
  self._logger.info("Read fwk binary file start")
238
236
  op_name_file_path = os.path.join(self._framework_path, self.FWK_BINARY_FILE_NAME)
239
237
  raw_bin_data = FileManager.read_file_content(op_name_file_path, mode="rb")
240
- self._op_memory_events = TLVDecoder.decode(
241
- raw_bin_data, OpMemoryEvent, OpMemoryEvent.FIX_DATA_SIZE
238
+ op_memory_decode_data = TLVDecoder.decode(
239
+ raw_bin_data, OpMemoryEvent.FIX_DATA_FORMAT, OpMemoryEvent.FIX_DATA_SIZE
242
240
  )
241
+ self._op_memory_events = [OpMemoryEvent(data) for data in op_memory_decode_data]
243
242
  self._op_memory_events = sorted(self._op_memory_events, key=lambda x: x.create_at)
244
243
  self._logger.info("Read fwk binary file done, %d events", len(self._op_memory_events))
245
244
 
@@ -271,21 +270,42 @@ class AscendOpMemoryViewer:
271
270
  return []
272
271
 
273
272
  return [
274
- alloc_event.owner, # "Name"
275
- alloc_event.size / self.BYTES_TO_KB, # "Size(KB)"
276
- alloc_event.create_at / self.NS_TO_US, # "Allocation Time(us)"
277
- self.EMPTY_VALUE if free_event is None else free_event.create_at / self.NS_TO_US, # "Release Time(us)"
278
- self.EMPTY_VALUE, # "Active Release Time(us)"
279
- self.EMPTY_VALUE if free_event is None else (free_event.create_at - alloc_event.create_at) / self.NS_TO_US, # "Duration(us)"
280
- self.EMPTY_VALUE, # "Active Duration(us)"
281
- alloc_event.alloc_size / self.BYTES_TO_MB, # "Allocation Total Allocated(MB)"
282
- alloc_event.used_size / self.BYTES_TO_MB, # "Allocation Total Reserved(MB)"
283
- alloc_event.alloc_size / self.BYTES_TO_MB, # "Allocation Total Active(MB)"
284
- self.EMPTY_VALUE if free_event is None else free_event.alloc_size / self.BYTES_TO_MB, # "Release Total Allocated(MB)"
285
- self.EMPTY_VALUE if free_event is None else free_event.used_size / self.BYTES_TO_MB, # "Release Total Reserved(MB)"
286
- self.EMPTY_VALUE if free_event is None else free_event.alloc_size / self.BYTES_TO_MB, # "Release Total Active(MB)"
287
- alloc_event.stream_ptr, # "Stream Ptr"
288
- self.DEVICE_TYPE_FMT.format(alloc_event.device_id), # "Device Type"
273
+ alloc_event.owner, # "Name"
274
+ alloc_event.size / self.BYTES_TO_KB, # "Size(KB)"
275
+ alloc_event.create_at / self.NS_TO_US, # "Allocation Time(us)"
276
+ (
277
+ self.EMPTY_VALUE
278
+ if free_event is None
279
+ else free_event.create_at / self.NS_TO_US
280
+ ), # "Release Time(us)"
281
+ self.EMPTY_VALUE, # "Active Release Time(us)"
282
+ (
283
+ self.EMPTY_VALUE
284
+ if free_event is None
285
+ else (free_event.create_at - alloc_event.create_at) / self.NS_TO_US
286
+ ), # "Duration(us)"
287
+ self.EMPTY_VALUE, # "Active Duration(us)"
288
+ alloc_event.used_size
289
+ / self.BYTES_TO_MB, # "Allocation Total Allocated(MB)"
290
+ alloc_event.alloc_size / self.BYTES_TO_MB, # "Allocation Total Reserved(MB)"
291
+ alloc_event.used_size / self.BYTES_TO_MB, # "Allocation Total Active(MB)"
292
+ (
293
+ self.EMPTY_VALUE
294
+ if free_event is None
295
+ else free_event.used_size / self.BYTES_TO_MB
296
+ ), # "Release Total Allocated(MB)"
297
+ (
298
+ self.EMPTY_VALUE
299
+ if free_event is None
300
+ else free_event.alloc_size / self.BYTES_TO_MB
301
+ ), # "Release Total Reserved(MB)"
302
+ (
303
+ self.EMPTY_VALUE
304
+ if free_event is None
305
+ else free_event.used_size / self.BYTES_TO_MB
306
+ ), # "Release Total Active(MB)"
307
+ alloc_event.stream_ptr, # "Stream Ptr"
308
+ self.DEVICE_TYPE_FMT.format(alloc_event.device_id), # "Device Type"
289
309
  ]
290
310
 
291
311
  def _get_op_mem_row_data(self, event_list: List[OpMemoryEvent]):
@@ -260,7 +260,7 @@ class AscendStepTraceTimeViewer(BaseViewer):
260
260
  # communication not overlapped time exclude receive
261
261
  step_trace_time_data[StepTraceTimeHeaders.COMMUNICATION_NOT_OVERLAPPED_EXCLUDE_RECEIVE.value] = (
262
262
  step_trace_time_data[StepTraceTimeHeaders.COMMUNICATION_NOT_OVERLAPPED.value]
263
- - step_trace_time_data[StepTraceTimeHeaders.BUBBLE.value]
263
+ - step_trace_time_data.get(StepTraceTimeHeaders.BUBBLE.value, Decimal('0.000'))
264
264
  )
265
265
  step_trace_time_data[StepTraceTimeHeaders.PREPARING.value] = self._calculate_prepare_time_by_step(
266
266
  self.computing_np, self.communication_np, start_time, step_id
@@ -355,7 +355,8 @@ class AscendStepTraceTimeViewer(BaseViewer):
355
355
  logger.info("No HCCL events in the given time range, skip calculate stage and bubble")
356
356
  return Decimal(0), Decimal(0)
357
357
 
358
- total_hccl_time = filtered_hccl_events_np["ts"][-1] - filtered_hccl_events_np["ts"][0]
358
+ total_hccl_time = filtered_hccl_events_np["ts"][-1] - filtered_hccl_events_np["ts"][0] + \
359
+ filtered_hccl_events_np["dur"][-1]
359
360
  bubble_time = np.sum(
360
361
  filtered_hccl_events_np["dur"][
361
362
  np.array(
@@ -367,11 +367,15 @@ class MindDataPiplineSummaryViewer(BaseViewer):
367
367
  raise ProfilerRawFileException('The contents of MindData CPU utilization JSON file is wrong.')
368
368
  # Note: The CPU utilization data may have an extra entry with op_id=-1
369
369
  # Omit info for op_id=1
370
- dict_opid_cpuutil = {
371
- op["op_id"]: [op_sys + op_usr for op_sys, op_usr in
372
- zip(op["metrics"]["sys_utilization"], op["metrics"]["user_utilization"])]
373
- for op in cpu_op_info if op and op["op_id"] != -1
374
- }
370
+ dict_opid_cpuutil = {}
371
+ for op in cpu_op_info:
372
+ if not op or op["op_id"] == -1:
373
+ continue
374
+ cpu_utilization = []
375
+ for op_sys, op_usr in zip(op["metrics"]["sys_utilization"], op["metrics"]["user_utilization"]):
376
+ cpu_utilization.append(op_sys + op_usr)
377
+ dict_opid_cpuutil[op["op_id"]] = cpu_utilization
378
+
375
379
  # Initialize oplist_avg_cpu_pct with -1 for each pipeline op, since
376
380
  # CPU utilization data may not have information for each pipeline op
377
381
  oplist_avg_cpu_pct = [-1] * len(dict_opid_cpuutil)
@@ -0,0 +1,132 @@
1
+ # Copyright 2025 Huawei Technologies Co., Ltd
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ # ============================================================================
15
+ """ms operator details viewer"""
16
+ import os
17
+ import struct
18
+ from abc import ABC
19
+ from enum import Enum
20
+ from typing import Dict, Any
21
+
22
+ from mindspore.profiler.common.file_manager import FileManager
23
+ from mindspore.profiler.analysis.viewer.base_viewer import BaseViewer
24
+ from mindspore.profiler.common.tlv_decoder import TLVDecoder
25
+ from mindspore.profiler.common.log import ProfilerLogger
26
+
27
+
28
+ class OperatorDetailsIndexEnum(Enum):
29
+ """Operator details index defining."""
30
+
31
+ NAME = 0
32
+ INPUT_SHAPES = 1
33
+ INPUT_TYPE = 2
34
+
35
+
36
+ class BaseEvent(ABC):
37
+ """Base class for all event types."""
38
+
39
+ def __init__(self, data: Dict):
40
+ if not isinstance(data, dict):
41
+ raise TypeError("Input data must be dict.")
42
+ self._origin_data = data
43
+
44
+
45
+ class OperatorDetailsEvent(BaseEvent):
46
+ """Operator details event."""
47
+
48
+ FIX_DATA_FORMAT = ""
49
+ FIX_DATA_SIZE = struct.calcsize(FIX_DATA_FORMAT)
50
+
51
+ @property
52
+ def name(self):
53
+ """Get name."""
54
+ return self._origin_data.get(OperatorDetailsIndexEnum.NAME.value, "")
55
+
56
+ @property
57
+ def input_shapes(self):
58
+ """Get input_shapes."""
59
+ return self._origin_data.get(OperatorDetailsIndexEnum.INPUT_SHAPES.value, "")
60
+
61
+ @property
62
+ def input_type(self):
63
+ """Get input_type."""
64
+ return self._origin_data.get(OperatorDetailsIndexEnum.INPUT_TYPE.value, "")
65
+
66
+
67
+ class MsOperatorDetailsViewer(BaseViewer):
68
+ """Viewer for MindSpore operator_details profiling data."""
69
+
70
+ FWK_BINARY_FILE_NAME = "mindspore.record_shapes"
71
+ _OPERATOR_DETAILS_FILE_NAME = 'operator_details.csv'
72
+ _COL_NAMES = ['Name', 'Input Shapes']
73
+
74
+ def __init__(self, **kwargs):
75
+ super().__init__()
76
+ self._save_path = os.path.join(
77
+ kwargs.get("ascend_profiler_output_path"),
78
+ self._OPERATOR_DETAILS_FILE_NAME
79
+ )
80
+ self._ascend_ms_dir = kwargs.get("ascend_ms_dir")
81
+ self._framework_path = kwargs.get("framework_path")
82
+ ProfilerLogger.init(self._ascend_ms_dir)
83
+ self._logger = ProfilerLogger.get_instance()
84
+
85
+ def save(self, data: Dict[str, Any]) -> None:
86
+ """Process and save operator_details profiling data."""
87
+ self._logger.info("MsOperatorDetailsViewer start")
88
+ try:
89
+ file_exist = self._read_fwk_binary_file()
90
+ if not file_exist:
91
+ return
92
+ self._calculate_operator_details_data()
93
+ self._write_data()
94
+ except Exception as e: # pylint: disable=W0703
95
+ self._logger.error("Failed to save operator_details.csv: %s", e, exc_info=True)
96
+ self._logger.info("MsOperatorDetailsViewer end")
97
+
98
+ def _read_fwk_binary_file(self):
99
+ """
100
+ Read fwk binary file
101
+ """
102
+ self._logger.info("Read fwk binary file start")
103
+ fwk_file_path = os.path.join(self._framework_path, self.FWK_BINARY_FILE_NAME)
104
+ if not os.path.isfile(fwk_file_path):
105
+ self._logger.warning("Fwk binary file %s does not exist.", fwk_file_path)
106
+ return False
107
+ raw_bin_data = FileManager.read_file_content(fwk_file_path, mode="rb")
108
+ operator_details_decode_data = TLVDecoder.decode(
109
+ raw_bin_data, OperatorDetailsEvent.FIX_DATA_FORMAT, OperatorDetailsEvent.FIX_DATA_SIZE
110
+ )
111
+ self._operator_details_events = [OperatorDetailsEvent(data) for data in operator_details_decode_data]
112
+ self._logger.info("Read fwk binary file done, %d events", len(self._operator_details_events))
113
+ return True
114
+
115
+ def _calculate_operator_details_data(self):
116
+ """
117
+ Calculate operator details data
118
+ """
119
+ self._operator_details_data = [
120
+ [event.name, event.input_shapes]
121
+ for event in self._operator_details_events
122
+ ]
123
+
124
+ def _write_data(self) -> None:
125
+ """
126
+ Save operator statistics to a CSV file
127
+ """
128
+ if not self._operator_details_data:
129
+ return
130
+ self._logger.info("Save operator statistics start")
131
+ FileManager.create_csv_file(self._save_path, self._operator_details_data, self._COL_NAMES)
132
+ self._logger.info("Save operator statistics done")
@@ -186,6 +186,8 @@ class TimelineLayerName(Enum):
186
186
  ACC_PMU = "Acc PMU"
187
187
  SIO = "SIO"
188
188
  QOS = "QoS"
189
+ NIC = "NIC"
190
+ ROCE = "RoCE"
189
191
  OVERLAP_ANALYSIS = "Overlap Analysis"
190
192
 
191
193
 
@@ -210,3 +212,17 @@ class ExportType(Enum):
210
212
  class CannLibName:
211
213
  """CANN lib name"""
212
214
  CANN_MSPTI = "libmspti.so"
215
+
216
+
217
+ class DynoMode:
218
+ """dyno mode"""
219
+ DYNO_DAEMON = "MSMONITOR_USE_DAEMON"
220
+
221
+
222
+ class HostSystem(Enum):
223
+ """host system"""
224
+ CPU = "cpu"
225
+ MEM = "mem"
226
+ DISK = "disk"
227
+ NETWORK = "network"
228
+ OSRT = "osrt"