mindspore 2.6.0rc1__cp310-cp310-win_amd64.whl → 2.7.0rc1__cp310-cp310-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mindspore might be problematic. Click here for more details.

Files changed (407) hide show
  1. mindspore/.commit_id +1 -1
  2. mindspore/Microsoft.VisualStudio.Telemetry.dll +0 -0
  3. mindspore/Newtonsoft.Json.dll +0 -0
  4. mindspore/__init__.py +1 -1
  5. mindspore/_c_dataengine.cp310-win_amd64.pyd +0 -0
  6. mindspore/_c_expression.cp310-win_amd64.pyd +0 -0
  7. mindspore/_c_mindrecord.cp310-win_amd64.pyd +0 -0
  8. mindspore/_checkparam.py +40 -9
  9. mindspore/{_deprecated → _extends/optimize}/__init__.py +9 -3
  10. mindspore/_extends/optimize/cell_utils.py +96 -0
  11. mindspore/_extends/parse/__init__.py +2 -2
  12. mindspore/_extends/parse/compile_config.py +44 -22
  13. mindspore/_extends/parse/deprecated/deprecated_tensor_method.py +1 -1
  14. mindspore/_extends/parse/parser.py +37 -62
  15. mindspore/_extends/parse/resources.py +39 -0
  16. mindspore/_extends/parse/standard_method.py +43 -13
  17. mindspore/_extends/parse/trope.py +8 -1
  18. mindspore/_extends/pijit/__init__.py +1 -2
  19. mindspore/amp.py +4 -4
  20. mindspore/atlprov.dll +0 -0
  21. mindspore/avcodec-59.dll +0 -0
  22. mindspore/avdevice-59.dll +0 -0
  23. mindspore/avfilter-8.dll +0 -0
  24. mindspore/avformat-59.dll +0 -0
  25. mindspore/avutil-57.dll +0 -0
  26. mindspore/boost/adasum.py +1 -1
  27. mindspore/boost/boost_cell_wrapper.py +4 -4
  28. mindspore/c1.dll +0 -0
  29. mindspore/c1xx.dll +0 -0
  30. mindspore/c2.dll +0 -0
  31. mindspore/common/__init__.py +27 -2
  32. mindspore/common/_grad_function.py +2 -1
  33. mindspore/common/_pijit_context.py +28 -7
  34. mindspore/common/_stub_tensor.py +1 -209
  35. mindspore/common/_tensor_cpp_method.py +1 -1
  36. mindspore/common/_tensor_docs.py +77 -16
  37. mindspore/common/api.py +238 -113
  38. mindspore/common/dtype.py +21 -11
  39. mindspore/common/dump.py +10 -15
  40. mindspore/common/generator.py +5 -3
  41. mindspore/common/hook_handle.py +11 -2
  42. mindspore/common/jit_config.py +1 -1
  43. mindspore/common/jit_trace.py +84 -105
  44. mindspore/common/parameter.py +26 -12
  45. mindspore/common/recompute.py +3 -3
  46. mindspore/common/sparse_tensor.py +0 -3
  47. mindspore/common/symbol.py +0 -1
  48. mindspore/common/tensor.py +81 -81
  49. mindspore/communication/_comm_helper.py +46 -4
  50. mindspore/communication/management.py +79 -7
  51. mindspore/context.py +58 -40
  52. mindspore/dataset/core/config.py +3 -3
  53. mindspore/dataset/engine/datasets.py +20 -7
  54. mindspore/dataset/engine/datasets_user_defined.py +33 -3
  55. mindspore/dataset/engine/iterators.py +2 -2
  56. mindspore/dataset/engine/obs/config_loader.py +2 -2
  57. mindspore/dataset/engine/obs/obs_mindrecord_dataset.py +8 -0
  58. mindspore/dataset/transforms/py_transforms.py +7 -3
  59. mindspore/dataset/transforms/transforms.py +7 -3
  60. mindspore/dataset/vision/validators.py +1 -0
  61. mindspore/device_context/ascend/device.py +1 -1
  62. mindspore/device_context/gpu/__init__.py +2 -2
  63. mindspore/device_context/gpu/device.py +1 -1
  64. mindspore/device_context/gpu/op_precision.py +4 -2
  65. mindspore/device_context/gpu/op_tuning.py +6 -3
  66. mindspore/device_manager.py +16 -9
  67. mindspore/dnnl.dll +0 -0
  68. mindspore/dpcmi.dll +0 -0
  69. mindspore/experimental/llm_boost/ascend_native/llama_boost_ascend_native.py +3 -7
  70. mindspore/experimental/llm_boost/atb/boost_base.py +2 -3
  71. mindspore/experimental/optim/adadelta.py +13 -20
  72. mindspore/experimental/optim/adagrad.py +15 -22
  73. mindspore/experimental/optim/adam.py +17 -24
  74. mindspore/experimental/optim/adamax.py +14 -22
  75. mindspore/experimental/optim/adamw.py +28 -34
  76. mindspore/experimental/optim/asgd.py +15 -25
  77. mindspore/experimental/optim/lr_scheduler.py +27 -45
  78. mindspore/experimental/optim/nadam.py +14 -24
  79. mindspore/experimental/optim/optimizer.py +13 -23
  80. mindspore/experimental/optim/radam.py +18 -24
  81. mindspore/experimental/optim/rmsprop.py +14 -25
  82. mindspore/experimental/optim/rprop.py +15 -26
  83. mindspore/experimental/optim/sgd.py +9 -19
  84. mindspore/hal/__init__.py +4 -4
  85. mindspore/hal/contiguous_tensors_handle.py +2 -2
  86. mindspore/hal/memory.py +27 -7
  87. mindspore/include/api/cell.h +37 -1
  88. mindspore/include/api/delegate.h +10 -0
  89. mindspore/include/api/model.h +3 -0
  90. mindspore/include/api/types.h +2 -2
  91. mindspore/include/c_api/model_c.h +0 -58
  92. mindspore/include/c_api/tensor_c.h +0 -26
  93. mindspore/include/dataset/vision_ascend.h +1 -1
  94. mindspore/jpeg62.dll +0 -0
  95. mindspore/mindrecord/tools/cifar10.py +60 -11
  96. mindspore/mindrecord/tools/cifar10_to_mr.py +5 -0
  97. mindspore/mindspore_backend_common.dll +0 -0
  98. mindspore/mindspore_backend_manager.dll +0 -0
  99. mindspore/mindspore_common.dll +0 -0
  100. mindspore/mindspore_core.dll +0 -0
  101. mindspore/mindspore_cpu_res_manager.dll +0 -0
  102. mindspore/mindspore_dump.dll +0 -0
  103. mindspore/mindspore_frontend.dll +0 -0
  104. mindspore/mindspore_glog.dll +0 -0
  105. mindspore/mindspore_memory_pool.dll +0 -0
  106. mindspore/mindspore_ms_backend.dll +0 -0
  107. mindspore/mindspore_ops.dll +0 -0
  108. mindspore/mindspore_ops_host.dll +0 -0
  109. mindspore/mindspore_ops_kernel_common.dll +0 -0
  110. mindspore/mindspore_profiler.dll +0 -0
  111. mindspore/mindspore_pyboost.dll +0 -0
  112. mindspore/mindspore_pynative.dll +0 -0
  113. mindspore/mindspore_res_manager.dll +0 -0
  114. mindspore/mindspore_runtime_pipeline.dll +0 -0
  115. mindspore/mint/__init__.py +6 -46
  116. mindspore/mint/distributed/__init__.py +1 -0
  117. mindspore/mint/distributed/distributed.py +212 -9
  118. mindspore/mint/nn/__init__.py +1 -1
  119. mindspore/mint/nn/functional.py +53 -6
  120. mindspore/mint/nn/layer/_functions.py +164 -294
  121. mindspore/mint/nn/layer/activation.py +8 -6
  122. mindspore/mint/nn/layer/conv.py +137 -101
  123. mindspore/mint/nn/layer/normalization.py +8 -22
  124. mindspore/mint/optim/adam.py +19 -18
  125. mindspore/mint/optim/adamw.py +14 -8
  126. mindspore/mint/optim/sgd.py +5 -5
  127. mindspore/msobj140.dll +0 -0
  128. mindspore/mspdb140.dll +0 -0
  129. mindspore/mspdbcore.dll +0 -0
  130. mindspore/mspdbst.dll +0 -0
  131. mindspore/mspft140.dll +0 -0
  132. mindspore/msvcdis140.dll +0 -0
  133. mindspore/msvcp140_1.dll +0 -0
  134. mindspore/msvcp140_2.dll +0 -0
  135. mindspore/msvcp140_atomic_wait.dll +0 -0
  136. mindspore/msvcp140_codecvt_ids.dll +0 -0
  137. mindspore/nn/cell.py +328 -502
  138. mindspore/nn/grad/cell_grad.py +11 -12
  139. mindspore/nn/layer/activation.py +32 -34
  140. mindspore/nn/layer/basic.py +67 -64
  141. mindspore/nn/layer/channel_shuffle.py +4 -4
  142. mindspore/nn/layer/combined.py +4 -2
  143. mindspore/nn/layer/conv.py +117 -110
  144. mindspore/nn/layer/dense.py +9 -7
  145. mindspore/nn/layer/embedding.py +50 -52
  146. mindspore/nn/layer/image.py +37 -39
  147. mindspore/nn/layer/math.py +111 -112
  148. mindspore/nn/layer/normalization.py +56 -44
  149. mindspore/nn/layer/pooling.py +58 -63
  150. mindspore/nn/layer/rnn_cells.py +33 -33
  151. mindspore/nn/layer/rnns.py +56 -56
  152. mindspore/nn/layer/thor_layer.py +74 -73
  153. mindspore/nn/layer/transformer.py +11 -1
  154. mindspore/nn/learning_rate_schedule.py +20 -20
  155. mindspore/nn/loss/loss.py +79 -81
  156. mindspore/nn/optim/adam.py +3 -3
  157. mindspore/nn/optim/adasum.py +2 -2
  158. mindspore/nn/optim/asgd.py +2 -0
  159. mindspore/nn/optim/optimizer.py +1 -1
  160. mindspore/nn/optim/thor.py +2 -2
  161. mindspore/nn/probability/distribution/exponential.py +2 -1
  162. mindspore/nn/probability/distribution/poisson.py +2 -1
  163. mindspore/nn/sparse/sparse.py +3 -3
  164. mindspore/nn/wrap/cell_wrapper.py +34 -37
  165. mindspore/nn/wrap/grad_reducer.py +37 -37
  166. mindspore/nn/wrap/loss_scale.py +72 -74
  167. mindspore/numpy/array_creations.py +5 -5
  168. mindspore/numpy/fft.py +1 -1
  169. mindspore/numpy/math_ops.py +5 -5
  170. mindspore/opencv_core452.dll +0 -0
  171. mindspore/opencv_imgcodecs452.dll +0 -0
  172. mindspore/opencv_imgproc452.dll +0 -0
  173. mindspore/ops/_grad_experimental/grad_comm_ops.py +51 -13
  174. mindspore/ops/_grad_experimental/grad_debug_ops.py +14 -0
  175. mindspore/ops/_vmap/vmap_array_ops.py +31 -13
  176. mindspore/ops/_vmap/vmap_nn_ops.py +8 -16
  177. mindspore/ops/auto_generate/cpp_create_prim_instance_helper.py +42 -11
  178. mindspore/ops/auto_generate/gen_extend_func.py +23 -141
  179. mindspore/ops/auto_generate/gen_ops_def.py +727 -321
  180. mindspore/ops/auto_generate/gen_ops_prim.py +1721 -984
  181. mindspore/ops/auto_generate/pyboost_inner_prim.py +31 -1
  182. mindspore/ops/composite/__init__.py +10 -0
  183. mindspore/ops/composite/base.py +8 -4
  184. mindspore/ops/composite/multitype_ops/__init__.py +12 -1
  185. mindspore/ops/composite/multitype_ops/_compile_utils.py +133 -109
  186. mindspore/ops/composite/multitype_ops/add_impl.py +70 -2
  187. mindspore/ops/composite/multitype_ops/div_impl.py +49 -0
  188. mindspore/ops/composite/multitype_ops/floordiv_impl.py +29 -0
  189. mindspore/ops/composite/multitype_ops/getitem_impl.py +11 -0
  190. mindspore/ops/composite/multitype_ops/mod_impl.py +5 -3
  191. mindspore/ops/composite/multitype_ops/mul_impl.py +49 -0
  192. mindspore/ops/composite/multitype_ops/setitem_impl.py +57 -0
  193. mindspore/ops/composite/multitype_ops/sub_impl.py +34 -0
  194. mindspore/ops/composite/multitype_ops/zeros_like_impl.py +14 -0
  195. mindspore/ops/function/__init__.py +3 -1
  196. mindspore/ops/function/_add_attr_func.py +11 -6
  197. mindspore/ops/function/array_func.py +9 -96
  198. mindspore/ops/function/debug_func.py +4 -3
  199. mindspore/ops/function/grad/grad_func.py +1 -1
  200. mindspore/ops/function/math_func.py +33 -540
  201. mindspore/ops/function/nn_func.py +28 -74
  202. mindspore/ops/function/other_func.py +4 -1
  203. mindspore/ops/function/random_func.py +44 -5
  204. mindspore/ops/function/vmap_func.py +2 -1
  205. mindspore/ops/functional.py +2 -3
  206. mindspore/ops/functional_overload.py +571 -6
  207. mindspore/ops/op_info_register.py +21 -0
  208. mindspore/ops/operations/__init__.py +16 -11
  209. mindspore/ops/operations/_custom_ops_utils.py +689 -34
  210. mindspore/ops/operations/_inner_ops.py +3 -6
  211. mindspore/ops/operations/_sequence_ops.py +1 -1
  212. mindspore/ops/operations/array_ops.py +2 -2
  213. mindspore/ops/operations/comm_ops.py +185 -26
  214. mindspore/ops/operations/custom_ops.py +294 -174
  215. mindspore/ops/operations/debug_ops.py +59 -4
  216. mindspore/ops/operations/image_ops.py +13 -13
  217. mindspore/ops/operations/manually_defined/ops_def.py +15 -16
  218. mindspore/ops/operations/math_ops.py +3 -4
  219. mindspore/ops/operations/nn_ops.py +7 -39
  220. mindspore/ops/primitive.py +6 -10
  221. mindspore/ops/tensor_method.py +47 -8
  222. mindspore/ops_generate/api/cpp_create_prim_instance_helper_generator.py +1 -1
  223. mindspore/ops_generate/api/functional_map_cpp_generator.py +10 -9
  224. mindspore/ops_generate/api/functions_cc_generator.py +58 -10
  225. mindspore/ops_generate/api/tensor_func_reg_cpp_generator.py +1 -1
  226. mindspore/ops_generate/common/base_generator.py +14 -0
  227. mindspore/ops_generate/common/gen_constants.py +8 -3
  228. mindspore/ops_generate/common/gen_utils.py +0 -19
  229. mindspore/ops_generate/common/op_proto.py +11 -4
  230. mindspore/ops_generate/common/template.py +88 -11
  231. mindspore/ops_generate/gen_ops.py +1 -1
  232. mindspore/ops_generate/op_def/lite_ops_cpp_generator.py +4 -4
  233. mindspore/ops_generate/op_def/ops_def_cc_generator.py +0 -3
  234. mindspore/ops_generate/op_def/ops_name_h_generator.py +0 -3
  235. mindspore/ops_generate/op_def/ops_primitive_h_generator.py +0 -4
  236. mindspore/ops_generate/op_def_py/op_prim_py_generator.py +5 -2
  237. mindspore/ops_generate/pyboost/auto_grad_impl_cc_generator.py +49 -8
  238. mindspore/ops_generate/pyboost/auto_grad_reg_cc_generator.py +2 -2
  239. mindspore/ops_generate/pyboost/gen_pyboost_func.py +31 -0
  240. mindspore/ops_generate/pyboost/op_template_parser.py +98 -72
  241. mindspore/ops_generate/pyboost/pyboost_functions_cpp_generator.py +70 -273
  242. mindspore/ops_generate/pyboost/pyboost_functions_h_generator.py +14 -6
  243. mindspore/ops_generate/pyboost/pyboost_functions_impl_cpp_generator.py +316 -0
  244. mindspore/ops_generate/pyboost/pyboost_functions_py_generator.py +1 -1
  245. mindspore/ops_generate/pyboost/pyboost_grad_function_cpp_generator.py +5 -3
  246. mindspore/ops_generate/pyboost/pyboost_inner_prim_generator.py +1 -1
  247. mindspore/ops_generate/pyboost/pyboost_internal_functions_cpp_generator.py +76 -0
  248. mindspore/ops_generate/pyboost/pyboost_internal_functions_h_generator.py +76 -0
  249. mindspore/ops_generate/pyboost/pyboost_internal_kernel_info_adapter_generator.py +125 -0
  250. mindspore/ops_generate/pyboost/pyboost_native_grad_functions_generator.py +4 -3
  251. mindspore/ops_generate/pyboost/pyboost_op_cpp_code_generator.py +348 -61
  252. mindspore/ops_generate/pyboost/pyboost_overload_functions_cpp_generator.py +1 -1
  253. mindspore/ops_generate/pyboost/pyboost_utils.py +118 -9
  254. mindspore/ops_generate/tensor_py_cc_generator.py +1 -24
  255. mindspore/parallel/_auto_parallel_context.py +11 -8
  256. mindspore/parallel/_cell_wrapper.py +113 -45
  257. mindspore/parallel/_parallel_serialization.py +1 -1
  258. mindspore/parallel/_ps_context.py +4 -6
  259. mindspore/parallel/_tensor.py +167 -12
  260. mindspore/parallel/_transformer/moe.py +1 -1
  261. mindspore/parallel/_transformer/transformer.py +13 -8
  262. mindspore/parallel/auto_parallel.py +14 -7
  263. mindspore/parallel/checkpoint_convert.py +3 -3
  264. mindspore/parallel/checkpoint_transform.py +11 -7
  265. mindspore/parallel/cluster/process_entity/_api.py +84 -48
  266. mindspore/parallel/cluster/process_entity/_utils.py +95 -7
  267. mindspore/parallel/cluster/run.py +43 -4
  268. mindspore/parallel/function/__init__.py +8 -1
  269. mindspore/parallel/function/reshard_func.py +6 -7
  270. mindspore/parallel/nn/__init__.py +15 -2
  271. mindspore/parallel/nn/parallel_cell_wrapper.py +9 -10
  272. mindspore/parallel/nn/parallel_grad_reducer.py +7 -6
  273. mindspore/parallel/shard.py +3 -4
  274. mindspore/parallel/transform_safetensors.py +463 -174
  275. mindspore/pgodb140.dll +0 -0
  276. mindspore/pgort140.dll +0 -0
  277. mindspore/profiler/__init__.py +2 -1
  278. mindspore/profiler/analysis/parser/timeline_assembly_factory/ascend_timeline_assembler.py +7 -7
  279. mindspore/profiler/analysis/parser/timeline_assembly_factory/base_timeline_assembler.py +3 -0
  280. mindspore/profiler/analysis/parser/timeline_assembly_factory/trace_view_container.py +12 -6
  281. mindspore/profiler/analysis/parser/timeline_creator/cpu_op_timeline_creator.py +3 -3
  282. mindspore/profiler/analysis/parser/timeline_creator/fwk_timeline_creator.py +3 -3
  283. mindspore/profiler/analysis/parser/timeline_creator/msprof_timeline_creator.py +4 -4
  284. mindspore/profiler/analysis/parser/timeline_creator/scope_layer_timeline_creator.py +3 -3
  285. mindspore/profiler/analysis/parser/timeline_event/fwk_event.py +4 -1
  286. mindspore/profiler/analysis/parser/timeline_event/timeline_event_pool.py +2 -1
  287. mindspore/profiler/analysis/task_manager.py +1 -1
  288. mindspore/profiler/analysis/viewer/ascend_communication_viewer.py +5 -1
  289. mindspore/profiler/analysis/viewer/ascend_integrate_viewer.py +2 -1
  290. mindspore/profiler/analysis/viewer/ascend_op_memory_viewer.py +42 -22
  291. mindspore/profiler/analysis/viewer/ascend_step_trace_time_viewer.py +3 -2
  292. mindspore/profiler/analysis/viewer/ms_minddata_viewer.py +9 -5
  293. mindspore/profiler/analysis/viewer/ms_operator_details_viewer.py +132 -0
  294. mindspore/profiler/common/constant.py +16 -0
  295. mindspore/profiler/common/profiler_context.py +25 -27
  296. mindspore/profiler/common/profiler_info.py +0 -16
  297. mindspore/profiler/common/profiler_op_analyse.py +235 -0
  298. mindspore/profiler/common/profiler_output_path.py +23 -8
  299. mindspore/profiler/common/profiler_parameters.py +128 -35
  300. mindspore/profiler/dynamic_profile/__init__.py +0 -0
  301. mindspore/profiler/dynamic_profile/dynamic_monitor_proxy.py +39 -0
  302. mindspore/profiler/dynamic_profile/dynamic_profiler_config_context.py +666 -0
  303. mindspore/profiler/dynamic_profile/dynamic_profiler_utils.py +62 -0
  304. mindspore/profiler/dynamic_profiler.py +305 -314
  305. mindspore/profiler/envprofiler.py +12 -7
  306. mindspore/profiler/experimental_config.py +96 -6
  307. mindspore/profiler/mstx.py +33 -12
  308. mindspore/profiler/platform/__init__.py +2 -3
  309. mindspore/profiler/platform/npu_profiler.py +29 -19
  310. mindspore/profiler/profiler.py +35 -19
  311. mindspore/profiler/profiler_action_controller.py +64 -76
  312. mindspore/profiler/schedule.py +10 -4
  313. mindspore/rewrite/common/config.py +1 -0
  314. mindspore/rewrite/common/namer.py +1 -0
  315. mindspore/rewrite/common/namespace.py +1 -0
  316. mindspore/rewrite/node/node.py +31 -11
  317. mindspore/rewrite/parsers/assign_parser.py +1 -1
  318. mindspore/rewrite/symbol_tree/symbol_tree.py +1 -1
  319. mindspore/run_check/_check_version.py +7 -10
  320. mindspore/runtime/__init__.py +5 -5
  321. mindspore/runtime/event.py +10 -4
  322. mindspore/runtime/executor.py +60 -45
  323. mindspore/runtime/memory.py +30 -32
  324. mindspore/runtime/thread_bind_core.py +298 -164
  325. mindspore/safeguard/rewrite_obfuscation.py +12 -13
  326. mindspore/swresample-4.dll +0 -0
  327. mindspore/swscale-6.dll +0 -0
  328. mindspore/tbbmalloc.dll +0 -0
  329. mindspore/tinyxml2.dll +0 -0
  330. mindspore/train/_utils.py +14 -4
  331. mindspore/train/amp.py +43 -20
  332. mindspore/train/callback/__init__.py +5 -5
  333. mindspore/train/callback/_checkpoint.py +3 -6
  334. mindspore/train/callback/_flops_collector.py +1 -1
  335. mindspore/train/callback/_landscape.py +0 -1
  336. mindspore/train/callback/_train_fault_tolerance.py +97 -16
  337. mindspore/train/data_sink.py +11 -2
  338. mindspore/train/dataset_helper.py +9 -0
  339. mindspore/train/model.py +135 -55
  340. mindspore/train/serialization.py +133 -111
  341. mindspore/train/summary/summary_record.py +13 -2
  342. mindspore/turbojpeg.dll +0 -0
  343. mindspore/utils/__init__.py +3 -2
  344. mindspore/utils/dryrun.py +0 -6
  345. mindspore/utils/runtime_execution_order_check.py +163 -77
  346. mindspore/utils/sdc_detect.py +68 -0
  347. mindspore/utils/utils.py +6 -9
  348. mindspore/vcmeta.dll +0 -0
  349. mindspore/vcruntime140.dll +0 -0
  350. mindspore/vcruntime140_1.dll +0 -0
  351. mindspore/version.py +1 -1
  352. {mindspore-2.6.0rc1.dist-info → mindspore-2.7.0rc1.dist-info}/METADATA +5 -4
  353. {mindspore-2.6.0rc1.dist-info → mindspore-2.7.0rc1.dist-info}/RECORD +356 -394
  354. mindspore/_deprecated/jit.py +0 -198
  355. mindspore/experimental/es/__init__.py +0 -22
  356. mindspore/experimental/es/embedding_service.py +0 -891
  357. mindspore/experimental/es/embedding_service_layer.py +0 -581
  358. mindspore/profiler/parser/__init__.py +0 -14
  359. mindspore/profiler/parser/aicpu_data_parser.py +0 -272
  360. mindspore/profiler/parser/ascend_analysis/__init__.py +0 -14
  361. mindspore/profiler/parser/ascend_analysis/constant.py +0 -71
  362. mindspore/profiler/parser/ascend_analysis/file_manager.py +0 -180
  363. mindspore/profiler/parser/ascend_analysis/function_event.py +0 -185
  364. mindspore/profiler/parser/ascend_analysis/fwk_cann_parser.py +0 -136
  365. mindspore/profiler/parser/ascend_analysis/fwk_file_parser.py +0 -131
  366. mindspore/profiler/parser/ascend_analysis/msprof_timeline_parser.py +0 -104
  367. mindspore/profiler/parser/ascend_analysis/path_manager.py +0 -313
  368. mindspore/profiler/parser/ascend_analysis/profiler_info_parser.py +0 -123
  369. mindspore/profiler/parser/ascend_analysis/tlv_decoder.py +0 -86
  370. mindspore/profiler/parser/ascend_analysis/trace_event_manager.py +0 -75
  371. mindspore/profiler/parser/ascend_cluster_generator.py +0 -116
  372. mindspore/profiler/parser/ascend_communicate_generator.py +0 -314
  373. mindspore/profiler/parser/ascend_flops_generator.py +0 -116
  374. mindspore/profiler/parser/ascend_fpbp_generator.py +0 -82
  375. mindspore/profiler/parser/ascend_hccl_generator.py +0 -271
  376. mindspore/profiler/parser/ascend_integrate_generator.py +0 -42
  377. mindspore/profiler/parser/ascend_memory_generator.py +0 -185
  378. mindspore/profiler/parser/ascend_msprof_exporter.py +0 -282
  379. mindspore/profiler/parser/ascend_msprof_generator.py +0 -187
  380. mindspore/profiler/parser/ascend_op_generator.py +0 -334
  381. mindspore/profiler/parser/ascend_steptrace_generator.py +0 -94
  382. mindspore/profiler/parser/ascend_timeline_generator.py +0 -545
  383. mindspore/profiler/parser/base_timeline_generator.py +0 -483
  384. mindspore/profiler/parser/container.py +0 -229
  385. mindspore/profiler/parser/cpu_gpu_timeline_generator.py +0 -697
  386. mindspore/profiler/parser/flops_parser.py +0 -531
  387. mindspore/profiler/parser/framework_enum.py +0 -111
  388. mindspore/profiler/parser/framework_parser.py +0 -464
  389. mindspore/profiler/parser/framework_struct.py +0 -61
  390. mindspore/profiler/parser/gpu_analysis/__init__.py +0 -14
  391. mindspore/profiler/parser/gpu_analysis/function_event.py +0 -44
  392. mindspore/profiler/parser/gpu_analysis/fwk_file_parser.py +0 -89
  393. mindspore/profiler/parser/gpu_analysis/profiler_info_parser.py +0 -72
  394. mindspore/profiler/parser/hccl_parser.py +0 -573
  395. mindspore/profiler/parser/hwts_log_parser.py +0 -122
  396. mindspore/profiler/parser/integrator.py +0 -526
  397. mindspore/profiler/parser/memory_usage_parser.py +0 -277
  398. mindspore/profiler/parser/minddata_analyzer.py +0 -800
  399. mindspore/profiler/parser/minddata_parser.py +0 -186
  400. mindspore/profiler/parser/minddata_pipeline_parser.py +0 -299
  401. mindspore/profiler/parser/op_intermediate_parser.py +0 -149
  402. mindspore/profiler/parser/optime_parser.py +0 -250
  403. mindspore/profiler/parser/profiler_info.py +0 -213
  404. mindspore/profiler/parser/step_trace_parser.py +0 -666
  405. {mindspore-2.6.0rc1.dist-info → mindspore-2.7.0rc1.dist-info}/WHEEL +0 -0
  406. {mindspore-2.6.0rc1.dist-info → mindspore-2.7.0rc1.dist-info}/entry_points.txt +0 -0
  407. {mindspore-2.6.0rc1.dist-info → mindspore-2.7.0rc1.dist-info}/top_level.txt +0 -0
@@ -1,185 +0,0 @@
1
- # Copyright 2023 Huawei Technologies Co., Ltd
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
- # ============================================================================
15
- """Function event data struct."""
16
- from typing import Dict, Optional
17
- from enum import Enum
18
- from decimal import Decimal
19
- import struct
20
- from abc import ABC, abstractmethod
21
-
22
- from mindspore.profiler.parser.ascend_analysis.constant import Constant
23
- from mindspore.profiler.parser.ascend_analysis.profiler_info_parser import ProfilerInfoParser
24
-
25
-
26
- class BaseEvent(ABC):
27
- """Base class of CANNEvent and MindSporeOpEvent"""
28
-
29
- def __init__(self, data: Dict):
30
- if not isinstance(data, dict):
31
- raise TypeError("Input data must be dict.")
32
- # common attributes
33
- self._orig_data = data
34
- self.name: str = ""
35
- self.pid: int = 0
36
- self.tid: int = 0
37
- self.ts: Decimal = Decimal(0)
38
- self.te: Decimal = Decimal(0)
39
- self.dur: float = 0.0
40
- self.args: Dict = {}
41
- self.parent: Optional[BaseEvent] = None
42
- self._init_params()
43
-
44
- @abstractmethod
45
- def _init_params(self):
46
- err_msg = "Protected function _init_params need to be implemented."
47
- raise NotImplementedError(err_msg)
48
-
49
-
50
- class CANNEvent(BaseEvent):
51
- """Function event collected on the CANN side"""
52
-
53
- def _init_params(self):
54
- """Initialize the attribute value of CANNEvent."""
55
- self.ts = Decimal(str(self._orig_data.get("ts", 0)))
56
- self.pid = self._orig_data.get("pid", 0)
57
- self.tid = self._orig_data.get("tid", 0)
58
- self.dur = self._orig_data.get("dur", 0.0)
59
- self.te = self.ts + Decimal(str(self.dur))
60
- self.name = self._orig_data.get("name", "")
61
- self.id = self._orig_data.get("id", 0)
62
- self.args = self._orig_data.get("args", {})
63
- self.ph = self._orig_data.get("ph")
64
- self.cat = self._orig_data.get("cat")
65
-
66
- def is_flow_start_event(self) -> bool:
67
- """Determine whether the event is flow start event or not."""
68
- return self._orig_data.get("cat") == Constant.HOST_TO_DEVICE and \
69
- self._orig_data.get("ph") == Constant.START_FLOW
70
-
71
- def is_flow_end_event(self) -> bool:
72
- """Determine whether the event is flow end event or not."""
73
- return self._orig_data.get("cat") == Constant.HOST_TO_DEVICE and \
74
- self._orig_data.get("ph") == Constant.END_FLOW
75
-
76
- def is_x_event(self) -> bool:
77
- """Determine whether the event x event or not."""
78
- return self._orig_data.get("ph") == Constant.COMPLETE_EVENT
79
-
80
- def get_sort_index(self):
81
- """get the process sort index"""
82
- if self.args.get('name', '') == 'Ascend Hardware':
83
- return Constant.ASCEND_HARDWARE
84
- if self.args.get('name', '') == 'CANN':
85
- return Constant.CANN
86
- if self.args.get('name', '') == 'HCCL':
87
- return Constant.HCCL
88
- if self.args.get('name', '') == 'Overlap Analysis':
89
- return Constant.OVERLAP
90
- return Constant.OTHERWISE
91
-
92
- def to_json(self):
93
- """Cast to trace event."""
94
- if self.ph == Constant.META_EVENT:
95
- if self.name == Constant.PROCESS_NAME:
96
- return [{'name': self.name, 'pid': self.pid, 'tid': self.tid,
97
- 'args': self.args, 'ph': self.ph},
98
- {'name': Constant.PROCESS_SORT, 'pid': self.pid, 'tid': self.tid,
99
- 'args': {'sort_index': self.get_sort_index()}, 'ph': self.ph}]
100
- if self.name == Constant.PROCESS_SORT:
101
- return None
102
- return {'name': self.name, 'pid': self.pid, 'tid': self.tid,
103
- 'args': self.args, 'ph': self.ph}
104
-
105
- if self.ph == Constant.COMPLETE_EVENT:
106
- if self.parent is not None:
107
- self.args.update({'mindspore_op': self.parent.name})
108
- res = {'name': self.name, 'pid': self.pid, 'tid': self.tid,
109
- 'ts': str(self.ts), 'dur': self.dur, 'args': self.args, 'ph': self.ph}
110
- if self.cat:
111
- res.update({'cat': self.cat})
112
- return res
113
-
114
- if self.ph == Constant.START_FLOW:
115
- return {"ph": self.ph, "name": self.name, "id": self.id, "pid": self.pid,
116
- "tid": self.tid, "ts": str(self.ts), "cat": self.cat}
117
-
118
- if self.ph == Constant.END_FLOW:
119
- return {"ph": self.ph, "name": self.name, "id": self.id, "pid": self.pid,
120
- "tid": self.tid, "ts": str(self.ts), "cat": self.cat, 'bp': "e"}
121
- return None
122
-
123
-
124
- class MindSporeOpEnum(Enum):
125
- """MindSporeOp index enum"""
126
- START_NS = 0
127
- END_NS = 1
128
- SEQUENCE_UNMBER = 2
129
- PROCESS_ID = 3
130
- START_THREAD_ID = 4
131
- END_THREAD_ID = 5
132
- FORWORD_THREAD_ID = 6
133
- FLOW_ID = 7
134
- STEP_ID = 8
135
- LEVEL = 9
136
- IS_ASYNC = 10
137
-
138
-
139
- class MindSporeOpEvent(BaseEvent):
140
- """
141
- Function event collected on the mindspore frame side.
142
-
143
- Args:
144
- data(Dict): The mindspore frame side data decoded by TLVDecoder.
145
- """
146
- _tlv_type_dict = {
147
- Constant.OP_NAME: 3, Constant.INPUT_SHAPES: 5, Constant.INPUT_DTYPES: 4,
148
- Constant.CALL_STACK: 6, Constant.MODULE_HIERARCHY: 7, Constant.FLOPS: 8,
149
- Constant.CUSTOM_INFO: 9
150
- }
151
- _fix_data_format = "<3q6Qb?"
152
-
153
- def _init_params(self):
154
- """Initialize the attribute value of MindSporeOpEvent."""
155
- fix_size_data = struct.unpack(self._fix_data_format, self._orig_data.get(Constant.FIX_SIZE_BYTES))
156
- self.pid = Constant.MINDSPORE
157
- self.tid = int(fix_size_data[MindSporeOpEnum.START_THREAD_ID.value])
158
- self.name = str(self._orig_data.get(self._tlv_type_dict.get(Constant.OP_NAME), ""))
159
- self.ts = ProfilerInfoParser.get_local_time(fix_size_data[MindSporeOpEnum.START_NS.value]) # unit is us
160
- self.te = ProfilerInfoParser.get_local_time(fix_size_data[MindSporeOpEnum.END_NS.value]) # unit is us
161
- self.dur = self.te - self.ts
162
- self.flow_id = int(fix_size_data[MindSporeOpEnum.FLOW_ID.value])
163
- self.step = int(fix_size_data[MindSporeOpEnum.STEP_ID.value])
164
- self.level = int(fix_size_data[MindSporeOpEnum.LEVEL.value])
165
- self.custom_info = ""
166
- self.args = self._get_args(fix_size_data)
167
-
168
- def _get_args(self, fix_size_data) -> Dict:
169
- """Get the rest information saved in args"""
170
- args = {
171
- Constant.SEQUENCE_UNMBER: int(fix_size_data[MindSporeOpEnum.SEQUENCE_UNMBER.value]),
172
- Constant.FORWORD_THREAD_ID: int(fix_size_data[MindSporeOpEnum.FORWORD_THREAD_ID.value])}
173
- for type_name, type_id in self._tlv_type_dict.items():
174
- if type_name == Constant.OP_NAME or type_id not in self._orig_data.keys():
175
- continue
176
- if type_name in set([Constant.INPUT_SHAPES, Constant.INPUT_DTYPES, Constant.CALL_STACK]):
177
- args[type_name] = self._orig_data.get(type_id).replace("|", "\r\n")
178
- elif type_name == Constant.CUSTOM_INFO and self._orig_data.get(type_id):
179
- pairs = self._orig_data.get(type_id).split(';')
180
- custom_info = {pair.split(':')[0]: pair.split(':')[1] for pair in pairs}
181
- args[type_name] = custom_info
182
- self.custom_info = custom_info.__str__()
183
- else:
184
- args[type_name] = self._orig_data.get(type_id)
185
- return args
@@ -1,136 +0,0 @@
1
- # Copyright 2023 Huawei Technologies Co., Ltd
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
- # ============================================================================
15
- """The parser for integrating data from the framework side and CANN side."""
16
- from collections import defaultdict
17
- from typing import List, Tuple, Optional
18
- from decimal import Decimal
19
-
20
- from mindspore import log as logger
21
- from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
22
- from mindspore.profiler.parser.ascend_analysis.function_event import CANNEvent, MindSporeOpEvent
23
- from mindspore.profiler.parser.ascend_analysis.fwk_file_parser import FwkFileParser
24
- from mindspore.profiler.parser.ascend_analysis.trace_event_manager import TraceEventManager
25
- from mindspore.profiler.parser.ascend_analysis.msprof_timeline_parser import MsprofTimelineParser
26
- from mindspore.profiler.parser.ascend_analysis.profiler_info_parser import ProfilerInfoParser
27
- from mindspore.profiler.parser.profiler_info import ProfilerInfo
28
-
29
-
30
- class FwkCANNParser:
31
- """The top-level trace view parser."""
32
-
33
- def __init__(self, source_path: str, msprof_data: List, rank_id: int, step_list=None):
34
- source_path = validate_and_normalize_path(source_path)
35
- ProfilerInfoParser.init_source_path(source_path)
36
- ProfilerInfoParser.init_rank_id(rank_id)
37
-
38
- fwk_parser = FwkFileParser(source_path, rank_id)
39
- self._fwk_data = fwk_parser.get_op_range_data(step_list)
40
- self._fwk_json, self._link_event_dict, self._fwk_launch_op = fwk_parser.get_fwk_trace_data(self._fwk_data)
41
-
42
- msprof_timeline_parser = MsprofTimelineParser(msprof_data)
43
- self._start_flow_to_npu_dict, self._device_mostly_json, self.scope_data_without_flow \
44
- = msprof_timeline_parser.get_acl_to_npu_data()
45
- self.scope_data_with_flow = []
46
- self.rank_id: int = rank_id
47
- self.kernels: List[CANNEvent] = []
48
-
49
- def generate_trace_data(self):
50
- """
51
- Generate trace view json timeline file which contains framework side and
52
- device side data.
53
- """
54
- fwk_flow_json = self.__link_msop_self()
55
- device_flow_and_x_json = self.__link_msop_kernel()
56
- return self._fwk_json + fwk_flow_json + self._device_mostly_json + device_flow_and_x_json
57
-
58
- def __link_msop_self(self):
59
- """Create flow between framework-side multi-level pipeline task."""
60
- flow_json = []
61
- for op_data_list in self._link_event_dict.values():
62
- if len(op_data_list) != 2:
63
- logger.info('Only alow 2 op_data have the same flow_id. but got %s', len(op_data_list))
64
- continue
65
- flow_json.extend(TraceEventManager.create_mindspore_to_self_flow(op_data_list[0],
66
- op_data_list[1]))
67
- return flow_json
68
-
69
- def __link_msop_kernel(self):
70
- """Associate the frame-side operator with the device-side kernel"""
71
- trace_data_json = []
72
- acl_to_npu_by_tid = {}
73
- for (cann_op_tid, cann_op_ts), event_list in self._start_flow_to_npu_dict.items():
74
- acl_to_npu_by_tid.setdefault(cann_op_tid, defaultdict(list))[cann_op_ts].extend(event_list)
75
-
76
- if self._fwk_launch_op and acl_to_npu_by_tid and self._fwk_launch_op.keys() != acl_to_npu_by_tid.keys():
77
- logger.warning("Failed to create link between mindspore operator and kernels.")
78
- is_not_O2 = bool(ProfilerInfo.get_profiler_info().get(ProfilerInfo.JIT_LEVEL, "") != "O2")
79
- for device_tid in acl_to_npu_by_tid:
80
- host_data_sorted = sorted(self._fwk_launch_op.get(device_tid, []), key=lambda x: x.ts)
81
- op_idx = 0
82
-
83
- for ts, device_data_list in sorted(acl_to_npu_by_tid.get(device_tid).items(), key=lambda x: x[0]):
84
- op_idx, status = FwkCANNParser.__find_launch_op(ts, host_data_sorted, op_idx)
85
- if not status and is_not_O2:
86
- logger.warning("There are %s device ops have no flows were found. The CANN ts is %s.",
87
- len(device_data_list), ts)
88
-
89
- for device_data in device_data_list:
90
- if status:
91
- device_data.parent = host_data_sorted[op_idx]
92
- trace_data_json.extend(TraceEventManager.create_mindspore_to_npu_flow(device_data.parent,
93
- device_data))
94
- self.kernels.append(device_data)
95
-
96
- scope_layer = MsprofTimelineParser.parse_ascend_hardware_scope(device_data)
97
-
98
- if scope_layer is not None:
99
- self.scope_data_with_flow.append(scope_layer)
100
-
101
- event_json = device_data.to_json()
102
- if event_json and isinstance(event_json, list):
103
- trace_data_json.extend(event_json)
104
- elif event_json and isinstance(event_json, dict):
105
- trace_data_json.append(event_json)
106
-
107
- return trace_data_json
108
-
109
- @staticmethod
110
- def __find_launch_op(ts: Decimal, op_list: List[MindSporeOpEvent],
111
- left: Optional[int] = None, right: Optional[int] = None
112
- ) -> Tuple[int, bool]:
113
- """
114
- Searching the op_list in [left, right) range and find the operator
115
- whose start time is larger than ts and end time is less than ts.
116
-
117
- Args:
118
- ts(Decimal): kernel start time
119
- op_list(List): MindSporeOpEvent list
120
- left & right(int): the searching index range is [left, right)
121
-
122
- Return:
123
- Tuple[int, bool]: the first element is the searched index, the second element
124
- marks where the operator index is found or not.
125
- """
126
- left = 0 if (left is None or left < 0) else left
127
- right = len(op_list) if (right is None or right < 0) else right
128
- # The data in range [left, right) is considered.
129
- while right > left:
130
- if op_list[left].ts > ts:
131
- return left, False
132
- if op_list[left].te < ts:
133
- left += 1
134
- else:
135
- return left, True
136
- return left, False
@@ -1,131 +0,0 @@
1
- # Copyright 2023 Huawei Technologies Co., Ltd
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
- # ============================================================================
15
- """MindSpore framework oprange file parser"""
16
-
17
- import os
18
- import stat
19
- import csv
20
- from typing import List
21
- from collections import defaultdict
22
-
23
- from mindspore import log as logger
24
- from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
25
- from mindspore.profiler.parser.ascend_analysis.file_manager import FileManager
26
- from mindspore.profiler.parser.ascend_analysis.tlv_decoder import TLVDecoder
27
- from mindspore.profiler.parser.ascend_analysis.function_event import MindSporeOpEvent
28
- from mindspore.profiler.parser.ascend_analysis.trace_event_manager import TraceEventManager
29
- from mindspore.profiler.parser.ascend_analysis.constant import Constant
30
-
31
-
32
- class FwkFileParser:
33
- """Framework-side operator file parser."""
34
-
35
- _op_range = "FRAMEWORK/op_range_{}"
36
- _op_range_struct_size = 74
37
-
38
- def __init__(self, source_path: str, rank_id: int):
39
- """
40
- source_path: The path of PROF_* directory
41
- """
42
- self.rank_id = rank_id
43
- self._prof_root = None
44
- self._op_range_path = None
45
- self._init_framework_path(source_path)
46
-
47
- def get_op_range_data(self, step_list=None) -> List[MindSporeOpEvent]:
48
- """Read and decode all the mindspore oprange data."""
49
- op_range_list = []
50
- if os.path.exists(self._op_range_path):
51
- op_range_bytes = FileManager.read_file_content(self._op_range_path, "rb")
52
- op_range_list = TLVDecoder.decode(op_range_bytes, MindSporeOpEvent, self._op_range_struct_size)
53
- else:
54
- logger.warning("Failed to find op_range data. skip parse host profiler data.")
55
- if step_list and isinstance(step_list, list):
56
- first_step = min(op.step for op in op_range_list)
57
- step_list = [step - 1 + first_step for step in step_list]
58
- op_range_list = list(filter(lambda op: op.step in step_list, op_range_list))
59
- return op_range_list
60
-
61
- def get_fwk_trace_data(self, mindspore_op_data: List[MindSporeOpEvent] = None):
62
- """Generate chrome trace format json data from decoded oprange data."""
63
- if not mindspore_op_data:
64
- mindspore_op_data = self.get_op_range_data()
65
- tid_map = defaultdict(set)
66
- fwk_x_event_list = []
67
- link_event_dict = defaultdict(list)
68
- fwk_launch_op = defaultdict(list)
69
- dataset_op_data = []
70
-
71
- for mindspore_op in mindspore_op_data:
72
- if mindspore_op.name == Constant.FLOW_OP:
73
- link_event_dict[mindspore_op.flow_id].insert(0, mindspore_op)
74
- continue
75
- if mindspore_op.flow_id != Constant.INVALID_FLOW_ID:
76
- link_event_dict[mindspore_op.flow_id].append(mindspore_op)
77
-
78
- if 'KernelLaunch' in mindspore_op.name or 'LaunchTask' in mindspore_op.name:
79
- fwk_launch_op[mindspore_op.tid].append(mindspore_op)
80
-
81
- if mindspore_op.name.split('::')[0] == 'Dataset':
82
- dataset_op_data.append(mindspore_op)
83
-
84
- tid_map[mindspore_op.pid].add(mindspore_op.tid)
85
- if mindspore_op.dur > 0:
86
- fwk_x_event_list.append(TraceEventManager.create_x_event(mindspore_op, "cpu_op"))
87
- else:
88
- fwk_x_event_list.append(TraceEventManager.create_i_event(mindspore_op))
89
-
90
- fwk_m_event_list = []
91
- for pid, tid_set in tid_map.items():
92
- fwk_m_event_list.extend(TraceEventManager.create_m_event(pid, tid_set, pid))
93
-
94
- self.calculate_dataset_item(dataset_op_data)
95
-
96
- return fwk_x_event_list + fwk_m_event_list, link_event_dict, fwk_launch_op
97
-
98
- def calculate_dataset_item(self, dataset_op_data: List[MindSporeOpEvent]):
99
- """
100
- Get the summary data of dataset op by parsing the dataset_op_data.
101
- """
102
- self._prof_root = validate_and_normalize_path(self._prof_root)
103
- dataset_file = os.path.join(self._prof_root, f'dataset_{self.rank_id}.csv')
104
- summary_data = defaultdict(list)
105
- for op_data in dataset_op_data:
106
- _, event, stage = op_data.name.split('::')
107
- key = event + '::' + stage + '::' + op_data.custom_info
108
- summary_data[key].append(op_data.dur)
109
-
110
- flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
111
- modes = stat.S_IWUSR | stat.S_IRUSR
112
- with os.fdopen(os.open(dataset_file, flags, modes), 'w', newline='') as fw:
113
- csv_writer = csv.writer(fw)
114
- csv_writer.writerow(['Operation', 'Stage', 'Occurrences', 'Avg. time (us)', 'Custom Info'])
115
- for k, v in summary_data.items():
116
- event, stage, custom_info = k.split('::')
117
- count = len(v)
118
- average_execution = round(float(sum(v) / count), 2)
119
- csv_writer.writerow([event, stage, count, average_execution, custom_info])
120
- os.chmod(dataset_file, modes)
121
-
122
- def _init_framework_path(self, source_path: str):
123
- """Init the oprange data path."""
124
- source_path = validate_and_normalize_path(source_path)
125
- if not os.path.exists(source_path):
126
- raise FileNotFoundError("Input source_path does not exist!")
127
- device_name = os.path.basename(source_path)
128
- if not device_name.startswith("device") and not os.path.isdir(source_path):
129
- raise RuntimeError("Input source_path is invalid!")
130
- self._prof_root = os.path.dirname(os.path.dirname(source_path))
131
- self._op_range_path = os.path.join(self._prof_root, self._op_range.format(self.rank_id))
@@ -1,104 +0,0 @@
1
- # Copyright 2023 Huawei Technologies Co., Ltd
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
- # ============================================================================
15
- """msprof timeline file parser"""
16
- from collections import defaultdict
17
- from typing import List
18
-
19
- from mindspore.profiler.parser.ascend_analysis.constant import Constant
20
- from mindspore.profiler.parser.ascend_analysis.function_event import CANNEvent, MindSporeOpEvent, BaseEvent
21
-
22
-
23
- class MsprofTimelineParser:
24
- """Msprof timeline file parser."""
25
-
26
- def __init__(self, msprof_data: List):
27
- self.timeline_data = msprof_data
28
-
29
- def get_acl_to_npu_data(self):
30
- """Get all the acl to npu flow events."""
31
- flow_start_dict, flow_end_dict = {}, {}
32
- x_event_list, other_event_json = [], []
33
- for data in self.timeline_data:
34
- cann_event = CANNEvent(data)
35
- if cann_event.is_x_event():
36
- x_event_list.append(cann_event)
37
- else:
38
- event_json = cann_event.to_json()
39
- if event_json and isinstance(event_json, list):
40
- other_event_json.extend(event_json)
41
- elif event_json and isinstance(event_json, dict):
42
- other_event_json.append(event_json)
43
-
44
- if cann_event.is_flow_start_event():
45
- flow_start_dict[cann_event.id] = (cann_event.tid, cann_event.ts)
46
- elif cann_event.is_flow_end_event():
47
- flow_end_dict[(cann_event.pid, cann_event.tid, cann_event.ts)] = cann_event.id
48
-
49
- start_flow_to_npu_dict = defaultdict(list)
50
- device_data_without_flow_json = []
51
- scope_data_without_flow = []
52
- for cann_event in x_event_list:
53
- flow_id = flow_end_dict.get((cann_event.pid, cann_event.tid, cann_event.ts))
54
- start_flow_info = flow_start_dict.get(flow_id)
55
- if flow_id is not None and start_flow_info is not None:
56
- start_flow_to_npu_dict[start_flow_info].append(cann_event)
57
- else:
58
- event_json = cann_event.to_json()
59
- if event_json and isinstance(event_json, list):
60
- device_data_without_flow_json.extend(event_json)
61
- elif event_json and isinstance(event_json, dict):
62
- device_data_without_flow_json.append(event_json)
63
-
64
- scope_layer = self.parse_ascend_hardware_scope(cann_event)
65
- if scope_layer is not None:
66
- scope_data_without_flow.append(scope_layer)
67
-
68
- return start_flow_to_npu_dict, device_data_without_flow_json + other_event_json, scope_data_without_flow
69
-
70
- @staticmethod
71
- def parse_ascend_hardware_scope(cann_event: BaseEvent):
72
- """
73
- parse ascend hardware scope
74
- """
75
- scope_from_flow_start = MsprofTimelineParser.get_scope_layer(cann_event.parent)
76
- scope = MsprofTimelineParser.get_scope_layer(cann_event)
77
-
78
- if scope and scope_from_flow_start:
79
- scope = scope_from_flow_start if len(scope_from_flow_start) > len(scope) else scope
80
- elif not scope and not scope_from_flow_start:
81
- return None
82
- else:
83
- scope = scope_from_flow_start if scope_from_flow_start else scope
84
-
85
- return (scope, cann_event.ts, cann_event.te)
86
-
87
- @staticmethod
88
- def get_scope_layer(event: BaseEvent):
89
- """
90
- get the scope of device op
91
- """
92
-
93
- if event is None:
94
- return None
95
-
96
- full_scope_name = None
97
- if isinstance(event, MindSporeOpEvent):
98
- full_scope_name = event.name.split('::')[-1]
99
- elif isinstance(event, CANNEvent):
100
- full_scope_name = event.name
101
-
102
- if full_scope_name and full_scope_name.startswith(Constant.TOP_SCOPE_NAMES):
103
- return full_scope_name.split('/')[:-1]
104
- return None