mindspore 2.6.0__cp310-cp310-win_amd64.whl → 2.7.0__cp310-cp310-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mindspore might be problematic. Click here for more details.

Files changed (455) hide show
  1. mindspore/.commit_id +1 -1
  2. mindspore/Microsoft.VisualStudio.Telemetry.dll +0 -0
  3. mindspore/Newtonsoft.Json.dll +0 -0
  4. mindspore/__init__.py +2 -2
  5. mindspore/_c_dataengine.cp310-win_amd64.pyd +0 -0
  6. mindspore/_c_expression.cp310-win_amd64.pyd +0 -0
  7. mindspore/_c_mindrecord.cp310-win_amd64.pyd +0 -0
  8. mindspore/_checkparam.py +42 -11
  9. mindspore/_extends/builtin_operations.py +3 -3
  10. mindspore/{_deprecated → _extends/optimize}/__init__.py +9 -3
  11. mindspore/_extends/optimize/cell_utils.py +96 -0
  12. mindspore/_extends/parallel_compile/akg_compiler/custom.py +1109 -0
  13. mindspore/_extends/parallel_compile/akg_compiler/gen_custom_op_files.py +1 -1
  14. mindspore/_extends/parse/__init__.py +3 -3
  15. mindspore/_extends/parse/compile_config.py +44 -22
  16. mindspore/_extends/parse/deprecated/deprecated_tensor_method.py +1 -2
  17. mindspore/_extends/parse/parser.py +64 -83
  18. mindspore/_extends/parse/resources.py +39 -0
  19. mindspore/_extends/parse/standard_method.py +47 -14
  20. mindspore/_extends/parse/trope.py +8 -1
  21. mindspore/_extends/pijit/__init__.py +1 -2
  22. mindspore/_extends/pijit/pijit_func_white_list.py +2 -5
  23. mindspore/amp.py +4 -22
  24. mindspore/atlprov.dll +0 -0
  25. mindspore/avcodec-59.dll +0 -0
  26. mindspore/avdevice-59.dll +0 -0
  27. mindspore/avfilter-8.dll +0 -0
  28. mindspore/avformat-59.dll +0 -0
  29. mindspore/avutil-57.dll +0 -0
  30. mindspore/boost/adasum.py +1 -1
  31. mindspore/boost/boost_cell_wrapper.py +4 -4
  32. mindspore/c1.dll +0 -0
  33. mindspore/c1xx.dll +0 -0
  34. mindspore/c2.dll +0 -0
  35. mindspore/common/__init__.py +43 -12
  36. mindspore/common/_grad_function.py +2 -1
  37. mindspore/common/_pijit_context.py +28 -7
  38. mindspore/common/_stub_tensor.py +1 -209
  39. mindspore/common/_tensor_cpp_method.py +1 -1
  40. mindspore/common/_tensor_docs.py +177 -52
  41. mindspore/common/_utils.py +9 -1
  42. mindspore/common/api.py +338 -208
  43. mindspore/common/dtype.py +108 -57
  44. mindspore/common/dump.py +11 -16
  45. mindspore/common/dynamic_shape/__init__.py +0 -0
  46. mindspore/common/{auto_dynamic_shape.py → dynamic_shape/auto_dynamic_shape.py} +17 -23
  47. mindspore/common/dynamic_shape/enable_dynamic.py +197 -0
  48. mindspore/common/file_system.py +59 -9
  49. mindspore/common/generator.py +2 -3
  50. mindspore/common/hook_handle.py +33 -5
  51. mindspore/common/jit_config.py +1 -1
  52. mindspore/common/jit_trace.py +84 -105
  53. mindspore/common/np_dtype.py +3 -3
  54. mindspore/common/parameter.py +27 -29
  55. mindspore/common/recompute.py +5 -7
  56. mindspore/common/sparse_tensor.py +0 -3
  57. mindspore/common/symbol.py +0 -1
  58. mindspore/common/tensor.py +84 -133
  59. mindspore/communication/_comm_helper.py +46 -4
  60. mindspore/communication/management.py +79 -7
  61. mindspore/context.py +47 -38
  62. mindspore/dataset/__init__.py +1 -1
  63. mindspore/dataset/audio/transforms.py +1 -1
  64. mindspore/dataset/core/config.py +38 -4
  65. mindspore/dataset/engine/datasets.py +350 -322
  66. mindspore/dataset/engine/datasets_user_defined.py +69 -23
  67. mindspore/dataset/engine/iterators.py +2 -2
  68. mindspore/dataset/engine/obs/config_loader.py +2 -2
  69. mindspore/dataset/engine/obs/obs_mindrecord_dataset.py +8 -0
  70. mindspore/dataset/transforms/c_transforms.py +2 -2
  71. mindspore/dataset/transforms/py_transforms.py +7 -3
  72. mindspore/dataset/transforms/transforms.py +10 -6
  73. mindspore/dataset/vision/__init__.py +1 -1
  74. mindspore/dataset/vision/py_transforms.py +8 -8
  75. mindspore/dataset/vision/transforms.py +17 -5
  76. mindspore/dataset/vision/utils.py +632 -21
  77. mindspore/dataset/vision/validators.py +1 -0
  78. mindspore/device_context/ascend/device.py +1 -1
  79. mindspore/device_context/ascend/op_tuning.py +35 -1
  80. mindspore/device_context/gpu/__init__.py +2 -2
  81. mindspore/device_context/gpu/device.py +1 -1
  82. mindspore/device_context/gpu/op_precision.py +4 -2
  83. mindspore/device_context/gpu/op_tuning.py +6 -3
  84. mindspore/device_manager.py +16 -9
  85. mindspore/dnnl.dll +0 -0
  86. mindspore/dpcmi.dll +0 -0
  87. mindspore/experimental/llm_boost/ascend_native/llama_boost_ascend_native.py +5 -4
  88. mindspore/experimental/llm_boost/atb/boost_base.py +2 -3
  89. mindspore/experimental/optim/adadelta.py +13 -20
  90. mindspore/experimental/optim/adagrad.py +15 -22
  91. mindspore/experimental/optim/adam.py +17 -24
  92. mindspore/experimental/optim/adamax.py +14 -22
  93. mindspore/experimental/optim/adamw.py +28 -34
  94. mindspore/experimental/optim/asgd.py +15 -25
  95. mindspore/experimental/optim/lr_scheduler.py +27 -45
  96. mindspore/experimental/optim/nadam.py +14 -24
  97. mindspore/experimental/optim/optimizer.py +13 -23
  98. mindspore/experimental/optim/radam.py +18 -24
  99. mindspore/experimental/optim/rmsprop.py +14 -25
  100. mindspore/experimental/optim/rprop.py +15 -26
  101. mindspore/experimental/optim/sgd.py +9 -19
  102. mindspore/hal/__init__.py +4 -4
  103. mindspore/hal/contiguous_tensors_handle.py +2 -2
  104. mindspore/hal/memory.py +1 -0
  105. mindspore/include/api/cell.h +65 -5
  106. mindspore/include/api/cfg.h +24 -7
  107. mindspore/include/api/context.h +1 -0
  108. mindspore/include/api/delegate.h +10 -2
  109. mindspore/include/api/dual_abi_helper.h +100 -19
  110. mindspore/include/api/graph.h +14 -1
  111. mindspore/include/api/kernel.h +16 -3
  112. mindspore/include/api/kernel_api.h +9 -1
  113. mindspore/include/api/metrics/accuracy.h +9 -0
  114. mindspore/include/api/model.h +8 -1
  115. mindspore/include/api/model_group.h +4 -0
  116. mindspore/include/api/model_parallel_runner.h +2 -0
  117. mindspore/include/api/status.h +48 -10
  118. mindspore/include/api/types.h +8 -3
  119. mindspore/include/c_api/model_c.h +0 -58
  120. mindspore/include/c_api/tensor_c.h +0 -26
  121. mindspore/include/dataset/constants.h +9 -0
  122. mindspore/include/dataset/vision_ascend.h +1 -1
  123. mindspore/jpeg62.dll +0 -0
  124. mindspore/mindrecord/tools/cifar10.py +61 -11
  125. mindspore/mindrecord/tools/cifar10_to_mr.py +5 -0
  126. mindspore/mindspore_backend_common.dll +0 -0
  127. mindspore/mindspore_backend_manager.dll +0 -0
  128. mindspore/mindspore_common.dll +0 -0
  129. mindspore/mindspore_core.dll +0 -0
  130. mindspore/mindspore_cpu_res_manager.dll +0 -0
  131. mindspore/mindspore_dump.dll +0 -0
  132. mindspore/mindspore_frontend.dll +0 -0
  133. mindspore/mindspore_glog.dll +0 -0
  134. mindspore/mindspore_memory_pool.dll +0 -0
  135. mindspore/mindspore_ms_backend.dll +0 -0
  136. mindspore/mindspore_ops.dll +0 -0
  137. mindspore/mindspore_ops_host.dll +0 -0
  138. mindspore/mindspore_ops_kernel_common.dll +0 -0
  139. mindspore/mindspore_profiler.dll +0 -0
  140. mindspore/mindspore_pyboost.dll +0 -0
  141. mindspore/mindspore_pynative.dll +0 -0
  142. mindspore/mindspore_res_manager.dll +0 -0
  143. mindspore/mindspore_runtime_pipeline.dll +0 -0
  144. mindspore/mint/__init__.py +4 -44
  145. mindspore/mint/distributed/__init__.py +5 -0
  146. mindspore/mint/distributed/distributed.py +425 -19
  147. mindspore/mint/nn/__init__.py +1 -1
  148. mindspore/mint/nn/functional.py +53 -6
  149. mindspore/mint/nn/layer/_functions.py +163 -294
  150. mindspore/mint/nn/layer/activation.py +8 -6
  151. mindspore/mint/nn/layer/conv.py +125 -101
  152. mindspore/mint/nn/layer/normalization.py +11 -25
  153. mindspore/mint/optim/adam.py +19 -18
  154. mindspore/mint/optim/adamw.py +14 -8
  155. mindspore/mint/optim/sgd.py +5 -5
  156. mindspore/msobj140.dll +0 -0
  157. mindspore/mspdb140.dll +0 -0
  158. mindspore/mspdbcore.dll +0 -0
  159. mindspore/mspdbst.dll +0 -0
  160. mindspore/mspft140.dll +0 -0
  161. mindspore/msvcdis140.dll +0 -0
  162. mindspore/msvcp140_1.dll +0 -0
  163. mindspore/msvcp140_2.dll +0 -0
  164. mindspore/msvcp140_atomic_wait.dll +0 -0
  165. mindspore/msvcp140_codecvt_ids.dll +0 -0
  166. mindspore/nn/cell.py +488 -620
  167. mindspore/nn/grad/cell_grad.py +11 -12
  168. mindspore/nn/layer/activation.py +36 -36
  169. mindspore/nn/layer/basic.py +74 -77
  170. mindspore/nn/layer/channel_shuffle.py +4 -4
  171. mindspore/nn/layer/combined.py +4 -2
  172. mindspore/nn/layer/conv.py +86 -85
  173. mindspore/nn/layer/dense.py +9 -7
  174. mindspore/nn/layer/embedding.py +50 -52
  175. mindspore/nn/layer/image.py +38 -40
  176. mindspore/nn/layer/math.py +111 -112
  177. mindspore/nn/layer/normalization.py +56 -44
  178. mindspore/nn/layer/pooling.py +58 -63
  179. mindspore/nn/layer/rnn_cells.py +33 -33
  180. mindspore/nn/layer/rnns.py +56 -56
  181. mindspore/nn/layer/thor_layer.py +74 -73
  182. mindspore/nn/layer/transformer.py +11 -1
  183. mindspore/nn/learning_rate_schedule.py +20 -20
  184. mindspore/nn/loss/loss.py +79 -81
  185. mindspore/nn/optim/adam.py +2 -4
  186. mindspore/nn/optim/adasum.py +2 -2
  187. mindspore/nn/optim/lamb.py +1 -3
  188. mindspore/nn/optim/optimizer.py +1 -1
  189. mindspore/nn/optim/tft_wrapper.py +2 -3
  190. mindspore/nn/optim/thor.py +2 -2
  191. mindspore/nn/probability/distribution/_utils/utils.py +2 -2
  192. mindspore/nn/probability/distribution/exponential.py +2 -1
  193. mindspore/nn/probability/distribution/poisson.py +2 -1
  194. mindspore/nn/sparse/sparse.py +3 -3
  195. mindspore/nn/wrap/cell_wrapper.py +73 -42
  196. mindspore/nn/wrap/grad_reducer.py +37 -52
  197. mindspore/nn/wrap/loss_scale.py +72 -74
  198. mindspore/numpy/array_creations.py +7 -7
  199. mindspore/numpy/fft.py +1 -1
  200. mindspore/numpy/math_ops.py +1 -1
  201. mindspore/numpy/utils_const.py +1 -1
  202. mindspore/opencv_core452.dll +0 -0
  203. mindspore/opencv_imgcodecs452.dll +0 -0
  204. mindspore/opencv_imgproc452.dll +0 -0
  205. mindspore/ops/_grad_experimental/grad_comm_ops.py +51 -13
  206. mindspore/ops/_grad_experimental/grad_debug_ops.py +14 -0
  207. mindspore/ops/_grad_experimental/grad_inner_ops.py +0 -9
  208. mindspore/ops/_op_impl/cpu/__init__.py +1 -0
  209. mindspore/{experimental/es/__init__.py → ops/_op_impl/cpu/joinedstr_op.py} +12 -6
  210. mindspore/ops/_vmap/vmap_array_ops.py +6 -13
  211. mindspore/ops/_vmap/vmap_nn_ops.py +8 -16
  212. mindspore/ops/auto_generate/cpp_create_prim_instance_helper.py +29 -10
  213. mindspore/ops/auto_generate/gen_extend_func.py +5 -55
  214. mindspore/ops/auto_generate/gen_ops_def.py +753 -273
  215. mindspore/ops/auto_generate/gen_ops_prim.py +1687 -958
  216. mindspore/ops/auto_generate/pyboost_inner_prim.py +31 -1
  217. mindspore/ops/composite/__init__.py +10 -0
  218. mindspore/ops/composite/base.py +9 -5
  219. mindspore/ops/composite/multitype_ops/__init__.py +12 -1
  220. mindspore/ops/composite/multitype_ops/_compile_utils.py +132 -108
  221. mindspore/ops/composite/multitype_ops/_constexpr_utils.py +1 -1
  222. mindspore/ops/composite/multitype_ops/add_impl.py +70 -2
  223. mindspore/ops/composite/multitype_ops/div_impl.py +49 -0
  224. mindspore/ops/composite/multitype_ops/floordiv_impl.py +29 -0
  225. mindspore/ops/composite/multitype_ops/getitem_impl.py +11 -0
  226. mindspore/ops/composite/multitype_ops/mod_impl.py +5 -3
  227. mindspore/ops/composite/multitype_ops/mul_impl.py +49 -0
  228. mindspore/ops/composite/multitype_ops/setitem_impl.py +57 -0
  229. mindspore/ops/composite/multitype_ops/sub_impl.py +34 -0
  230. mindspore/ops/composite/multitype_ops/zeros_like_impl.py +14 -0
  231. mindspore/ops/function/__init__.py +4 -1
  232. mindspore/ops/function/_add_attr_func.py +11 -6
  233. mindspore/ops/function/array_func.py +17 -100
  234. mindspore/ops/function/debug_func.py +8 -5
  235. mindspore/ops/function/grad/grad_func.py +5 -13
  236. mindspore/ops/function/math_func.py +65 -399
  237. mindspore/ops/function/nn_func.py +44 -61
  238. mindspore/ops/function/other_func.py +4 -1
  239. mindspore/ops/function/random_func.py +31 -4
  240. mindspore/ops/functional.py +2 -3
  241. mindspore/ops/functional_overload.py +486 -18
  242. mindspore/ops/op_info_register.py +21 -0
  243. mindspore/ops/operations/__init__.py +5 -2
  244. mindspore/ops/operations/_custom_ops_utils.py +675 -8
  245. mindspore/ops/operations/_inner_ops.py +14 -18
  246. mindspore/ops/operations/_sequence_ops.py +1 -1
  247. mindspore/ops/operations/array_ops.py +4 -50
  248. mindspore/ops/operations/comm_ops.py +186 -41
  249. mindspore/ops/operations/custom_ops.py +244 -175
  250. mindspore/ops/operations/debug_ops.py +55 -4
  251. mindspore/ops/operations/image_ops.py +13 -13
  252. mindspore/ops/operations/manually_defined/ops_def.py +27 -28
  253. mindspore/ops/operations/math_ops.py +8 -9
  254. mindspore/ops/operations/nn_ops.py +6 -7
  255. mindspore/ops/primitive.py +9 -20
  256. mindspore/ops/tensor_method.py +52 -11
  257. mindspore/ops_generate/api/cpp_create_prim_instance_helper_generator.py +1 -1
  258. mindspore/ops_generate/api/functional_map_cpp_generator.py +10 -9
  259. mindspore/ops_generate/api/functions_cc_generator.py +58 -10
  260. mindspore/ops_generate/api/tensor_func_reg_cpp_generator.py +1 -1
  261. mindspore/ops_generate/common/base_generator.py +14 -0
  262. mindspore/ops_generate/common/gen_constants.py +7 -2
  263. mindspore/ops_generate/common/gen_utils.py +0 -19
  264. mindspore/ops_generate/common/op_proto.py +11 -4
  265. mindspore/ops_generate/common/template.py +88 -11
  266. mindspore/ops_generate/gen_ops.py +1 -1
  267. mindspore/ops_generate/op_def/lite_ops_cpp_generator.py +4 -4
  268. mindspore/ops_generate/op_def/ops_name_h_generator.py +0 -3
  269. mindspore/ops_generate/op_def/ops_primitive_h_generator.py +0 -4
  270. mindspore/ops_generate/op_def_py/op_prim_py_generator.py +5 -2
  271. mindspore/ops_generate/pyboost/auto_grad_impl_cc_generator.py +49 -8
  272. mindspore/ops_generate/pyboost/auto_grad_reg_cc_generator.py +2 -2
  273. mindspore/ops_generate/pyboost/gen_pyboost_func.py +31 -16
  274. mindspore/ops_generate/pyboost/op_template_parser.py +98 -72
  275. mindspore/ops_generate/pyboost/pyboost_functions_cpp_generator.py +70 -273
  276. mindspore/ops_generate/pyboost/pyboost_functions_h_generator.py +14 -6
  277. mindspore/ops_generate/pyboost/pyboost_functions_impl_cpp_generator.py +316 -0
  278. mindspore/ops_generate/pyboost/pyboost_functions_py_generator.py +1 -1
  279. mindspore/ops_generate/pyboost/pyboost_grad_function_cpp_generator.py +5 -3
  280. mindspore/ops_generate/pyboost/pyboost_inner_prim_generator.py +1 -1
  281. mindspore/ops_generate/pyboost/pyboost_internal_functions_cpp_generator.py +76 -0
  282. mindspore/ops_generate/pyboost/pyboost_internal_functions_h_generator.py +76 -0
  283. mindspore/ops_generate/pyboost/pyboost_internal_kernel_info_adapter_generator.py +125 -0
  284. mindspore/ops_generate/pyboost/pyboost_native_grad_functions_generator.py +4 -3
  285. mindspore/ops_generate/pyboost/pyboost_op_cpp_code_generator.py +348 -61
  286. mindspore/ops_generate/pyboost/pyboost_overload_functions_cpp_generator.py +1 -1
  287. mindspore/ops_generate/pyboost/pyboost_utils.py +118 -9
  288. mindspore/ops_generate/tensor_py_cc_generator.py +1 -24
  289. mindspore/parallel/_auto_parallel_context.py +9 -17
  290. mindspore/parallel/_cell_wrapper.py +106 -40
  291. mindspore/parallel/_parallel_serialization.py +4 -3
  292. mindspore/parallel/_ps_context.py +4 -6
  293. mindspore/parallel/_tensor.py +167 -12
  294. mindspore/parallel/_transformer/moe.py +1 -1
  295. mindspore/parallel/_transformer/transformer.py +17 -12
  296. mindspore/parallel/_utils.py +5 -11
  297. mindspore/parallel/auto_parallel.py +33 -12
  298. mindspore/parallel/checkpoint_convert.py +3 -3
  299. mindspore/parallel/checkpoint_transform.py +5 -1
  300. mindspore/parallel/cluster/process_entity/_api.py +88 -49
  301. mindspore/parallel/cluster/process_entity/_utils.py +95 -7
  302. mindspore/parallel/cluster/run.py +48 -7
  303. mindspore/parallel/function/__init__.py +8 -1
  304. mindspore/parallel/function/reshard_func.py +7 -6
  305. mindspore/parallel/nn/__init__.py +15 -2
  306. mindspore/parallel/nn/parallel_cell_wrapper.py +50 -14
  307. mindspore/parallel/nn/parallel_grad_reducer.py +7 -14
  308. mindspore/parallel/shard.py +9 -23
  309. mindspore/parallel/transform_safetensors.py +468 -174
  310. mindspore/pgodb140.dll +0 -0
  311. mindspore/pgort140.dll +0 -0
  312. mindspore/profiler/__init__.py +2 -1
  313. mindspore/profiler/analysis/parser/timeline_assembly_factory/ascend_timeline_assembler.py +7 -7
  314. mindspore/profiler/analysis/parser/timeline_assembly_factory/base_timeline_assembler.py +3 -0
  315. mindspore/profiler/analysis/parser/timeline_assembly_factory/trace_view_container.py +3 -0
  316. mindspore/profiler/analysis/parser/timeline_creator/cpu_op_timeline_creator.py +3 -3
  317. mindspore/profiler/analysis/parser/timeline_creator/fwk_timeline_creator.py +3 -3
  318. mindspore/profiler/analysis/parser/timeline_creator/msprof_timeline_creator.py +4 -4
  319. mindspore/profiler/analysis/parser/timeline_creator/scope_layer_timeline_creator.py +3 -3
  320. mindspore/profiler/analysis/parser/timeline_event/fwk_event.py +4 -1
  321. mindspore/profiler/analysis/parser/timeline_event/timeline_event_pool.py +2 -1
  322. mindspore/profiler/analysis/task_manager.py +1 -1
  323. mindspore/profiler/analysis/viewer/ascend_communication_viewer.py +5 -1
  324. mindspore/profiler/analysis/viewer/ascend_integrate_viewer.py +2 -1
  325. mindspore/profiler/analysis/viewer/ascend_kernel_details_viewer.py +10 -9
  326. mindspore/profiler/analysis/viewer/ascend_op_memory_viewer.py +43 -23
  327. mindspore/profiler/analysis/viewer/ascend_step_trace_time_viewer.py +3 -2
  328. mindspore/profiler/analysis/viewer/ms_minddata_viewer.py +9 -5
  329. mindspore/profiler/analysis/viewer/ms_operator_details_viewer.py +132 -0
  330. mindspore/profiler/common/constant.py +16 -0
  331. mindspore/profiler/common/msprof_cmd_tool.py +2 -2
  332. mindspore/profiler/common/path_manager.py +9 -0
  333. mindspore/profiler/common/profiler_context.py +50 -29
  334. mindspore/profiler/common/profiler_info.py +0 -16
  335. mindspore/profiler/common/profiler_meta_data.py +1 -0
  336. mindspore/profiler/common/profiler_op_analyse.py +239 -0
  337. mindspore/profiler/common/profiler_output_path.py +23 -8
  338. mindspore/profiler/common/profiler_parameters.py +128 -35
  339. mindspore/profiler/dynamic_profile/__init__.py +0 -0
  340. mindspore/profiler/dynamic_profile/dynamic_monitor_proxy.py +39 -0
  341. mindspore/profiler/dynamic_profile/dynamic_profiler_config_context.py +666 -0
  342. mindspore/profiler/dynamic_profile/dynamic_profiler_utils.py +62 -0
  343. mindspore/profiler/dynamic_profiler.py +374 -338
  344. mindspore/profiler/envprofiler.py +42 -12
  345. mindspore/profiler/experimental_config.py +112 -7
  346. mindspore/profiler/mstx.py +33 -12
  347. mindspore/profiler/platform/__init__.py +2 -3
  348. mindspore/profiler/platform/cpu_profiler.py +10 -4
  349. mindspore/profiler/platform/npu_profiler.py +30 -20
  350. mindspore/profiler/profiler.py +218 -154
  351. mindspore/profiler/profiler_action_controller.py +65 -77
  352. mindspore/profiler/profiler_interface.py +2 -2
  353. mindspore/profiler/schedule.py +10 -4
  354. mindspore/rewrite/common/config.py +1 -0
  355. mindspore/rewrite/common/namer.py +1 -0
  356. mindspore/rewrite/common/namespace.py +1 -0
  357. mindspore/rewrite/node/node.py +31 -11
  358. mindspore/rewrite/parsers/assign_parser.py +1 -1
  359. mindspore/rewrite/symbol_tree/symbol_tree.py +2 -2
  360. mindspore/run_check/_check_version.py +7 -10
  361. mindspore/runtime/__init__.py +8 -6
  362. mindspore/runtime/event.py +10 -4
  363. mindspore/runtime/executor.py +87 -45
  364. mindspore/runtime/memory.py +22 -30
  365. mindspore/runtime/thread_bind_core.py +299 -165
  366. mindspore/safeguard/rewrite_obfuscation.py +12 -13
  367. mindspore/swresample-4.dll +0 -0
  368. mindspore/swscale-6.dll +0 -0
  369. mindspore/tbbmalloc.dll +0 -0
  370. mindspore/tinyxml2.dll +0 -0
  371. mindspore/train/_utils.py +9 -5
  372. mindspore/train/amp.py +43 -23
  373. mindspore/train/callback/__init__.py +5 -5
  374. mindspore/train/callback/_callback.py +2 -1
  375. mindspore/train/callback/_checkpoint.py +4 -14
  376. mindspore/train/callback/_flops_collector.py +11 -7
  377. mindspore/train/callback/_landscape.py +0 -1
  378. mindspore/train/callback/_train_fault_tolerance.py +72 -18
  379. mindspore/train/data_sink.py +15 -6
  380. mindspore/train/dataset_helper.py +14 -5
  381. mindspore/train/model.py +49 -47
  382. mindspore/train/serialization.py +168 -126
  383. mindspore/train/summary/summary_record.py +13 -2
  384. mindspore/train/train_thor/model_thor.py +2 -2
  385. mindspore/turbojpeg.dll +0 -0
  386. mindspore/utils/__init__.py +3 -2
  387. mindspore/utils/dryrun.py +0 -6
  388. mindspore/utils/runtime_execution_order_check.py +162 -78
  389. mindspore/utils/sdc_detect.py +68 -0
  390. mindspore/utils/utils.py +14 -17
  391. mindspore/vcmeta.dll +0 -0
  392. mindspore/vcruntime140.dll +0 -0
  393. mindspore/vcruntime140_1.dll +0 -0
  394. mindspore/version.py +1 -1
  395. {mindspore-2.6.0.dist-info → mindspore-2.7.0.dist-info}/METADATA +5 -4
  396. {mindspore-2.6.0.dist-info → mindspore-2.7.0.dist-info}/RECORD +400 -439
  397. mindspore/_deprecated/jit.py +0 -198
  398. mindspore/_extends/remote/kernel_build_server_ascend.py +0 -75
  399. mindspore/communication/_hccl_management.py +0 -297
  400. mindspore/experimental/es/embedding_service.py +0 -891
  401. mindspore/experimental/es/embedding_service_layer.py +0 -581
  402. mindspore/profiler/common/validator/__init__.py +0 -14
  403. mindspore/profiler/common/validator/validate_path.py +0 -84
  404. mindspore/profiler/parser/__init__.py +0 -14
  405. mindspore/profiler/parser/aicpu_data_parser.py +0 -272
  406. mindspore/profiler/parser/ascend_analysis/__init__.py +0 -14
  407. mindspore/profiler/parser/ascend_analysis/constant.py +0 -71
  408. mindspore/profiler/parser/ascend_analysis/file_manager.py +0 -180
  409. mindspore/profiler/parser/ascend_analysis/function_event.py +0 -185
  410. mindspore/profiler/parser/ascend_analysis/fwk_cann_parser.py +0 -136
  411. mindspore/profiler/parser/ascend_analysis/fwk_file_parser.py +0 -131
  412. mindspore/profiler/parser/ascend_analysis/msprof_timeline_parser.py +0 -104
  413. mindspore/profiler/parser/ascend_analysis/path_manager.py +0 -313
  414. mindspore/profiler/parser/ascend_analysis/profiler_info_parser.py +0 -123
  415. mindspore/profiler/parser/ascend_analysis/tlv_decoder.py +0 -86
  416. mindspore/profiler/parser/ascend_analysis/trace_event_manager.py +0 -75
  417. mindspore/profiler/parser/ascend_cluster_generator.py +0 -116
  418. mindspore/profiler/parser/ascend_communicate_generator.py +0 -314
  419. mindspore/profiler/parser/ascend_flops_generator.py +0 -116
  420. mindspore/profiler/parser/ascend_fpbp_generator.py +0 -82
  421. mindspore/profiler/parser/ascend_hccl_generator.py +0 -271
  422. mindspore/profiler/parser/ascend_integrate_generator.py +0 -42
  423. mindspore/profiler/parser/ascend_memory_generator.py +0 -185
  424. mindspore/profiler/parser/ascend_msprof_exporter.py +0 -282
  425. mindspore/profiler/parser/ascend_msprof_generator.py +0 -187
  426. mindspore/profiler/parser/ascend_op_generator.py +0 -334
  427. mindspore/profiler/parser/ascend_steptrace_generator.py +0 -94
  428. mindspore/profiler/parser/ascend_timeline_generator.py +0 -545
  429. mindspore/profiler/parser/base_timeline_generator.py +0 -483
  430. mindspore/profiler/parser/container.py +0 -229
  431. mindspore/profiler/parser/cpu_gpu_timeline_generator.py +0 -697
  432. mindspore/profiler/parser/flops_parser.py +0 -531
  433. mindspore/profiler/parser/framework_enum.py +0 -111
  434. mindspore/profiler/parser/framework_parser.py +0 -464
  435. mindspore/profiler/parser/framework_struct.py +0 -61
  436. mindspore/profiler/parser/gpu_analysis/__init__.py +0 -14
  437. mindspore/profiler/parser/gpu_analysis/function_event.py +0 -44
  438. mindspore/profiler/parser/gpu_analysis/fwk_file_parser.py +0 -89
  439. mindspore/profiler/parser/gpu_analysis/profiler_info_parser.py +0 -72
  440. mindspore/profiler/parser/hccl_parser.py +0 -573
  441. mindspore/profiler/parser/hwts_log_parser.py +0 -122
  442. mindspore/profiler/parser/integrator.py +0 -526
  443. mindspore/profiler/parser/memory_usage_parser.py +0 -277
  444. mindspore/profiler/parser/minddata_analyzer.py +0 -800
  445. mindspore/profiler/parser/minddata_parser.py +0 -186
  446. mindspore/profiler/parser/minddata_pipeline_parser.py +0 -299
  447. mindspore/profiler/parser/op_intermediate_parser.py +0 -149
  448. mindspore/profiler/parser/optime_parser.py +0 -250
  449. mindspore/profiler/parser/profiler_info.py +0 -213
  450. mindspore/profiler/parser/step_trace_parser.py +0 -666
  451. mindspore/utils/hooks.py +0 -81
  452. /mindspore/common/{_auto_dynamic.py → dynamic_shape/_auto_dynamic.py} +0 -0
  453. {mindspore-2.6.0.dist-info → mindspore-2.7.0.dist-info}/WHEEL +0 -0
  454. {mindspore-2.6.0.dist-info → mindspore-2.7.0.dist-info}/entry_points.txt +0 -0
  455. {mindspore-2.6.0.dist-info → mindspore-2.7.0.dist-info}/top_level.txt +0 -0
mindspore/pgodb140.dll CHANGED
Binary file
mindspore/pgort140.dll CHANGED
Binary file
@@ -28,6 +28,7 @@ __all__ = [
28
28
  "ProfilerActivity",
29
29
  "AicoreMetrics",
30
30
  "ExportType",
31
+ "HostSystem",
31
32
  "DynamicProfilerMonitor",
32
33
  "mstx",
33
34
  "_ExperimentalConfig",
@@ -39,7 +40,7 @@ from mindspore.profiler.profiler import Profiler
39
40
  from mindspore.profiler.profiler import tensorboard_trace_handler
40
41
  from mindspore.profiler.schedule import Schedule as schedule
41
42
  from mindspore.profiler.envprofiler import EnvProfiler
42
- from mindspore.profiler.common.constant import ProfilerLevel, ProfilerActivity, AicoreMetrics, ExportType
43
+ from mindspore.profiler.common.constant import ProfilerLevel, ProfilerActivity, AicoreMetrics, ExportType, HostSystem
43
44
  from mindspore.profiler.dynamic_profiler import DynamicProfilerMonitor
44
45
  from mindspore.profiler.experimental_config import _ExperimentalConfig
45
46
  from mindspore.profiler.profiler import Profile as profile
@@ -17,8 +17,8 @@ from typing import List, Dict, Any
17
17
  from decimal import Decimal
18
18
  from collections import defaultdict
19
19
 
20
- from mindspore import context
21
20
  from mindspore import log as logger
21
+ from mindspore.profiler.common.log import ProfilerLogger
22
22
  from mindspore.profiler.common.constant import EventConstant, TimelineLayerName, ProfilerLevel, JitLevel
23
23
  from mindspore.profiler.analysis.parser.timeline_event.base_event import BaseEvent
24
24
  from mindspore.profiler.analysis.parser.timeline_event.timeline_event_pool import TimelineEventPool
@@ -39,9 +39,10 @@ class AscendTimelineAssembler(BaseTimelineAssembler):
39
39
  def __init__(self, **kwargs):
40
40
  super().__init__()
41
41
  self._profiler_level = kwargs.get("profiler_level")
42
- self._context_mode = kwargs.get("context_mode")
43
42
  self._jit_level = kwargs.get("jit_level")
44
43
  self._init_creators()
44
+ ProfilerLogger.init(kwargs.get("ascend_ms_dir"))
45
+ self._logger = ProfilerLogger.get_instance()
45
46
 
46
47
  def _init_creators(self):
47
48
  """Initialize trace creators."""
@@ -128,17 +129,16 @@ class AscendTimelineAssembler(BaseTimelineAssembler):
128
129
  """Create flow events between framework and hardware events."""
129
130
  acl_to_npu_flow_dict = self._msprof_creator.get_acl_to_npu_flow_dict()
130
131
  fwk_launch_op_list = self.trace_view_container.kernel_launch_op_event
131
- # The graph mode O2 does not have the flow from CANN to hardware at each step
132
+ # The GE backend does not have the flow from CANN to hardware at each step
132
133
  if not acl_to_npu_flow_dict and self._jit_level != JitLevel.GRAPH_LEVEL:
133
134
  logger.error("Cannot find connection between CANN layer and Ascend Hardware layer.")
134
135
  return []
135
- # The graph model O2 does not have "KernelLaunch" or "LaunchTask" keywords
136
+ # The GE backend does not have "KernelLaunch" or "LaunchTask" keywords
136
137
  if not fwk_launch_op_list and self._jit_level != JitLevel.GRAPH_LEVEL:
137
138
  logger.warning("Cannot find launch op in MindSpore framework.")
138
139
  return []
139
- if (set(acl_to_npu_flow_dict.keys()) != set(fwk_launch_op_list.keys()) and
140
- self._context_mode == context.PYNATIVE_MODE):
141
- logger.warning(
140
+ if set(acl_to_npu_flow_dict.keys()) != set(fwk_launch_op_list.keys()):
141
+ self._logger.warning(
142
142
  "The number of launch op threads in MindSpore framework is inconsistent with the CANN layer.")
143
143
 
144
144
  fwk_to_npu_flows = []
@@ -1,3 +1,6 @@
1
+ # Copyright 2025 Huawei Technologies Co., Ltd
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
1
4
  # you may not use this file except in compliance with the License.
2
5
  # You may obtain a copy of the License at
3
6
  #
@@ -1,3 +1,6 @@
1
+ # Copyright 2025 Huawei Technologies Co., Ltd
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
1
4
  # you may not use this file except in compliance with the License.
2
5
  # You may obtain a copy of the License at
3
6
  #
@@ -31,15 +31,15 @@ class CpuOpTimelineCreator(BaseTimelineCreator):
31
31
  super().__init__()
32
32
  self.scope_data: List[CpuOpCompleteEvent] = []
33
33
 
34
- def create(self, cpu_info_lines: List[str]) -> None:
34
+ def create(self, data: List[str]) -> None:
35
35
  """Create timeline event pools from CPU info lines."""
36
- if not cpu_info_lines:
36
+ if not data:
37
37
  return
38
38
 
39
39
  pool = TimelineEventPool(EventConstant.CPU_OP_PID)
40
40
  self.event_pools[EventConstant.CPU_OP_PID] = pool
41
41
 
42
- self._create_base_events(pool, cpu_info_lines)
42
+ self._create_base_events(pool, data)
43
43
  self._create_meta_event(pool)
44
44
 
45
45
  def _create_base_events(self, pool: TimelineEventPool, cpu_info_lines: List[str]) -> None:
@@ -29,15 +29,15 @@ from mindspore.profiler.analysis.parser.timeline_event.fwk_event import (
29
29
  class FwkTimelineCreator(BaseTimelineCreator):
30
30
  """Create timeline event pools for framework operations."""
31
31
 
32
- def create(self, fwk_tlv_data: List[Dict]) -> None:
32
+ def create(self, data: List[Dict]) -> None:
33
33
  """Create timeline event pools from framework TLV data."""
34
- if not fwk_tlv_data:
34
+ if not data:
35
35
  return
36
36
 
37
37
  pool = TimelineEventPool(EventConstant.MINDSPORE_PID)
38
38
  self.event_pools[EventConstant.MINDSPORE_PID] = pool
39
39
 
40
- self._create_base_events(pool, fwk_tlv_data)
40
+ self._create_base_events(pool, data)
41
41
  self._create_meta_event(pool)
42
42
 
43
43
  def _create_base_events(self, pool: TimelineEventPool, fwk_tlv_data: List[Dict]) -> None:
@@ -35,12 +35,12 @@ class MsprofTimelineCreator(BaseTimelineCreator):
35
35
  self.msprof_timeline_raw_data = []
36
36
  self.acl_to_npu_flow_dict: Dict[int, Dict[str, List[MsprofCompleteEvent]]] = {}
37
37
 
38
- def create(self, msprof_timeline_data: List[Dict]) -> None:
38
+ def create(self, data: List[Dict]) -> None:
39
39
  """Create timeline event pools from MsProf timeline data."""
40
- if not msprof_timeline_data:
40
+ if not data:
41
41
  return
42
- self.msprof_timeline_raw_data = msprof_timeline_data
43
- flow_dict, complete_event_map = self._create_base_events(msprof_timeline_data)
42
+ self.msprof_timeline_raw_data = data
43
+ flow_dict, complete_event_map = self._create_base_events(data)
44
44
  self._create_acl_to_npu_flow_dict(flow_dict, complete_event_map)
45
45
 
46
46
  def _create_base_events(self, msprof_timeline_data: List[Dict]) -> Tuple[Dict, Dict]:
@@ -32,15 +32,15 @@ from mindspore.profiler.analysis.parser.timeline_event.scope_layer_event import
32
32
  class ScopeLayerTimelineCreator(BaseTimelineCreator):
33
33
  """Create timeline event pools for scope layer operations."""
34
34
 
35
- def create(self, event_list: List[BaseEvent]) -> None:
35
+ def create(self, data: List[BaseEvent]) -> None:
36
36
  """Create timeline event pools from scope layer events."""
37
- if not event_list:
37
+ if not data:
38
38
  return
39
39
 
40
40
  pool = TimelineEventPool(EventConstant.SCOPE_LAYER_PID)
41
41
  self.event_pools[EventConstant.SCOPE_LAYER_PID] = pool
42
42
 
43
- self._create_base_events(pool, event_list)
43
+ self._create_base_events(pool, data)
44
44
  self._create_meta_event(pool)
45
45
 
46
46
  def _create_base_events(self, pool: TimelineEventPool, event_list: List[BaseEvent]) -> None:
@@ -110,6 +110,7 @@ class ProfilerEvent(Enum):
110
110
  PYBOOST_MALLOC_INPUT = "PyBoostMallocInput"
111
111
  PYBOOST_MALLOC_OUTPUT = "PyBoostMallocOutput"
112
112
  PYBOOST_LAUNCH_ACLLNN = "PyBoostLaunchAclnn"
113
+ PYBOOST_LAUNCH_ATB = "PyBoostLaunchAtb"
113
114
  # pybind api
114
115
  PYNATIVE_NEW_GRAPH = "PyNativeNewGraph"
115
116
  PYNATIVE_END_GRAPH = "PyNativeEndGraph"
@@ -179,7 +180,9 @@ class FwkProfileDataField:
179
180
  The enum value string.
180
181
  """
181
182
  try:
182
- return list(enum_class)[index].value
183
+ # pylint: disable=protected-access
184
+ name = enum_class._member_names_[index]
185
+ return enum_class[name].value
183
186
  except IndexError:
184
187
  logger.warning(f"Invalid {enum_type} index: {index}")
185
188
  return enum_class.DEFAULT.value
@@ -86,7 +86,8 @@ class TimelineEventPool:
86
86
  self.start_to_end_events_pairs[flow_key] = {"start": [], "end": []}
87
87
  self.start_to_end_events_pairs[flow_key]["end"].append(event)
88
88
 
89
- def _get_events(self, event_dict: dict) -> List[BaseEvent]:
89
+ @staticmethod
90
+ def _get_events(event_dict: dict) -> List[BaseEvent]:
90
91
  """Helper function to get events from a dictionary."""
91
92
  events = []
92
93
  for event_list in event_dict.values():
@@ -110,8 +110,8 @@ class TaskManager:
110
110
  else workflow
111
111
  )
112
112
 
113
+ data = {}
113
114
  try:
114
- data = {}
115
115
  for parser in parsers:
116
116
  parser_start_time = time.perf_counter()
117
117
  data = parser.parse(data)
@@ -352,7 +352,11 @@ class AscendCommunicationViewer(BaseViewer):
352
352
  if bandwidth_msg in add_list:
353
353
  total_bandwidth_info_dict[transport_type][bandwidth_msg] += value
354
354
  if bandwidth_msg in dict_list:
355
- self._combine_size_distribution(value, total_bandwidth_info_dict[transport_type][bandwidth_msg])
355
+ self._combine_size_distribution(
356
+ value,
357
+ total_bandwidth_info_dict.get(transport_type, {})
358
+ .get(bandwidth_msg, defaultdict(lambda: [0, 0]))
359
+ )
356
360
 
357
361
  def _compute_time_ratio(self, total_time_info_dict: dict):
358
362
  """compute time ratio"""
@@ -25,7 +25,8 @@ from mindspore.profiler.common.log import ProfilerLogger
25
25
  class AscendIntegrateViewer(BaseViewer):
26
26
  """Ascend integrate viewer"""
27
27
 
28
- CSV_PREFIX_NAME = ["l2_cache", "api_statistic", "op_statistic", "static_op_mem"]
28
+ CSV_PREFIX_NAME = ["l2_cache", "api_statistic", "op_statistic", "static_op_mem",
29
+ "hbm", "nic", "roce", "pcie", "hccs"]
29
30
  AI_CPU_CSV_PATTERN = "aicpu_[0-9]*.csv"
30
31
 
31
32
  def __init__(self, **kwargs):
@@ -13,7 +13,6 @@
13
13
  # limitations under the License.
14
14
  # ============================================================================
15
15
  """Ascend kernel details viewer"""
16
- import csv
17
16
  import os
18
17
  from decimal import Decimal
19
18
 
@@ -25,8 +24,7 @@ from mindspore.profiler.common.constant import (
25
24
  ProfilerActivity
26
25
  )
27
26
  from mindspore.profiler.common.log import ProfilerLogger
28
- from mindspore.profiler.common.path_manager import PathManager
29
-
27
+ from mindspore.profiler.common.file_manager import FileManager
30
28
  from mindspore import log as logger
31
29
 
32
30
 
@@ -110,12 +108,15 @@ class AscendKernelDetailsViewer(BaseViewer):
110
108
  Write data to csv file.
111
109
  """
112
110
  self._logger.info("Kernel details saved start")
113
- PathManager.check_directory_path_writeable(os.path.dirname(self._save_path))
114
- with open(self._save_path, "w", newline="", encoding="utf-8") as csvfile:
115
- writer = csv.writer(csvfile)
116
- writer.writerow(self.kernel_details_headers)
117
- for row in self.op_summary:
118
- writer.writerow([row[field] for field in self.op_summary_headers])
111
+ csv_data = []
112
+ for row in self.op_summary:
113
+ csv_row = [row[field] for field in self.op_summary_headers]
114
+ csv_data.append(csv_row)
115
+ FileManager.create_csv_file(
116
+ file_path=self._save_path,
117
+ data=csv_data,
118
+ headers=self.kernel_details_headers
119
+ )
119
120
  self._logger.info("Kernel details saved done")
120
121
 
121
122
  def _update_headers(self):
@@ -20,10 +20,10 @@ from enum import Enum
20
20
  from typing import Any, Dict, List
21
21
  from abc import ABC
22
22
 
23
- from mindspore.profiler.parser.ascend_analysis.tlv_decoder import TLVDecoder
23
+ from mindspore.profiler.common.tlv_decoder import TLVDecoder
24
24
  from mindspore.profiler.common.file_manager import FileManager
25
25
  from mindspore.profiler.common.log import ProfilerLogger
26
- from mindspore.profiler.common.constant import ProfilerActivity
26
+ from mindspore.profiler.common.constant import ProfilerActivity, FileConstant
27
27
 
28
28
 
29
29
  class OpMemoryIndexEnum(Enum):
@@ -68,9 +68,7 @@ class OpMemoryEvent(BaseEvent):
68
68
 
69
69
  def __init__(self, data: Dict):
70
70
  super().__init__(data)
71
- self.fix_size_data = struct.unpack(
72
- self.FIX_DATA_FORMAT, self._origin_data.get("fix_size_bytes")
73
- )
71
+ self.fix_size_data = self._origin_data[FileConstant.FIX_SIZE_DATA]
74
72
 
75
73
  @property
76
74
  def device_id(self):
@@ -237,9 +235,10 @@ class AscendOpMemoryViewer:
237
235
  self._logger.info("Read fwk binary file start")
238
236
  op_name_file_path = os.path.join(self._framework_path, self.FWK_BINARY_FILE_NAME)
239
237
  raw_bin_data = FileManager.read_file_content(op_name_file_path, mode="rb")
240
- self._op_memory_events = TLVDecoder.decode(
241
- raw_bin_data, OpMemoryEvent, OpMemoryEvent.FIX_DATA_SIZE
238
+ op_memory_decode_data = TLVDecoder.decode(
239
+ raw_bin_data, OpMemoryEvent.FIX_DATA_FORMAT, OpMemoryEvent.FIX_DATA_SIZE
242
240
  )
241
+ self._op_memory_events = [OpMemoryEvent(data) for data in op_memory_decode_data]
243
242
  self._op_memory_events = sorted(self._op_memory_events, key=lambda x: x.create_at)
244
243
  self._logger.info("Read fwk binary file done, %d events", len(self._op_memory_events))
245
244
 
@@ -271,21 +270,42 @@ class AscendOpMemoryViewer:
271
270
  return []
272
271
 
273
272
  return [
274
- alloc_event.owner, # "Name"
275
- alloc_event.size / self.BYTES_TO_KB, # "Size(KB)"
276
- alloc_event.create_at / self.NS_TO_US, # "Allocation Time(us)"
277
- self.EMPTY_VALUE if free_event is None else free_event.create_at / self.NS_TO_US, # "Release Time(us)"
278
- self.EMPTY_VALUE, # "Active Release Time(us)"
279
- self.EMPTY_VALUE if free_event is None else (free_event.create_at - alloc_event.create_at) / self.NS_TO_US, # "Duration(us)"
280
- self.EMPTY_VALUE, # "Active Duration(us)"
281
- alloc_event.alloc_size / self.BYTES_TO_MB, # "Allocation Total Allocated(MB)"
282
- alloc_event.used_size / self.BYTES_TO_MB, # "Allocation Total Reserved(MB)"
283
- alloc_event.alloc_size / self.BYTES_TO_MB, # "Allocation Total Active(MB)"
284
- self.EMPTY_VALUE if free_event is None else free_event.alloc_size / self.BYTES_TO_MB, # "Release Total Allocated(MB)"
285
- self.EMPTY_VALUE if free_event is None else free_event.used_size / self.BYTES_TO_MB, # "Release Total Reserved(MB)"
286
- self.EMPTY_VALUE if free_event is None else free_event.alloc_size / self.BYTES_TO_MB, # "Release Total Active(MB)"
287
- alloc_event.stream_ptr, # "Stream Ptr"
288
- self.DEVICE_TYPE_FMT.format(alloc_event.device_id), # "Device Type"
273
+ alloc_event.owner, # "Name"
274
+ alloc_event.size / self.BYTES_TO_KB, # "Size(KB)"
275
+ alloc_event.create_at / self.NS_TO_US, # "Allocation Time(us)"
276
+ (
277
+ self.EMPTY_VALUE
278
+ if free_event is None
279
+ else free_event.create_at / self.NS_TO_US
280
+ ), # "Release Time(us)"
281
+ self.EMPTY_VALUE, # "Active Release Time(us)"
282
+ (
283
+ self.EMPTY_VALUE
284
+ if free_event is None
285
+ else (free_event.create_at - alloc_event.create_at) / self.NS_TO_US
286
+ ), # "Duration(us)"
287
+ self.EMPTY_VALUE, # "Active Duration(us)"
288
+ alloc_event.used_size
289
+ / self.BYTES_TO_MB, # "Allocation Total Allocated(MB)"
290
+ alloc_event.alloc_size / self.BYTES_TO_MB, # "Allocation Total Reserved(MB)"
291
+ alloc_event.used_size / self.BYTES_TO_MB, # "Allocation Total Active(MB)"
292
+ (
293
+ self.EMPTY_VALUE
294
+ if free_event is None
295
+ else free_event.used_size / self.BYTES_TO_MB
296
+ ), # "Release Total Allocated(MB)"
297
+ (
298
+ self.EMPTY_VALUE
299
+ if free_event is None
300
+ else free_event.alloc_size / self.BYTES_TO_MB
301
+ ), # "Release Total Reserved(MB)"
302
+ (
303
+ self.EMPTY_VALUE
304
+ if free_event is None
305
+ else free_event.used_size / self.BYTES_TO_MB
306
+ ), # "Release Total Active(MB)"
307
+ alloc_event.stream_ptr, # "Stream Ptr"
308
+ self.DEVICE_TYPE_FMT.format(alloc_event.device_id), # "Device Type"
289
309
  ]
290
310
 
291
311
  def _get_op_mem_row_data(self, event_list: List[OpMemoryEvent]):
@@ -310,7 +330,7 @@ class AscendOpMemoryViewer:
310
330
  res.append(self._combine_alloc_and_free_event(alloc_event, free_event))
311
331
  alloc_event, free_event = None, None
312
332
  elif alloc_event is None and free_event:
313
- self._logger.error("Alloc event is None, but free event is not None")
333
+ self._logger.warning("Alloc event is None, but free event is not None")
314
334
 
315
335
  if alloc_event:
316
336
  res.append(self._combine_alloc_and_free_event(alloc_event))
@@ -260,7 +260,7 @@ class AscendStepTraceTimeViewer(BaseViewer):
260
260
  # communication not overlapped time exclude receive
261
261
  step_trace_time_data[StepTraceTimeHeaders.COMMUNICATION_NOT_OVERLAPPED_EXCLUDE_RECEIVE.value] = (
262
262
  step_trace_time_data[StepTraceTimeHeaders.COMMUNICATION_NOT_OVERLAPPED.value]
263
- - step_trace_time_data[StepTraceTimeHeaders.BUBBLE.value]
263
+ - step_trace_time_data.get(StepTraceTimeHeaders.BUBBLE.value, Decimal('0.000'))
264
264
  )
265
265
  step_trace_time_data[StepTraceTimeHeaders.PREPARING.value] = self._calculate_prepare_time_by_step(
266
266
  self.computing_np, self.communication_np, start_time, step_id
@@ -355,7 +355,8 @@ class AscendStepTraceTimeViewer(BaseViewer):
355
355
  logger.info("No HCCL events in the given time range, skip calculate stage and bubble")
356
356
  return Decimal(0), Decimal(0)
357
357
 
358
- total_hccl_time = filtered_hccl_events_np["ts"][-1] - filtered_hccl_events_np["ts"][0]
358
+ total_hccl_time = filtered_hccl_events_np["ts"][-1] - filtered_hccl_events_np["ts"][0] + \
359
+ filtered_hccl_events_np["dur"][-1]
359
360
  bubble_time = np.sum(
360
361
  filtered_hccl_events_np["dur"][
361
362
  np.array(
@@ -367,11 +367,15 @@ class MindDataPiplineSummaryViewer(BaseViewer):
367
367
  raise ProfilerRawFileException('The contents of MindData CPU utilization JSON file is wrong.')
368
368
  # Note: The CPU utilization data may have an extra entry with op_id=-1
369
369
  # Omit info for op_id=1
370
- dict_opid_cpuutil = {
371
- op["op_id"]: [op_sys + op_usr for op_sys, op_usr in
372
- zip(op["metrics"]["sys_utilization"], op["metrics"]["user_utilization"])]
373
- for op in cpu_op_info if op and op["op_id"] != -1
374
- }
370
+ dict_opid_cpuutil = {}
371
+ for op in cpu_op_info:
372
+ if not op or op["op_id"] == -1:
373
+ continue
374
+ cpu_utilization = []
375
+ for op_sys, op_usr in zip(op["metrics"]["sys_utilization"], op["metrics"]["user_utilization"]):
376
+ cpu_utilization.append(op_sys + op_usr)
377
+ dict_opid_cpuutil[op["op_id"]] = cpu_utilization
378
+
375
379
  # Initialize oplist_avg_cpu_pct with -1 for each pipeline op, since
376
380
  # CPU utilization data may not have information for each pipeline op
377
381
  oplist_avg_cpu_pct = [-1] * len(dict_opid_cpuutil)
@@ -0,0 +1,132 @@
1
+ # Copyright 2025 Huawei Technologies Co., Ltd
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ # ============================================================================
15
+ """ms operator details viewer"""
16
+ import os
17
+ import struct
18
+ from abc import ABC
19
+ from enum import Enum
20
+ from typing import Dict, Any
21
+
22
+ from mindspore.profiler.common.file_manager import FileManager
23
+ from mindspore.profiler.analysis.viewer.base_viewer import BaseViewer
24
+ from mindspore.profiler.common.tlv_decoder import TLVDecoder
25
+ from mindspore.profiler.common.log import ProfilerLogger
26
+
27
+
28
+ class OperatorDetailsIndexEnum(Enum):
29
+ """Operator details index defining."""
30
+
31
+ NAME = 0
32
+ INPUT_SHAPES = 1
33
+ INPUT_TYPE = 2
34
+
35
+
36
+ class BaseEvent(ABC):
37
+ """Base class for all event types."""
38
+
39
+ def __init__(self, data: Dict):
40
+ if not isinstance(data, dict):
41
+ raise TypeError("Input data must be dict.")
42
+ self._origin_data = data
43
+
44
+
45
+ class OperatorDetailsEvent(BaseEvent):
46
+ """Operator details event."""
47
+
48
+ FIX_DATA_FORMAT = ""
49
+ FIX_DATA_SIZE = struct.calcsize(FIX_DATA_FORMAT)
50
+
51
+ @property
52
+ def name(self):
53
+ """Get name."""
54
+ return self._origin_data.get(OperatorDetailsIndexEnum.NAME.value, "")
55
+
56
+ @property
57
+ def input_shapes(self):
58
+ """Get input_shapes."""
59
+ return self._origin_data.get(OperatorDetailsIndexEnum.INPUT_SHAPES.value, "")
60
+
61
+ @property
62
+ def input_type(self):
63
+ """Get input_type."""
64
+ return self._origin_data.get(OperatorDetailsIndexEnum.INPUT_TYPE.value, "")
65
+
66
+
67
+ class MsOperatorDetailsViewer(BaseViewer):
68
+ """Viewer for MindSpore operator_details profiling data."""
69
+
70
+ FWK_BINARY_FILE_NAME = "mindspore.record_shapes"
71
+ _OPERATOR_DETAILS_FILE_NAME = 'operator_details.csv'
72
+ _COL_NAMES = ['Name', 'Input Shapes']
73
+
74
+ def __init__(self, **kwargs):
75
+ super().__init__()
76
+ self._save_path = os.path.join(
77
+ kwargs.get("ascend_profiler_output_path"),
78
+ self._OPERATOR_DETAILS_FILE_NAME
79
+ )
80
+ self._ascend_ms_dir = kwargs.get("ascend_ms_dir")
81
+ self._framework_path = kwargs.get("framework_path")
82
+ ProfilerLogger.init(self._ascend_ms_dir)
83
+ self._logger = ProfilerLogger.get_instance()
84
+
85
+ def save(self, data: Dict[str, Any]) -> None:
86
+ """Process and save operator_details profiling data."""
87
+ self._logger.info("MsOperatorDetailsViewer start")
88
+ try:
89
+ file_exist = self._read_fwk_binary_file()
90
+ if not file_exist:
91
+ return
92
+ self._calculate_operator_details_data()
93
+ self._write_data()
94
+ except Exception as e: # pylint: disable=W0703
95
+ self._logger.error("Failed to save operator_details.csv: %s", e, exc_info=True)
96
+ self._logger.info("MsOperatorDetailsViewer end")
97
+
98
+ def _read_fwk_binary_file(self):
99
+ """
100
+ Read fwk binary file
101
+ """
102
+ self._logger.info("Read fwk binary file start")
103
+ fwk_file_path = os.path.join(self._framework_path, self.FWK_BINARY_FILE_NAME)
104
+ if not os.path.isfile(fwk_file_path):
105
+ self._logger.warning("Fwk binary file %s does not exist.", fwk_file_path)
106
+ return False
107
+ raw_bin_data = FileManager.read_file_content(fwk_file_path, mode="rb")
108
+ operator_details_decode_data = TLVDecoder.decode(
109
+ raw_bin_data, OperatorDetailsEvent.FIX_DATA_FORMAT, OperatorDetailsEvent.FIX_DATA_SIZE
110
+ )
111
+ self._operator_details_events = [OperatorDetailsEvent(data) for data in operator_details_decode_data]
112
+ self._logger.info("Read fwk binary file done, %d events", len(self._operator_details_events))
113
+ return True
114
+
115
+ def _calculate_operator_details_data(self):
116
+ """
117
+ Calculate operator details data
118
+ """
119
+ self._operator_details_data = [
120
+ [event.name, event.input_shapes]
121
+ for event in self._operator_details_events
122
+ ]
123
+
124
+ def _write_data(self) -> None:
125
+ """
126
+ Save operator statistics to a CSV file
127
+ """
128
+ if not self._operator_details_data:
129
+ return
130
+ self._logger.info("Save operator statistics start")
131
+ FileManager.create_csv_file(self._save_path, self._operator_details_data, self._COL_NAMES)
132
+ self._logger.info("Save operator statistics done")
@@ -186,6 +186,8 @@ class TimelineLayerName(Enum):
186
186
  ACC_PMU = "Acc PMU"
187
187
  SIO = "SIO"
188
188
  QOS = "QoS"
189
+ NIC = "NIC"
190
+ ROCE = "RoCE"
189
191
  OVERLAP_ANALYSIS = "Overlap Analysis"
190
192
 
191
193
 
@@ -210,3 +212,17 @@ class ExportType(Enum):
210
212
  class CannLibName:
211
213
  """CANN lib name"""
212
214
  CANN_MSPTI = "libmspti.so"
215
+
216
+
217
+ class DynoMode:
218
+ """dyno mode"""
219
+ DYNO_DAEMON = "MSMONITOR_USE_DAEMON"
220
+
221
+
222
+ class HostSystem(Enum):
223
+ """host system"""
224
+ CPU = "cpu"
225
+ MEM = "mem"
226
+ DISK = "disk"
227
+ NETWORK = "network"
228
+ OSRT = "osrt"
@@ -145,14 +145,14 @@ class MsprofCmdTool:
145
145
  if os.environ.get("ASCEND_TOOLKIT_HOME"):
146
146
  temp_path = os.path.join(os.environ.get("ASCEND_TOOLKIT_HOME"), "bin")
147
147
  if os.path.isdir(temp_path) and self._MSPROF_CMD in os.listdir(temp_path):
148
- return temp_path
148
+ return os.path.abspath(temp_path)
149
149
 
150
150
  for path in os.environ.get("PATH", "").split(":"):
151
151
  if self._ASCEND_MARK in path:
152
152
  prefix = path.split(self._ASCEND_MARK)[0]
153
153
  temp_path = os.path.join(prefix, self._HIAI_MSPROF_TAIL)
154
154
  if os.path.isdir(temp_path) and self._MSPROF_CMD in os.listdir(temp_path):
155
- return temp_path
155
+ return os.path.abspath(temp_path)
156
156
 
157
157
  return None
158
158
 
@@ -73,6 +73,15 @@ class PathManager:
73
73
  msg = f"file size exceeds the limit: {cls.MAX_FILE_SIZE}, file size: {file_size}"
74
74
  raise ProfilerPathErrorException(msg)
75
75
 
76
+ file_stat = os.stat(path)
77
+ if file_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH):
78
+ msg = f"File path {path} has group or others writable permissions, which is not allowed."
79
+ raise ProfilerPathErrorException(msg)
80
+
81
+ if stat.S_ISCHR(file_stat.st_mode) or stat.S_ISBLK(file_stat.st_mode):
82
+ msg = f"Invalid input path is a character or block device path: {path}"
83
+ raise ProfilerPathErrorException(msg)
84
+
76
85
  @classmethod
77
86
  def get_directory_size(cls, directory: str, unit: str = 'MB') -> float:
78
87
  """