mindstudio-probe 8.3.3__py3-none-any.whl → 26.0.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (689) hide show
  1. {mindstudio_probe-8.3.3.dist-info → mindstudio_probe-26.0.0a1.dist-info}/METADATA +26 -14
  2. mindstudio_probe-26.0.0a1.dist-info/RECORD +498 -0
  3. {mindstudio_probe-8.3.3.dist-info → mindstudio_probe-26.0.0a1.dist-info}/WHEEL +1 -1
  4. mindstudio_probe-26.0.0a1.dist-info/entry_points.txt +5 -0
  5. mindstudio_probe-26.0.0a1.dist-info/licenses/LICENSE +124 -0
  6. mindstudio_probe-26.0.0a1.dist-info/top_level.txt +2 -0
  7. msprobe/__init__.py +12 -13
  8. msprobe/config.json +9 -31
  9. msprobe/core/__init__.py +12 -11
  10. msprobe/core/acc_check/acc_check_cli.py +145 -0
  11. msprobe/core/common/const.py +97 -38
  12. msprobe/core/common/db_manager.py +133 -12
  13. msprobe/core/common/decorator.py +12 -11
  14. msprobe/core/common/exceptions.py +12 -11
  15. msprobe/core/common/file_utils.py +101 -25
  16. msprobe/core/common/framework_adapter.py +36 -25
  17. msprobe/core/common/global_lock.py +12 -11
  18. msprobe/core/common/inplace_op_checker.py +12 -11
  19. msprobe/core/common/log.py +22 -11
  20. msprobe/core/common/megatron_utils.py +566 -11
  21. msprobe/core/common/parallel_state.py +12 -11
  22. msprobe/core/common/runtime.py +12 -11
  23. msprobe/core/common/utils.py +41 -41
  24. msprobe/core/compare/acc_compare.py +361 -104
  25. msprobe/core/compare/atb_data_compare.py +422 -0
  26. msprobe/core/compare/auto_compare.py +134 -0
  27. msprobe/core/compare/check.py +14 -17
  28. msprobe/core/compare/compare_cli.py +72 -149
  29. msprobe/core/compare/config.py +12 -13
  30. msprobe/core/compare/diff_analyze/first_diff_analyze.py +28 -15
  31. msprobe/core/compare/diff_analyze/ignore_op_list.yaml +3 -0
  32. msprobe/core/compare/find_first/analyzer.py +18 -18
  33. msprobe/core/compare/find_first/graph.py +12 -11
  34. msprobe/core/compare/find_first/utils.py +13 -12
  35. msprobe/core/compare/indicator_analysis/__init__.py +15 -0
  36. msprobe/core/compare/indicator_analysis/algorithm.py +363 -0
  37. msprobe/core/compare/indicator_analysis/api_data.py +141 -0
  38. msprobe/core/compare/indicator_analysis/calculator.py +181 -0
  39. msprobe/core/compare/indicator_analysis/utils.py +116 -0
  40. msprobe/core/compare/layer_mapping/__init__.py +12 -11
  41. msprobe/core/compare/layer_mapping/data_scope_parser.py +20 -11
  42. msprobe/core/compare/layer_mapping/layer_mapping.py +14 -13
  43. msprobe/core/compare/layer_mapping/postprocess_pass.py +13 -11
  44. msprobe/core/compare/merge_result/merge_result.py +12 -11
  45. msprobe/core/compare/merge_result/merge_result_cli.py +12 -11
  46. msprobe/core/compare/merge_result/utils.py +12 -11
  47. msprobe/core/compare/multiprocessing_compute.py +13 -14
  48. msprobe/core/compare/npy_compare.py +13 -11
  49. msprobe/core/compare/offline_data_compare.py +160 -0
  50. msprobe/core/compare/stats_diff_calc.py +39 -0
  51. msprobe/core/compare/torchair_acc_cmp.py +764 -0
  52. msprobe/core/compare/torchair_cmp_utils.py +338 -0
  53. msprobe/core/compare/utils.py +140 -49
  54. msprobe/core/config_check/__init__.py +12 -11
  55. msprobe/core/config_check/checkers/__init__.py +12 -11
  56. msprobe/core/config_check/checkers/base_checker.py +15 -14
  57. msprobe/core/config_check/checkers/dataset_checker.py +13 -12
  58. msprobe/core/config_check/checkers/env_args_checker.py +13 -12
  59. msprobe/core/config_check/checkers/hyperparameter_checker.py +16 -15
  60. msprobe/core/config_check/checkers/pip_checker.py +15 -15
  61. msprobe/core/config_check/checkers/random_checker.py +13 -12
  62. msprobe/core/config_check/checkers/weights_checker.py +14 -12
  63. msprobe/core/config_check/ckpt_compare/ckpt_comparator.py +13 -17
  64. msprobe/core/config_check/ckpt_compare/megatron_loader.py +13 -12
  65. msprobe/core/config_check/ckpt_compare/metrics.py +12 -11
  66. msprobe/core/config_check/config_check_cli.py +18 -17
  67. msprobe/core/config_check/config_checker.py +16 -14
  68. msprobe/core/config_check/resource/dependency.yaml +15 -12
  69. msprobe/core/config_check/resource/env.yaml +12 -11
  70. msprobe/core/config_check/utils/hyperparameter_parser.py +12 -11
  71. msprobe/core/config_check/utils/utils.py +12 -11
  72. msprobe/core/{data_dump → dump/api_dump}/api_registry.py +12 -11
  73. msprobe/core/{common_config.py → dump/common_config.py} +13 -24
  74. msprobe/core/dump/data_dump/data_collector.py +257 -0
  75. msprobe/core/{data_dump → dump/data_dump}/data_processor/base.py +45 -36
  76. msprobe/core/{data_dump → dump/data_dump}/data_processor/factory.py +33 -25
  77. msprobe/core/{data_dump → dump/data_dump}/data_processor/mindspore_processor.py +37 -113
  78. msprobe/core/{data_dump → dump/data_dump}/data_processor/pytorch_processor.py +364 -131
  79. msprobe/core/{data_dump → dump/data_dump}/json_writer.py +24 -31
  80. msprobe/core/{data_dump → dump/data_dump}/scope.py +12 -13
  81. msprobe/core/{debugger → dump/debugger}/precision_debugger.py +15 -23
  82. msprobe/core/dump/dump2db/db_utils.py +215 -0
  83. msprobe/core/dump/dump2db/dump2db.py +409 -0
  84. msprobe/core/{hook_manager.py → dump/hook_manager.py} +38 -87
  85. msprobe/core/dump/kernel_dump/kernel_config.py +34 -0
  86. msprobe/core/{service.py → dump/service.py} +43 -27
  87. msprobe/core/install_deps/install_deps.py +51 -0
  88. msprobe/core/monitor/anomaly_processor.py +13 -11
  89. msprobe/core/monitor/csv2db.py +73 -93
  90. msprobe/core/monitor/db_utils.py +140 -205
  91. msprobe/core/monitor/utils.py +18 -17
  92. msprobe/core/monitor_v2/__init__.py +20 -0
  93. msprobe/core/monitor_v2/base.py +83 -0
  94. msprobe/core/monitor_v2/cc.py +287 -0
  95. msprobe/core/monitor_v2/factory.py +81 -0
  96. msprobe/core/monitor_v2/module.py +201 -0
  97. msprobe/core/monitor_v2/optimizer.py +245 -0
  98. msprobe/core/monitor_v2/param.py +154 -0
  99. msprobe/core/monitor_v2/trainer.py +326 -0
  100. msprobe/core/monitor_v2/utils.py +122 -0
  101. msprobe/core/monitor_v2/weight_grad.py +419 -0
  102. msprobe/core/monitor_v2/writer.py +162 -0
  103. msprobe/core/overflow_check/abnormal_scene.py +12 -11
  104. msprobe/core/overflow_check/api_info.py +12 -11
  105. msprobe/core/overflow_check/checker.py +12 -11
  106. msprobe/core/overflow_check/filter.py +13 -11
  107. msprobe/core/overflow_check/level.py +12 -11
  108. msprobe/core/overflow_check/utils.py +12 -11
  109. msprobe/core/single_save/single_comparator.py +12 -11
  110. msprobe/core/single_save/single_saver.py +12 -11
  111. msprobe/infer/__init__.py +16 -0
  112. msprobe/infer/offline/__init__.py +16 -0
  113. msprobe/infer/offline/compare/__init__.py +16 -0
  114. msprobe/infer/offline/compare/msquickcmp/__init__.py +16 -0
  115. msprobe/infer/offline/compare/msquickcmp/adapter_cli/__init__.py +16 -0
  116. msprobe/infer/offline/compare/msquickcmp/adapter_cli/args_adapter.py +46 -0
  117. msprobe/infer/offline/compare/msquickcmp/atc/__init__.py +16 -0
  118. msprobe/infer/offline/compare/msquickcmp/atc/atc_utils.py +98 -0
  119. msprobe/infer/offline/compare/msquickcmp/cmp_process.py +328 -0
  120. msprobe/infer/offline/compare/msquickcmp/common/__init__.py +16 -0
  121. msprobe/infer/offline/compare/msquickcmp/common/args_check.py +112 -0
  122. msprobe/infer/offline/compare/msquickcmp/common/convert.py +74 -0
  123. msprobe/infer/offline/compare/msquickcmp/common/dump_data.py +121 -0
  124. msprobe/infer/offline/compare/msquickcmp/common/dynamic_argument_bean.py +39 -0
  125. msprobe/infer/offline/compare/msquickcmp/common/utils.py +669 -0
  126. msprobe/infer/offline/compare/msquickcmp/config.ini +6 -0
  127. msprobe/infer/offline/compare/msquickcmp/dump/__init__.py +16 -0
  128. msprobe/infer/offline/compare/msquickcmp/dump/args_adapter.py +50 -0
  129. msprobe/infer/offline/compare/msquickcmp/dump/dump_process.py +91 -0
  130. msprobe/infer/offline/compare/msquickcmp/install_aclruntime_aisbench.sh +180 -0
  131. msprobe/infer/offline/compare/msquickcmp/main.py +199 -0
  132. msprobe/infer/offline/compare/msquickcmp/net_compare/__init__.py +16 -0
  133. msprobe/infer/offline/compare/msquickcmp/net_compare/net_compare.py +277 -0
  134. msprobe/infer/offline/compare/msquickcmp/npu/__init__.py +16 -0
  135. msprobe/infer/offline/compare/msquickcmp/npu/npu_dump_data.py +558 -0
  136. msprobe/infer/offline/compare/msquickcmp/npu/om_parser.py +416 -0
  137. msprobe/infer/offline/compare/msquickcmp/onnx_model/__init__.py +16 -0
  138. msprobe/infer/offline/compare/msquickcmp/onnx_model/onnx_dump_data.py +374 -0
  139. msprobe/infer/utils/__init__.py +15 -0
  140. msprobe/infer/utils/acc_cmp.py +94 -0
  141. msprobe/infer/utils/check/__init__.py +37 -0
  142. msprobe/infer/utils/check/args_checker.py +35 -0
  143. msprobe/infer/utils/check/checker.py +227 -0
  144. msprobe/infer/utils/check/dict_checker.py +78 -0
  145. msprobe/infer/utils/check/func_wrapper.py +96 -0
  146. msprobe/infer/utils/check/list_checker.py +56 -0
  147. msprobe/infer/utils/check/number_checker.py +64 -0
  148. msprobe/infer/utils/check/obj_checker.py +41 -0
  149. msprobe/infer/utils/check/path_checker.py +249 -0
  150. msprobe/infer/utils/check/rule.py +126 -0
  151. msprobe/infer/utils/check/string_checker.py +66 -0
  152. msprobe/infer/utils/cmp_algorithm.py +261 -0
  153. msprobe/infer/utils/constants.py +112 -0
  154. msprobe/infer/utils/file_open_check.py +337 -0
  155. msprobe/infer/utils/util.py +177 -0
  156. msprobe/mindspore/__init__.py +14 -13
  157. msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py +14 -13
  158. msprobe/mindspore/api_accuracy_checker/api_info.py +12 -11
  159. msprobe/mindspore/api_accuracy_checker/api_runner.py +12 -11
  160. msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py +12 -11
  161. msprobe/mindspore/api_accuracy_checker/bench_functions/flash_attention_score.py +12 -11
  162. msprobe/mindspore/api_accuracy_checker/bench_functions/fusion_operator.py +12 -11
  163. msprobe/mindspore/api_accuracy_checker/checker_support_api.yaml +12 -11
  164. msprobe/mindspore/api_accuracy_checker/cmd_parser.py +15 -14
  165. msprobe/mindspore/api_accuracy_checker/compute_element.py +12 -11
  166. msprobe/mindspore/api_accuracy_checker/data_manager.py +13 -11
  167. msprobe/mindspore/api_accuracy_checker/main.py +12 -11
  168. msprobe/mindspore/api_accuracy_checker/multi_api_accuracy_checker.py +14 -12
  169. msprobe/mindspore/api_accuracy_checker/multi_data_manager.py +13 -11
  170. msprobe/mindspore/api_accuracy_checker/torch_mindtorch_importer.py +12 -11
  171. msprobe/mindspore/api_accuracy_checker/type_mapping.py +12 -11
  172. msprobe/mindspore/api_accuracy_checker/utils.py +12 -11
  173. msprobe/mindspore/common/const.py +15 -74
  174. msprobe/mindspore/common/log.py +12 -11
  175. msprobe/mindspore/common/utils.py +30 -15
  176. msprobe/mindspore/compare/common_dir_compare.py +21 -23
  177. msprobe/mindspore/compare/distributed_compare.py +18 -16
  178. msprobe/mindspore/compare/ms_compare.py +14 -14
  179. msprobe/mindspore/compare/ms_graph_compare.py +26 -20
  180. msprobe/mindspore/compare/utils.py +14 -12
  181. msprobe/mindspore/{cell_processor.py → dump/cell_processor.py} +15 -14
  182. msprobe/mindspore/{debugger → dump/debugger}/debugger_config.py +12 -30
  183. msprobe/mindspore/{debugger → dump/debugger}/precision_debugger.py +43 -45
  184. msprobe/mindspore/dump/{cell_dump_process.py → dump_processor/cell_dump_process.py} +31 -17
  185. msprobe/mindspore/dump/{cell_dump_with_insert_gradient.py → dump_processor/cell_dump_with_insert_gradient.py} +18 -14
  186. msprobe/mindspore/dump/{dump_tool_factory.py → dump_processor/dump_tool_factory.py} +16 -15
  187. msprobe/mindspore/dump/{graph_mode_cell_dump.py → dump_processor/graph_mode_cell_dump.py} +16 -15
  188. msprobe/mindspore/dump/{graph_tensor_dump.py → dump_processor/graph_tensor_dump.py} +134 -133
  189. msprobe/mindspore/dump/{hook_cell → dump_processor/hook_cell}/api_register.py +15 -14
  190. msprobe/mindspore/dump/{hook_cell → dump_processor/hook_cell}/hook_cell.py +12 -11
  191. msprobe/mindspore/dump/{hook_cell → dump_processor/hook_cell}/ms_hook_manager.py +47 -20
  192. msprobe/mindspore/dump/{hook_cell → dump_processor/hook_cell}/primitive_hooks.py +14 -13
  193. msprobe/mindspore/dump/{hook_cell → dump_processor/hook_cell}/support_wrap_ops.yaml +13 -11
  194. msprobe/mindspore/dump/{jit_dump.py → dump_processor/jit_dump.py} +14 -13
  195. msprobe/mindspore/dump/{kernel_graph_dump.py → dump_processor/kernel_graph_dump.py} +13 -12
  196. msprobe/mindspore/dump/{kernel_kbyk_dump.py → dump_processor/kernel_kbyk_dump.py} +13 -12
  197. msprobe/mindspore/{exception_dump → dump/exception_dump}/exception_dump_tool_factory.py +14 -13
  198. msprobe/mindspore/{exception_dump → dump/exception_dump}/kernel_graph_exception_dump.py +13 -12
  199. msprobe/mindspore/{mindspore_service.py → dump/mindspore_service.py} +18 -17
  200. msprobe/mindspore/dump/mindtorch/__init__.py +19 -0
  201. msprobe/mindspore/dump/ms_config.py +105 -0
  202. msprobe/mindspore/{overflow_check → dump/overflow_check}/kernel_graph_overflow_check.py +13 -12
  203. msprobe/mindspore/{overflow_check → dump/overflow_check}/overflow_check_tool_factory.py +14 -13
  204. msprobe/mindspore/dump/task_handler_factory.py +43 -0
  205. msprobe/mindspore/monitor/common_func.py +12 -11
  206. msprobe/mindspore/monitor/data_writers.py +12 -11
  207. msprobe/mindspore/monitor/distributed/wrap_distributed.py +93 -39
  208. msprobe/mindspore/monitor/features.py +12 -11
  209. msprobe/mindspore/monitor/module_hook.py +19 -22
  210. msprobe/mindspore/monitor/optimizer_collect.py +29 -25
  211. msprobe/mindspore/monitor/utils.py +13 -11
  212. msprobe/msaccucmp/advisor/__init__.py +16 -0
  213. msprobe/msaccucmp/advisor/advisor_const.py +65 -0
  214. msprobe/msaccucmp/advisor/advisor_result.py +73 -0
  215. msprobe/msaccucmp/advisor/compare_advisor.py +99 -0
  216. msprobe/msaccucmp/advisor/input_advisor.py +66 -0
  217. msprobe/msaccucmp/advisor/node_advisor.py +68 -0
  218. msprobe/msaccucmp/advisor/overflow_advisor.py +58 -0
  219. msprobe/msaccucmp/algorithm_manager/__init__.py +16 -0
  220. msprobe/msaccucmp/algorithm_manager/algorithm_manager.py +464 -0
  221. msprobe/msaccucmp/algorithm_manager/algorithm_parameter.py +42 -0
  222. msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_AccumulatedRelativeError.py +46 -0
  223. msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_CosineSimilarity.py +58 -0
  224. msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_KullbackLeiblerDivergence.py +84 -0
  225. msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_MaxAbsoluteError.py +41 -0
  226. msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_MaxRelativeError.py +46 -0
  227. msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_MeanAbsoluteError.py +41 -0
  228. msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_MeanRelativeError.py +46 -0
  229. msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_RelativeEuclideanDistance.py +46 -0
  230. msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_RootMeanSquareError.py +40 -0
  231. msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_StandardDeviation.py +47 -0
  232. msprobe/msaccucmp/cmp_utils/__init__.py +16 -0
  233. msprobe/msaccucmp/cmp_utils/common.py +113 -0
  234. msprobe/msaccucmp/cmp_utils/constant/__init__.py +16 -0
  235. msprobe/msaccucmp/cmp_utils/constant/compare_error.py +81 -0
  236. msprobe/msaccucmp/cmp_utils/constant/const_manager.py +530 -0
  237. msprobe/msaccucmp/cmp_utils/file_utils.py +497 -0
  238. msprobe/msaccucmp/cmp_utils/log.py +257 -0
  239. msprobe/msaccucmp/cmp_utils/multi_process/__init__.py +16 -0
  240. msprobe/msaccucmp/cmp_utils/multi_process/multi_convert_process.py +140 -0
  241. msprobe/msaccucmp/cmp_utils/multi_process/progress.py +78 -0
  242. msprobe/msaccucmp/cmp_utils/path_check.py +274 -0
  243. msprobe/msaccucmp/cmp_utils/reg_manager.py +98 -0
  244. msprobe/msaccucmp/cmp_utils/tlv_parse.py +279 -0
  245. msprobe/msaccucmp/cmp_utils/utils.py +356 -0
  246. msprobe/msaccucmp/cmp_utils/utils_type.py +63 -0
  247. msprobe/msaccucmp/compare_vector.py +48 -0
  248. msprobe/msaccucmp/conversion/__init__.py +16 -0
  249. msprobe/msaccucmp/conversion/data_conversion.py +277 -0
  250. msprobe/msaccucmp/conversion/dtype_conversion.py +99 -0
  251. msprobe/msaccucmp/conversion/shape_format_conversion.py +477 -0
  252. msprobe/msaccucmp/conversion/tensor_conversion.py +369 -0
  253. msprobe/msaccucmp/dump_data_conversion.py +46 -0
  254. msprobe/msaccucmp/dump_parse/__init__.py +16 -0
  255. msprobe/msaccucmp/dump_parse/big_dump_data.py +317 -0
  256. msprobe/msaccucmp/dump_parse/dump.py +423 -0
  257. msprobe/msaccucmp/dump_parse/dump_data_object.py +322 -0
  258. msprobe/msaccucmp/dump_parse/dump_data_parser.py +436 -0
  259. msprobe/msaccucmp/dump_parse/dump_utils.py +246 -0
  260. msprobe/msaccucmp/dump_parse/ffts_parser.py +137 -0
  261. msprobe/msaccucmp/dump_parse/mapping.py +62 -0
  262. msprobe/msaccucmp/dump_parse/nano_dump_data.py +392 -0
  263. msprobe/msaccucmp/dump_parse/proto_dump_data.py +308 -0
  264. msprobe/msaccucmp/dump_parser.py +90 -0
  265. msprobe/msaccucmp/format_manager/__init__.py +16 -0
  266. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_FRACTAL_NZ_to_NCHW.py +53 -0
  267. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_FRACTAL_NZ_to_ND.py +52 -0
  268. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_FRACTAL_NZ_to_NHWC.py +53 -0
  269. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_FRACTAL_Z_to_HWCN.py +47 -0
  270. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_FRACTAL_Z_to_NCHW.py +47 -0
  271. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_HWCN_to_FRACTAL_Z.py +89 -0
  272. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_HWCN_to_NCHW.py +37 -0
  273. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_HWCN_to_NHWC.py +37 -0
  274. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NC1HWC0_to_HWCN.py +43 -0
  275. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NC1HWC0_to_NCHW.py +48 -0
  276. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NC1HWC0_to_NHWC.py +43 -0
  277. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NCHW_to_FRACTAL_Z.py +87 -0
  278. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NCHW_to_NHWC.py +37 -0
  279. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NDC1HWC0_to_NCDHW.py +48 -0
  280. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NDC1HWC0_to_ND.py +44 -0
  281. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NHWC_to_FRACTAL_Z.py +87 -0
  282. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NHWC_to_HWCN.py +37 -0
  283. msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NHWC_to_NCHW.py +37 -0
  284. msprobe/msaccucmp/format_manager/format_manager.py +307 -0
  285. msprobe/msaccucmp/inplace_layer_process.py +186 -0
  286. msprobe/msaccucmp/msaccucmp.py +532 -0
  287. msprobe/msaccucmp/mscmp_advisor.py +128 -0
  288. msprobe/msaccucmp/overflow/__init__.py +16 -0
  289. msprobe/msaccucmp/overflow/overflow_analyse.py +305 -0
  290. msprobe/msaccucmp/overflow/overflow_detection.py +143 -0
  291. msprobe/msaccucmp/pytorch_cmp/__init__.py +16 -0
  292. msprobe/msaccucmp/pytorch_cmp/compare_pytorch.py +389 -0
  293. msprobe/msaccucmp/pytorch_cmp/hdf5_parser.py +377 -0
  294. msprobe/msaccucmp/pytorch_cmp/pytorch_dump_data.py +461 -0
  295. msprobe/msaccucmp/shape_conversion.py +41 -0
  296. msprobe/msaccucmp/vector_cmp/__init__.py +16 -0
  297. msprobe/msaccucmp/vector_cmp/batch_compare.py +197 -0
  298. msprobe/msaccucmp/vector_cmp/compare_detail/__init__.py +16 -0
  299. msprobe/msaccucmp/vector_cmp/compare_detail/compare_detail.py +245 -0
  300. msprobe/msaccucmp/vector_cmp/compare_detail/detail.py +182 -0
  301. msprobe/msaccucmp/vector_cmp/compare_detail/detail_writer.py +580 -0
  302. msprobe/msaccucmp/vector_cmp/fusion_manager/__init__.py +16 -0
  303. msprobe/msaccucmp/vector_cmp/fusion_manager/compare_fusion_op.py +588 -0
  304. msprobe/msaccucmp/vector_cmp/fusion_manager/compare_npu_vs_npu.py +339 -0
  305. msprobe/msaccucmp/vector_cmp/fusion_manager/compare_result.py +326 -0
  306. msprobe/msaccucmp/vector_cmp/fusion_manager/compare_rule.py +156 -0
  307. msprobe/msaccucmp/vector_cmp/fusion_manager/fusion_op.py +204 -0
  308. msprobe/msaccucmp/vector_cmp/fusion_manager/fusion_rule_parser.py +635 -0
  309. msprobe/msaccucmp/vector_cmp/fusion_manager/quant_filter.py +187 -0
  310. msprobe/msaccucmp/vector_cmp/range_manager/__init__.py +16 -0
  311. msprobe/msaccucmp/vector_cmp/range_manager/range_manager.py +100 -0
  312. msprobe/msaccucmp/vector_cmp/range_manager/range_mode.py +94 -0
  313. msprobe/msaccucmp/vector_cmp/range_manager/select_mode.py +86 -0
  314. msprobe/msaccucmp/vector_cmp/vector_comparison.py +535 -0
  315. msprobe/msprobe.py +101 -130
  316. msprobe/overflow_check/__init__.py +15 -0
  317. msprobe/{nan_analyze → overflow_check}/analyzer.py +38 -27
  318. msprobe/{nan_analyze → overflow_check}/graph.py +28 -27
  319. msprobe/{nan_analyze → overflow_check}/utils.py +15 -14
  320. msprobe/pytorch/__init__.py +20 -14
  321. msprobe/pytorch/aclgraph_dump/__init__.py +45 -0
  322. msprobe/pytorch/aclgraph_dump/_meta.py +26 -0
  323. msprobe/pytorch/api_accuracy_checker/{run_ut/run_ut.py → acc_check/acc_check.py} +50 -45
  324. msprobe/pytorch/api_accuracy_checker/{run_ut/run_ut_utils.py → acc_check/acc_check_utils.py} +201 -30
  325. msprobe/pytorch/api_accuracy_checker/{run_ut → acc_check}/data_generate.py +56 -16
  326. msprobe/pytorch/api_accuracy_checker/{run_ut/multi_run_ut.py → acc_check/multi_acc_check.py} +32 -47
  327. msprobe/pytorch/api_accuracy_checker/{run_ut → acc_check}/run_overflow_check.py +19 -18
  328. msprobe/pytorch/api_accuracy_checker/common/config.py +22 -20
  329. msprobe/pytorch/api_accuracy_checker/common/utils.py +72 -13
  330. msprobe/pytorch/api_accuracy_checker/compare/algorithm.py +41 -11
  331. msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +23 -14
  332. msprobe/pytorch/api_accuracy_checker/compare/compare.py +45 -32
  333. msprobe/pytorch/api_accuracy_checker/compare/compare_column.py +12 -11
  334. msprobe/pytorch/api_accuracy_checker/compare/compare_input.py +14 -12
  335. msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py +14 -12
  336. msprobe/pytorch/api_accuracy_checker/precision_standard/absolute_threshold.py +12 -11
  337. msprobe/pytorch/api_accuracy_checker/precision_standard/accumulative_error_compare.py +12 -11
  338. msprobe/pytorch/api_accuracy_checker/precision_standard/base_standard.py +21 -19
  339. msprobe/pytorch/api_accuracy_checker/precision_standard/benchmark_compare.py +14 -13
  340. msprobe/pytorch/api_accuracy_checker/precision_standard/binary_consistency.py +12 -11
  341. msprobe/pytorch/api_accuracy_checker/precision_standard/standard_config.py +60 -11
  342. msprobe/pytorch/api_accuracy_checker/precision_standard/standard_register.py +27 -16
  343. msprobe/pytorch/api_accuracy_checker/precision_standard/thousandth_standard.py +13 -11
  344. msprobe/pytorch/api_accuracy_checker/precision_standard/ulp_compare.py +39 -18
  345. msprobe/pytorch/bench_functions/__init__.py +12 -11
  346. msprobe/pytorch/bench_functions/apply_adam.py +12 -11
  347. msprobe/pytorch/bench_functions/apply_adam_w.py +12 -11
  348. msprobe/pytorch/bench_functions/confusion_transpose.py +12 -11
  349. msprobe/pytorch/bench_functions/fast_gelu.py +12 -11
  350. msprobe/pytorch/bench_functions/group_norm_silu.py +12 -11
  351. msprobe/pytorch/bench_functions/layer_norm_eval.py +12 -11
  352. msprobe/pytorch/bench_functions/linear.py +12 -11
  353. msprobe/pytorch/bench_functions/matmul_backward.py +12 -11
  354. msprobe/pytorch/bench_functions/mish.py +12 -11
  355. msprobe/pytorch/bench_functions/moe_gating_top_k_softmax.py +12 -11
  356. msprobe/pytorch/bench_functions/npu_fusion_attention.py +12 -11
  357. msprobe/pytorch/bench_functions/rms_norm.py +12 -11
  358. msprobe/pytorch/bench_functions/rotary_mul.py +12 -11
  359. msprobe/pytorch/bench_functions/scaled_mask_softmax.py +12 -11
  360. msprobe/pytorch/bench_functions/sort_v2.py +12 -11
  361. msprobe/pytorch/bench_functions/swiglu.py +12 -11
  362. msprobe/pytorch/common/__init__.py +12 -11
  363. msprobe/pytorch/common/log.py +12 -11
  364. msprobe/pytorch/common/parse_json.py +12 -11
  365. msprobe/pytorch/common/utils.py +52 -19
  366. msprobe/pytorch/compare/distributed_compare.py +13 -13
  367. msprobe/pytorch/compare/match.py +12 -11
  368. msprobe/pytorch/compare/pt_compare.py +14 -20
  369. msprobe/pytorch/compare/pt_diff_analyze.py +12 -11
  370. msprobe/pytorch/compare/utils.py +12 -11
  371. msprobe/pytorch/{hook_module → dump/api_dump}/api_register.py +18 -16
  372. msprobe/pytorch/{hook_module → dump/api_dump}/hook_module.py +14 -13
  373. msprobe/pytorch/{hook_module → dump/api_dump}/pt_hook_manager.py +68 -23
  374. msprobe/pytorch/{hook_module → dump/api_dump}/register_optimizer_hook.py +13 -11
  375. msprobe/pytorch/{hook_module → dump/api_dump}/script_wrapper.py +17 -14
  376. msprobe/pytorch/{hook_module → dump/api_dump}/utils.py +12 -11
  377. msprobe/pytorch/{debugger → dump/debugger}/debugger_config.py +23 -38
  378. msprobe/pytorch/dump/debugger/precision_debugger.py +130 -0
  379. msprobe/pytorch/{function_factory.py → dump/function_factory.py} +12 -11
  380. msprobe/pytorch/dump/module_dump/hook_wrapper.py +17 -13
  381. msprobe/pytorch/dump/module_dump/module_dump.py +16 -15
  382. msprobe/pytorch/dump/module_dump/{module_processer.py → module_processor.py} +54 -42
  383. msprobe/pytorch/dump/pt_config.py +128 -0
  384. msprobe/pytorch/{pytorch_service.py → dump/pytorch_service.py} +22 -21
  385. msprobe/pytorch/monitor/csv2tb.py +13 -11
  386. msprobe/pytorch/monitor/data_writers.py +13 -11
  387. msprobe/pytorch/monitor/distributed/wrap_distributed.py +13 -11
  388. msprobe/pytorch/monitor/features.py +12 -11
  389. msprobe/pytorch/monitor/module_hook.py +67 -59
  390. msprobe/pytorch/monitor/module_metric.py +13 -11
  391. msprobe/pytorch/monitor/optimizer_collect.py +37 -35
  392. msprobe/pytorch/monitor/utils.py +13 -11
  393. msprobe/pytorch/monitor/visualizer.py +12 -11
  394. msprobe/pytorch/torchair_dump/__init__.py +17 -0
  395. msprobe/pytorch/torchair_dump/torchair_dump.py +114 -0
  396. msprobe/scripts/atb/config_example.json +10 -0
  397. msprobe/scripts/atb/load_atb_probe.sh +101 -0
  398. msprobe/scripts/atb/unload_atb_probe.sh +27 -0
  399. msprobe/scripts/build_msaccucmp.sh +186 -0
  400. msprobe/scripts/conf/help.info +6 -0
  401. msprobe/scripts/conf/version.info +3 -0
  402. msprobe/scripts/run_script/common.sh +538 -0
  403. msprobe/scripts/run_script/main_msaccucmp.sh +232 -0
  404. msprobe/visualization/__init__.py +12 -11
  405. msprobe/visualization/builder/__init__.py +12 -11
  406. msprobe/visualization/builder/graph_builder.py +45 -30
  407. msprobe/visualization/builder/graph_merger.py +53 -32
  408. msprobe/visualization/builder/msprobe_adapter.py +34 -44
  409. msprobe/visualization/compare/__init__.py +12 -11
  410. msprobe/visualization/compare/graph_comparator.py +63 -51
  411. msprobe/visualization/compare/mode_adapter.py +28 -113
  412. msprobe/visualization/db_utils.py +133 -22
  413. msprobe/visualization/graph/__init__.py +12 -11
  414. msprobe/visualization/graph/base_node.py +15 -27
  415. msprobe/visualization/graph/distributed_analyzer.py +97 -40
  416. msprobe/visualization/graph/graph.py +14 -16
  417. msprobe/visualization/graph/node_colors.py +34 -31
  418. msprobe/visualization/graph/node_op.py +12 -11
  419. msprobe/visualization/graph_service.py +580 -205
  420. msprobe/visualization/utils.py +278 -31
  421. tb_graph_ascend/secure_build.py +175 -0
  422. tb_graph_ascend/server/__init__.py +15 -0
  423. tb_graph_ascend/server/app/__init__.py +15 -0
  424. tb_graph_ascend/server/app/model/__init__.py +15 -0
  425. tb_graph_ascend/server/app/model/hierarchy.py +348 -0
  426. tb_graph_ascend/server/app/model/layout_hierarchy_model.py +69 -0
  427. tb_graph_ascend/server/app/model/match_nodes_model.py +573 -0
  428. tb_graph_ascend/server/app/repositories/__init__.py +15 -0
  429. tb_graph_ascend/server/app/repositories/graph_repo_base.py +32 -0
  430. tb_graph_ascend/server/app/repositories/graph_repo_db.py +879 -0
  431. tb_graph_ascend/server/app/repositories/graph_repo_vis.py +83 -0
  432. tb_graph_ascend/server/app/service/__init__.py +18 -0
  433. tb_graph_ascend/server/app/service/graph_service_base.py +158 -0
  434. tb_graph_ascend/server/app/service/graph_service_db.py +438 -0
  435. tb_graph_ascend/server/app/service/graph_service_factory.py +54 -0
  436. tb_graph_ascend/server/app/service/graph_service_vis.py +480 -0
  437. tb_graph_ascend/server/app/utils/__init__.py +15 -0
  438. tb_graph_ascend/server/app/utils/constant.py +80 -0
  439. tb_graph_ascend/server/app/utils/file_check_wrapper.py +46 -0
  440. tb_graph_ascend/server/app/utils/global_state.py +95 -0
  441. tb_graph_ascend/server/app/utils/graph_utils.py +661 -0
  442. tb_graph_ascend/server/app/utils/i18n.py +153 -0
  443. tb_graph_ascend/server/app/utils/request_method.py +46 -0
  444. tb_graph_ascend/server/app/views/__init__.py +15 -0
  445. tb_graph_ascend/server/app/views/graph_views.py +304 -0
  446. tb_graph_ascend/server/plugin.py +108 -0
  447. tb_graph_ascend/server/static/index.html +9250 -0
  448. tb_graph_ascend/server/static/index.js +21 -0
  449. tb_graph_ascend/setup.py +57 -0
  450. mindstudio_probe-8.3.3.dist-info/LICENSE +0 -201
  451. mindstudio_probe-8.3.3.dist-info/RECORD +0 -491
  452. mindstudio_probe-8.3.3.dist-info/entry_points.txt +0 -2
  453. mindstudio_probe-8.3.3.dist-info/top_level.txt +0 -1
  454. msprobe/CMakeLists.txt +0 -5
  455. msprobe/README.md +0 -203
  456. msprobe/core/advisor/advisor.py +0 -129
  457. msprobe/core/advisor/advisor_const.py +0 -58
  458. msprobe/core/advisor/advisor_result.py +0 -58
  459. msprobe/core/compare/find_first/data_processor.py +0 -35
  460. msprobe/core/compare/highlight.py +0 -390
  461. msprobe/core/data_dump/data_collector.py +0 -356
  462. msprobe/core/grad_probe/constant.py +0 -90
  463. msprobe/core/grad_probe/grad_compare.py +0 -187
  464. msprobe/core/grad_probe/utils.py +0 -105
  465. msprobe/core/kernel_dump/kernel_config.py +0 -33
  466. msprobe/docs/01.installation.md +0 -250
  467. msprobe/docs/02.config_introduction.md +0 -221
  468. msprobe/docs/03.config_examples.md +0 -281
  469. msprobe/docs/04.kernel_dump_PyTorch.md +0 -73
  470. msprobe/docs/05.data_dump_PyTorch.md +0 -518
  471. msprobe/docs/06.data_dump_MindSpore.md +0 -618
  472. msprobe/docs/07.accuracy_checker_PyTorch.md +0 -310
  473. msprobe/docs/09.accuracy_checker_MindSpore.md +0 -120
  474. msprobe/docs/10.accuracy_compare_PyTorch.md +0 -637
  475. msprobe/docs/11.accuracy_compare_MindSpore.md +0 -769
  476. msprobe/docs/12.overflow_check_PyTorch.md +0 -82
  477. msprobe/docs/13.overflow_check_MindSpore.md +0 -33
  478. msprobe/docs/14.data_parse_PyTorch.md +0 -282
  479. msprobe/docs/15.free_benchmarking_PyTorch.md +0 -169
  480. msprobe/docs/16.free_benchmarking_MindSpore.md +0 -159
  481. msprobe/docs/17.grad_probe.md +0 -205
  482. msprobe/docs/18.online_dispatch.md +0 -89
  483. msprobe/docs/19.monitor.md +0 -753
  484. msprobe/docs/20.monitor_performance_baseline.md +0 -52
  485. msprobe/docs/21.visualization_PyTorch.md +0 -519
  486. msprobe/docs/22.visualization_MindSpore.md +0 -515
  487. msprobe/docs/23.generate_operator_PyTorch.md +0 -107
  488. msprobe/docs/24.code_mapping_Mindspore.md +0 -29
  489. msprobe/docs/25.tool_function_introduction.md +0 -29
  490. msprobe/docs/26.data_dump_PyTorch_baseline.md +0 -48
  491. msprobe/docs/27.dump_json_instruction.md +0 -795
  492. msprobe/docs/28.debugger_save_instruction.md +0 -288
  493. msprobe/docs/28.kernel_dump_MindSpore.md +0 -69
  494. msprobe/docs/29.data_dump_MSAdapter.md +0 -235
  495. msprobe/docs/30.overflow_check_MSAdapter.md +0 -31
  496. msprobe/docs/31.config_check.md +0 -107
  497. msprobe/docs/32.ckpt_compare.md +0 -69
  498. msprobe/docs/33.generate_operator_MindSpore.md +0 -181
  499. msprobe/docs/34.RL_collect.md +0 -101
  500. msprobe/docs/35.nan_analyze.md +0 -73
  501. msprobe/docs/36.calculation_result_change.md +0 -75
  502. msprobe/docs/FAQ.md +0 -232
  503. msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md +0 -146
  504. msprobe/docs/accuracy_checker_MindSpore/accuracy_checker_MindSpore_baseline.md +0 -14
  505. msprobe/docs/data_dump_MindSpore/data_dump_MindSpore_baseline.md +0 -33
  506. msprobe/docs/data_dump_MindSpore/dynamic_graph_quick_start_example.md +0 -217
  507. msprobe/docs/img/BLOOM-7B_1.png +0 -0
  508. msprobe/docs/img/BLOOM-7B_2.png +0 -0
  509. msprobe/docs/img/BLOOM-7B_3.png +0 -0
  510. msprobe/docs/img/BLOOM-7B_4.png +0 -0
  511. msprobe/docs/img/GPT-3_1.png +0 -0
  512. msprobe/docs/img/GPT-3_2.png +0 -0
  513. msprobe/docs/img/GPT-3_3.png +0 -0
  514. msprobe/docs/img/GPT-3_4.png +0 -0
  515. msprobe/docs/img/GPT-3_5.png +0 -0
  516. msprobe/docs/img/GPT-3_6.png +0 -0
  517. msprobe/docs/img/GPT-3_7.png +0 -0
  518. msprobe/docs/img/GPT-3_8.png +0 -0
  519. msprobe/docs/img/YOLOV5S_1.png +0 -0
  520. msprobe/docs/img/YOLOV5S_2.png +0 -0
  521. msprobe/docs/img/accuracy_checking_details.png +0 -0
  522. msprobe/docs/img/accuracy_checking_result.png +0 -0
  523. msprobe/docs/img/api_precision_compare_details.png +0 -0
  524. msprobe/docs/img/api_precision_compare_result.png +0 -0
  525. msprobe/docs/img/auto_analyze_log.png +0 -0
  526. msprobe/docs/img/compare_result.png +0 -0
  527. msprobe/docs/img/compare_result_pkl.png +0 -0
  528. msprobe/docs/img/compare_result_pkl_md5.png.png +0 -0
  529. msprobe/docs/img/cpu_info.png +0 -0
  530. msprobe/docs/img/free_benchmark.png +0 -0
  531. msprobe/docs/img/free_benchmark_framework.png +0 -0
  532. msprobe/docs/img/grad_probe_image-1.png +0 -0
  533. msprobe/docs/img/grad_probe_image-2.png +0 -0
  534. msprobe/docs/img/grad_probe_image-3.png +0 -0
  535. msprobe/docs/img/grad_probe_image-4.png +0 -0
  536. msprobe/docs/img/grad_probe_image.png +0 -0
  537. msprobe/docs/img/merge_result.png +0 -0
  538. msprobe/docs/img/module_compare.png +0 -0
  539. msprobe/docs/img/monitor/cpu_info.png +0 -0
  540. msprobe/docs/img/monitor/step_count_per_record.png +0 -0
  541. msprobe/docs/img/ms_dump.png +0 -0
  542. msprobe/docs/img/ms_layer.png +0 -0
  543. msprobe/docs/img/pt_dump.png +0 -0
  544. msprobe/docs/img/save_compare_result_sample.png +0 -0
  545. msprobe/docs/img/visualization/fuzzy_match_ms.png +0 -0
  546. msprobe/docs/img/visualization/fuzzy_match_pt.png +0 -0
  547. msprobe/docs/img/visualization/proxy.png +0 -0
  548. msprobe/docs/img/visualization/tensorboard_1.png +0 -0
  549. msprobe/docs/img/visualization/tensorboard_2.png +0 -0
  550. msprobe/docs/img/visualization/vis_browser_1.png +0 -0
  551. msprobe/docs/img/visualization/vis_browser_2.png +0 -0
  552. msprobe/docs/img/visualization/vis_match_info.png +0 -0
  553. msprobe/docs/img/visualization/vis_precision_info.png +0 -0
  554. msprobe/docs/img/visualization/vis_search_info.png +0 -0
  555. msprobe/docs/img/visualization/vis_show_info.png +0 -0
  556. msprobe/docs/img/visualization/vis_showcase.png +0 -0
  557. msprobe/docs/img/visualization/vis_unmatch_info.png +0 -0
  558. msprobe/docs/visualization/GPTModel.png +0 -0
  559. msprobe/docs/visualization/ParallelMLP.png +0 -0
  560. msprobe/docs/visualization/layer_mapping_example.md +0 -132
  561. msprobe/docs/visualization/mapping.png +0 -0
  562. msprobe/docs/visualization/mapping1.png +0 -0
  563. msprobe/docs/visualization/mindspeed_llamafactoary_img/1.png +0 -0
  564. msprobe/docs/visualization/mindspeed_llamafactoary_img/2.png +0 -0
  565. msprobe/docs/visualization/mindspeed_llamafactoary_img/3.png +0 -0
  566. msprobe/docs/visualization/mindspeed_llamafactoary_img/4.png +0 -0
  567. msprobe/docs/visualization/mindspeed_llamafactoary_img/5.png +0 -0
  568. msprobe/docs/visualization/mindspeed_llamafactoary_img/6.png +0 -0
  569. msprobe/docs/visualization/mindspeed_llamafactoary_img/7.png +0 -0
  570. msprobe/docs/visualization/mindspeed_llamafactoary_img/llamafactory-qwen25vl.txt +0 -59
  571. msprobe/docs/visualization/mindspeed_llamafactoary_img/llamafactory1.png +0 -0
  572. msprobe/docs/visualization/mindspeed_llamafactoary_img/llamafactory2.png +0 -0
  573. msprobe/docs/visualization/mindspeed_llamafactoary_img/mindspeed-mm-qwen25vl.txt +0 -80
  574. msprobe/docs/visualization/mindspeed_llamafactoary_img/mindspeed1.png +0 -0
  575. msprobe/docs/visualization/mindspeed_llamafactoary_img/mindspeed2.png +0 -0
  576. msprobe/docs/visualization/mindspeed_llamafactory_mapping.md +0 -330
  577. msprobe/docs/visualization/module_name.png +0 -0
  578. msprobe/docs/visualization/module_name1.png +0 -0
  579. msprobe/docs/visualization/no_mapping.png +0 -0
  580. msprobe/docs/visualization/no_mapping1.png +0 -0
  581. msprobe/docs/visualization/no_mapping_analyze.png +0 -0
  582. msprobe/docs/visualization/top_layer.png +0 -0
  583. msprobe/mindspore/api_accuracy_checker/generate_op_script/op_generator.py +0 -460
  584. msprobe/mindspore/api_accuracy_checker/generate_op_script/operator_replication.template +0 -2081
  585. msprobe/mindspore/code_mapping/bind.py +0 -283
  586. msprobe/mindspore/code_mapping/cmd_parser.py +0 -40
  587. msprobe/mindspore/code_mapping/graph.py +0 -49
  588. msprobe/mindspore/code_mapping/graph_parser.py +0 -211
  589. msprobe/mindspore/code_mapping/main.py +0 -24
  590. msprobe/mindspore/code_mapping/processor.py +0 -34
  591. msprobe/mindspore/dym_loader/hook_dynamic_loader.cpp +0 -111
  592. msprobe/mindspore/dym_loader/hook_dynamic_loader.h +0 -52
  593. msprobe/mindspore/free_benchmark/api_pynative_self_check.py +0 -257
  594. msprobe/mindspore/free_benchmark/common/config.py +0 -27
  595. msprobe/mindspore/free_benchmark/common/handler_params.py +0 -31
  596. msprobe/mindspore/free_benchmark/common/utils.py +0 -100
  597. msprobe/mindspore/free_benchmark/data/support_wrap_ops.yaml +0 -638
  598. msprobe/mindspore/free_benchmark/handler/base_handler.py +0 -105
  599. msprobe/mindspore/free_benchmark/handler/check_handler.py +0 -55
  600. msprobe/mindspore/free_benchmark/handler/fix_handler.py +0 -51
  601. msprobe/mindspore/free_benchmark/handler/handler_factory.py +0 -36
  602. msprobe/mindspore/free_benchmark/perturbation/add_noise.py +0 -82
  603. msprobe/mindspore/free_benchmark/perturbation/base_perturbation.py +0 -45
  604. msprobe/mindspore/free_benchmark/perturbation/bit_noise.py +0 -78
  605. msprobe/mindspore/free_benchmark/perturbation/exchange_value.py +0 -77
  606. msprobe/mindspore/free_benchmark/perturbation/improve_precision.py +0 -56
  607. msprobe/mindspore/free_benchmark/perturbation/no_change.py +0 -27
  608. msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py +0 -46
  609. msprobe/mindspore/free_benchmark/self_check_tool_factory.py +0 -51
  610. msprobe/mindspore/grad_probe/global_context.py +0 -127
  611. msprobe/mindspore/grad_probe/grad_analyzer.py +0 -260
  612. msprobe/mindspore/grad_probe/grad_monitor.py +0 -42
  613. msprobe/mindspore/grad_probe/grad_stat_csv.py +0 -161
  614. msprobe/mindspore/grad_probe/hook.py +0 -115
  615. msprobe/mindspore/grad_probe/utils.py +0 -43
  616. msprobe/mindspore/mindtorch/__init__.py +0 -18
  617. msprobe/mindspore/ms_config.py +0 -153
  618. msprobe/mindspore/task_handler_factory.py +0 -44
  619. msprobe/nan_analyze/__init__.py +0 -14
  620. msprobe/pytorch/api_accuracy_checker/generate_op_script/config_op.json +0 -9
  621. msprobe/pytorch/api_accuracy_checker/generate_op_script/op_generator.py +0 -480
  622. msprobe/pytorch/api_accuracy_checker/generate_op_script/operator_replication.template +0 -567
  623. msprobe/pytorch/debugger/precision_debugger.py +0 -181
  624. msprobe/pytorch/free_benchmark/__init__.py +0 -23
  625. msprobe/pytorch/free_benchmark/common/constant.py +0 -85
  626. msprobe/pytorch/free_benchmark/common/counter.py +0 -87
  627. msprobe/pytorch/free_benchmark/common/enums.py +0 -80
  628. msprobe/pytorch/free_benchmark/common/params.py +0 -152
  629. msprobe/pytorch/free_benchmark/common/utils.py +0 -143
  630. msprobe/pytorch/free_benchmark/compare/grad_saver.py +0 -215
  631. msprobe/pytorch/free_benchmark/compare/single_benchmark.py +0 -121
  632. msprobe/pytorch/free_benchmark/main.py +0 -123
  633. msprobe/pytorch/free_benchmark/perturbed_layers/base_layer.py +0 -28
  634. msprobe/pytorch/free_benchmark/perturbed_layers/layer_factory.py +0 -56
  635. msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py +0 -107
  636. msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py +0 -121
  637. msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py +0 -89
  638. msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py +0 -87
  639. msprobe/pytorch/free_benchmark/perturbed_layers/npu/no_change.py +0 -43
  640. msprobe/pytorch/free_benchmark/perturbed_layers/npu/npu_base_layser.py +0 -60
  641. msprobe/pytorch/free_benchmark/perturbed_layers/run_cpu.py +0 -34
  642. msprobe/pytorch/free_benchmark/result_handlers/base_handler.py +0 -252
  643. msprobe/pytorch/free_benchmark/result_handlers/check_handler.py +0 -54
  644. msprobe/pytorch/free_benchmark/result_handlers/fix_handler.py +0 -40
  645. msprobe/pytorch/free_benchmark/result_handlers/handler_factory.py +0 -45
  646. msprobe/pytorch/free_benchmark/result_handlers/preheat_handler.py +0 -181
  647. msprobe/pytorch/grad_probe/__init__.py +0 -0
  648. msprobe/pytorch/grad_probe/grad_monitor.py +0 -108
  649. msprobe/pytorch/grad_probe/grad_stat_csv.py +0 -160
  650. msprobe/pytorch/hook_module/__init__.py +0 -16
  651. msprobe/pytorch/hook_module/wrap_aten.py +0 -111
  652. msprobe/pytorch/online_dispatch/__init__.py +0 -19
  653. msprobe/pytorch/online_dispatch/compare.py +0 -224
  654. msprobe/pytorch/online_dispatch/dispatch.py +0 -332
  655. msprobe/pytorch/online_dispatch/dump_compare.py +0 -179
  656. msprobe/pytorch/online_dispatch/single_compare.py +0 -412
  657. msprobe/pytorch/online_dispatch/torch_ops_config.yaml +0 -58
  658. msprobe/pytorch/online_dispatch/utils.py +0 -158
  659. msprobe/pytorch/parse_tool/__init__.py +0 -0
  660. msprobe/pytorch/parse_tool/cli.py +0 -31
  661. msprobe/pytorch/parse_tool/lib/__init__.py +0 -0
  662. msprobe/pytorch/parse_tool/lib/compare.py +0 -253
  663. msprobe/pytorch/parse_tool/lib/config.py +0 -50
  664. msprobe/pytorch/parse_tool/lib/file_desc.py +0 -45
  665. msprobe/pytorch/parse_tool/lib/interactive_cli.py +0 -97
  666. msprobe/pytorch/parse_tool/lib/parse_exception.py +0 -54
  667. msprobe/pytorch/parse_tool/lib/parse_tool.py +0 -161
  668. msprobe/pytorch/parse_tool/lib/utils.py +0 -299
  669. msprobe/pytorch/parse_tool/lib/visualization.py +0 -85
  670. msprobe/pytorch/pt_config.py +0 -299
  671. /msprobe/core/{grad_probe → dump}/__init__.py +0 -0
  672. /msprobe/{mindspore/code_mapping → core/dump/api_dump}/__init__.py +0 -0
  673. /msprobe/{mindspore/debugger → core/dump/data_dump}/__init__.py +0 -0
  674. /msprobe/{mindspore/exception_dump → core/dump/data_dump/data_processor}/__init__.py +0 -0
  675. /msprobe/{mindspore/free_benchmark → core/dump/debugger}/__init__.py +0 -0
  676. /msprobe/{mindspore/free_benchmark/common → core/dump/kernel_dump}/__init__.py +0 -0
  677. /msprobe/mindspore/{free_benchmark/handler → dump/debugger}/__init__.py +0 -0
  678. /msprobe/mindspore/{grad_probe → dump/dump_processor}/__init__.py +0 -0
  679. /msprobe/mindspore/{overflow_check → dump/exception_dump}/__init__.py +0 -0
  680. /msprobe/mindspore/{mindtorch → dump/mindtorch}/mindtorch_adaptor.py +0 -0
  681. /msprobe/{pytorch/api_accuracy_checker/run_ut → mindspore/dump/overflow_check}/__init__.py +0 -0
  682. /msprobe/{pytorch/debugger → mindspore/monitor}/__init__.py +0 -0
  683. /msprobe/{pytorch/free_benchmark/common → msaccucmp}/__init__.py +0 -0
  684. /msprobe/pytorch/api_accuracy_checker/{run_ut → acc_check}/.keep +0 -0
  685. /msprobe/pytorch/{free_benchmark/perturbed_layers → api_accuracy_checker/acc_check}/__init__.py +0 -0
  686. /msprobe/pytorch/api_accuracy_checker/{run_ut → acc_check}/torch_ut_setting.json +0 -0
  687. /msprobe/pytorch/{free_benchmark/perturbed_layers/npu → dump/api_dump}/__init__.py +0 -0
  688. /msprobe/pytorch/{hook_module → dump/api_dump}/support_wrap_ops.yaml +0 -0
  689. /msprobe/pytorch/{free_benchmark/result_handlers → dump/debugger}/__init__.py +0 -0
@@ -0,0 +1,661 @@
1
+ # -------------------------------------------------------------------------
2
+ # This file is part of the MindStudio project.
3
+ # Copyright (c) 2025 Huawei Technologies Co.,Ltd.
4
+ #
5
+ # MindStudio is licensed under Mulan PSL v2.
6
+ # You can use this software according to the terms and conditions of the Mulan PSL v2.
7
+ # You may obtain a copy of Mulan PSL v2 at:
8
+ #
9
+ # http://license.coscl.org.cn/MulanPSL2
10
+ #
11
+ # THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12
+ # EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13
+ # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14
+ # See the Mulan PSL v2 for more details.
15
+ # -------------------------------------------------------------------------
16
+
17
+ import math
18
+ import os
19
+ import json
20
+ import re
21
+ import stat
22
+ import threading
23
+ from functools import cmp_to_key
24
+ from pathlib import Path
25
+ from tensorboard.util import tb_logging
26
+ from .global_state import GraphState
27
+ from .constant import DataType, FILE_NAME_REGEX, MAX_FILE_SIZE, PERM_GROUP_WRITE, PERM_OTHER_WRITE, COLOR_PATTERN
28
+ from .i18n import language, ZH_CN
29
+ # 创建一个全局锁
30
+ _thread_local_lock = threading.Lock()
31
+ logger = tb_logging.get_logger()
32
+ FILE_PATH_MAX_LENGTH = 4096
33
+
34
+
35
+ class GraphUtils:
36
+
37
+ @staticmethod
38
+ def t(key):
39
+ lang = GraphState.get_global_value('lang', ZH_CN)
40
+ return language.get(lang).get(key, '')
41
+
42
+ @staticmethod
43
+ def get_graph_data(meta_data):
44
+ if not meta_data:
45
+ return None, 'Error: no query parameters provided'
46
+ try:
47
+ run_name = meta_data.get('run')
48
+ runs = GraphState.get_global_value('runs', {})
49
+ run = runs.get(run_name) or run_name
50
+ tag = meta_data.get('tag')
51
+ current_tag = GraphState.get_global_value('current_tag')
52
+ current_run = GraphState.get_global_value('current_run')
53
+ if current_tag == tag and current_run == run:
54
+ return GraphState.get_global_value('current_file_data'), None # 直接返回获取结果
55
+ else:
56
+ json_data, error_message = GraphUtils.safe_load_data(run, f"{tag}.vis", False)
57
+ if error_message:
58
+ return None, 'Error: fail to get graph data'
59
+ GraphState.set_global_value('current_file_data', json_data)
60
+ GraphState.set_global_value('current_tag', tag)
61
+ GraphState.set_global_value('current_run', run)
62
+ return json_data, error_message
63
+ except Exception as e:
64
+ logger.error(f'Error: fail to get graph data by {meta_data}, error: {e}')
65
+ return None, 'Error: fail to get graph data'
66
+
67
+ @staticmethod
68
+ def get_opposite_node_name(node_name):
69
+ opposite_node_name = ''
70
+ # 如果npu_node_name包含forward,则opposite_npu_node_name为npu_node_name替换forward为backward
71
+ if 'forward' in node_name:
72
+ opposite_node_name = node_name.replace('forward', 'backward')
73
+ else:
74
+ opposite_node_name = node_name.replace('backward', 'forward')
75
+ return opposite_node_name
76
+
77
+ @staticmethod
78
+ def get_parent_node_list(graph_data, node_name):
79
+ """获取父节点列表"""
80
+ # 如果 graph_data 为空或 node_name 不存在,直接返回空列表
81
+ if not graph_data or not node_name:
82
+ return []
83
+
84
+ node_list = [] # 存储结果的列表
85
+ current_node = node_name # 当前节点名称
86
+ nodes = graph_data['node'] # 提取节点数据,避免重复访问
87
+
88
+ while current_node:
89
+ node_list.append(current_node) # 将当前节点加入列表
90
+ # 获取当前节点的 upnode
91
+ current_node = nodes.get(current_node, {}).get('upnode')
92
+
93
+ # 检测循环引用(防止死循环)
94
+ if current_node in node_list:
95
+ raise ValueError(
96
+ f"{GraphUtils.t('circularReferenceError1')}"
97
+ f"{current_node}{GraphUtils.t('circularReferenceError2')}"
98
+ )
99
+
100
+ return list(reversed(node_list)) # 返回结果列表
101
+
102
+ @staticmethod
103
+ def split_graph_data_by_microstep(graph_data, micro_step):
104
+ graph_nodes = graph_data.get('node', {})
105
+ if str(micro_step) == str(-1):
106
+ return graph_nodes
107
+ splited_graph_data = {}
108
+ node_list = [] # 存储已遍历的列表
109
+
110
+ def traverse_npu(graph_nodes, subnodes):
111
+ for node_name in subnodes:
112
+ node_data = graph_nodes.get(node_name, {})
113
+ micro_step_id = node_data.get('micro_step_id')
114
+ if str(micro_step_id) == str(micro_step) or micro_step_id is None:
115
+ splited_graph_data[node_name] = (node_data)
116
+ if node_name in node_list:
117
+ raise ValueError(
118
+ f"{GraphUtils.t('circularReferenceError1')}"
119
+ f"{node_name}{GraphUtils.t('circularReferenceError2')}"
120
+ )
121
+ node_list.append(node_name)
122
+ traverse_npu(graph_nodes, node_data.get('subnodes', []))
123
+
124
+ root = graph_data.get('root')
125
+ root_subnodes = graph_nodes.get(root, {}).get('subnodes', [])
126
+ node_list.append(root)
127
+ traverse_npu(graph_nodes, root_subnodes)
128
+ return splited_graph_data
129
+
130
+ @staticmethod
131
+ def walk_with_max_depth(logdir, max_depth):
132
+ for root, dirs, files in os.walk(logdir):
133
+ # 计算当前 root 相对于 top 的深度
134
+ depth = root[len(logdir):].count(os.sep) + 1
135
+ if depth >= max_depth:
136
+ del dirs[:]
137
+ yield root, dirs, files
138
+
139
+ @staticmethod
140
+ def safe_json_loads(json_str, default_value=None):
141
+ """
142
+ 安全地解析 JSON 字符串,带长度限制和异常处理。
143
+ :param json_str: 要解析的 JSON 字符串
144
+ :param default_value: 如果解析失败返回的默认值
145
+ :return: 解析后的 Python 对象 或 default_value
146
+ """
147
+ # 类型检查
148
+ if not isinstance(json_str, str):
149
+ return default_value
150
+ # 长度限制
151
+ if len(json_str) > MAX_FILE_SIZE:
152
+ return default_value
153
+ try:
154
+ result = json.loads(json_str)
155
+ return result
156
+ except json.JSONDecodeError as e:
157
+ logger.error(f"JSON decode error: {e}")
158
+ return default_value
159
+ except Exception as e:
160
+ logger.error(f"Unexpected error: {e}")
161
+ return default_value
162
+
163
+ @staticmethod
164
+ def safe_get_node_info(data, default_value=None):
165
+ node_info = data.get('nodeInfo')
166
+ try:
167
+ # 长度限制 - 检查字典转为字符串后的长度
168
+ node_info_str = str(node_info)
169
+ if len(node_info_str) > MAX_FILE_SIZE:
170
+ logger.error(f"Input length exceeds {MAX_FILE_SIZE} characters.")
171
+ return default_value
172
+ # 验证必要字段是否存在
173
+ required_fields = ["nodeName", "nodeType"]
174
+ for field in required_fields:
175
+ if field not in node_info:
176
+ logger.error(f"Field {field} is missing in metadata.")
177
+ return default_value
178
+ return node_info
179
+ except json.JSONDecodeError:
180
+ logger.error("NodeInfo parameter is not in valid JSON format.")
181
+ return default_value
182
+ except Exception as e:
183
+ logger.error(f"An error occurred while parsing the nodeInfo parameter: {str(e)}")
184
+ return default_value
185
+
186
+ @staticmethod
187
+ def safe_get_meta_data(data, default_value=None):
188
+ meta_data = data.get('metaData')
189
+ try:
190
+ # 长度限制
191
+ meta_data_str = str(meta_data)
192
+ if len(meta_data_str) > MAX_FILE_SIZE:
193
+ logger.error(f"Input length exceeds {MAX_FILE_SIZE} characters.")
194
+ return default_value
195
+ # 验证必要字段是否存在
196
+ required_fields = ["tag", "microStep", "run", "type", 'lang']
197
+ for field in required_fields:
198
+ if field not in meta_data:
199
+ logger.error(f"Field {field} is missing in metadata.")
200
+ return default_value
201
+ config_info = GraphState.get_global_value('config_info', {})
202
+ micro_step = meta_data.get('microStep')
203
+ # 验证非必要字段如果存在,进行范围限制
204
+ if config_info.get('microSteps'):
205
+ if int(micro_step) != -1 and int(micro_step) not in list(range(config_info.get('microSteps'))):
206
+ logger.error(f"Field microStep {micro_step} is not in config_info.")
207
+ return default_value
208
+ if meta_data.get('type') == DataType.DB.value and config_info:
209
+ rank = meta_data.get('rank')
210
+ step = meta_data.get('step')
211
+ if config_info.get('ranks') and rank:
212
+ if rank not in config_info.get('ranks'):
213
+ logger.error(f"Field rank {rank} is not in config_info.")
214
+ return default_value
215
+ if config_info.get('steps') and step:
216
+ if step not in config_info.get('steps'):
217
+ logger.error(f"Field step {step} is not in config_info.")
218
+ return default_value
219
+ return meta_data
220
+
221
+ except json.JSONDecodeError:
222
+ logger.error("MetaData parameter is not in valid JSON format.")
223
+ return default_value
224
+ except Exception as e:
225
+ logger.error(f"An error occurred while parsing the metatdata parameter: {str(e)}")
226
+ return default_value
227
+
228
+ @staticmethod
229
+ def remove_prefix(node_data, prefix):
230
+ if node_data is None:
231
+ return {}
232
+ return {k[len(prefix):] if k.startswith(prefix) else k: v for k, v in node_data.items()}
233
+
234
+ @staticmethod
235
+ def convert_to_float(value):
236
+ try:
237
+ if isinstance(value, str):
238
+ # 处理'0.0%, 由于Mean小于1e-06, 建议不参考此相对误差,请参考绝对误差'和'0.0%'的情况
239
+ value = value.split(',')[0]
240
+ if value.endswith('%'):
241
+ value = value.replace('%', '').strip()
242
+ return float(value) / 100.0
243
+ return float(value)
244
+ except ValueError:
245
+ return float('nan')
246
+
247
+ @staticmethod
248
+ def format_relative_err(value):
249
+ """格式化相对误差为百分比,保留四位小数"""
250
+ if value is None or math.isnan(value):
251
+ return "NaN"
252
+ else:
253
+ return "{:.4%}".format(value)
254
+
255
+ @staticmethod
256
+ def nan_to_str(value):
257
+ """将 NaN 转换为 'NaN' 字符串"""
258
+ return "NaN" if math.isnan(value) else value
259
+
260
+ @staticmethod
261
+ def is_relative_to(path, base):
262
+ abs_path = os.path.abspath(path)
263
+ abs_base = os.path.abspath(base)
264
+ return os.path.commonpath([abs_path, abs_base]) == str(abs_base)
265
+
266
+ @staticmethod
267
+ def bytes_to_human_readable(size_bytes, decimal_places=2):
268
+ """
269
+ 将字节大小转换为更易读的格式(如 KB、MB、GB 等)。
270
+
271
+ :param size_bytes: int 或 float,表示字节大小
272
+ :param decimal_places: 保留的小数位数,默认为 2
273
+ :return: str,人类可读的大小表示
274
+ """
275
+ if size_bytes == 0:
276
+ return "0 B"
277
+
278
+ units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
279
+ unit_index = 0
280
+
281
+ while size_bytes >= 1024 and unit_index < len(units) - 1:
282
+ size_bytes /= 1024.0
283
+ unit_index += 1
284
+
285
+ return f"{size_bytes:.{decimal_places}f} {units[unit_index]}"
286
+
287
+ @staticmethod
288
+ def safe_save_data(data, run_name, tag):
289
+ runs = GraphState.get_global_value('runs', {})
290
+ run = runs.get(run_name) or run_name
291
+ if not run or not tag:
292
+ error_message = 'The query parameters "run" and "tag" are required'
293
+ return None, error_message
294
+ try:
295
+ # 检查 tag 是否为合法文件名
296
+ if not re.match(FILE_NAME_REGEX, tag):
297
+ raise ValueError(f"{GraphUtils.t('invalidTag')}{tag}.")
298
+ # 构建文件路径并标准化
299
+ file_path = os.path.join(run, tag)
300
+ # 目录安全校验
301
+ success, error = GraphUtils.safe_check_save_file_path(run, True)
302
+ if not success:
303
+ raise PermissionError(error)
304
+ # 文件安全校验
305
+ success, error = GraphUtils.safe_check_save_file_path(file_path)
306
+ if not success:
307
+ raise PermissionError(error)
308
+ # 尝试写入文件
309
+ with _thread_local_lock:
310
+ with open(file_path, "w", encoding="utf-8") as f:
311
+ json.dump(data, f, ensure_ascii=False, indent=4)
312
+ f.flush() # 强制将缓冲区内容写入操作系统
313
+ os.fsync(f.fileno()) # 强制将缓冲区内容写入磁盘
314
+ os.chmod(file_path, 0o640)
315
+ # 最终校验(防御TOCTOU攻击)
316
+ if os.path.islink(file_path):
317
+ raise RuntimeError("The file has been replaced with a symbolic link")
318
+ return True, None
319
+ except (TypeError, ValueError) as e:
320
+ logger.error(f"Invalid data: {e}")
321
+ return None, 'Invalid data'
322
+ except OSError as e:
323
+ logger.error(f"Failed to create directory: {run}. Error: {e}\n")
324
+ return None, 'failed to create directory '
325
+ except Exception as e:
326
+ logger.error(f'Error: File "{file_path}" is not accessible. Error: {e}')
327
+ return None, 'failed to save file'
328
+
329
+ @staticmethod
330
+ def safe_load_data(run_name, tag, only_check=False):
331
+ runs = GraphState.get_global_value('runs', {})
332
+ run_dir = runs.get(str(run_name)) or run_name
333
+ safe_base_dir = GraphState.get_global_value('logdir')
334
+ """Load a single .vis file from a given directory based on the tag."""
335
+ if run_dir is None or tag is None:
336
+ error_message = 'The query parameters "run" and "tag" are required'
337
+ return None, error_message
338
+ try:
339
+ file_path = os.path.join(run_dir, tag)
340
+ # 安全验证:基础路径校验
341
+ if not GraphUtils.is_relative_to(file_path, safe_base_dir):
342
+ raise ValueError(GraphUtils.t('pathMayNotInSecureDirectory'))
343
+ # 目录安全校验
344
+ success, error = GraphUtils.safe_check_load_file_path(run_dir, True)
345
+ if not success:
346
+ raise PermissionError(error)
347
+ # 文件安全校验
348
+ success, error = GraphUtils.safe_check_load_file_path(file_path)
349
+ if not success:
350
+ raise PermissionError(error)
351
+ # 读取文件比较耗时,支持onlyCheck参数,仅进行安全校验
352
+ if only_check:
353
+ return True, None
354
+ # 尝试解析 JSON 文件,校验文件内容是否合理
355
+ with _thread_local_lock:
356
+ with open(file_path, 'r', encoding='utf-8') as f:
357
+ return json.load(f), None
358
+ except json.JSONDecodeError:
359
+ logger.error(f'Error: File "{file_path}" is not a valid JSON file!')
360
+ return None, "File is not a valid JSON file!"
361
+ except Exception as e:
362
+ logger.error(f'Error: File "{file_path}" is not accessible. Error: {e}')
363
+ return None, e
364
+
365
+ @staticmethod
366
+ def safe_check_save_file_path(file_path, is_dir=False):
367
+ file_path = os.path.normpath(file_path) # 标准化路径
368
+ real_path = os.path.realpath(file_path)
369
+ safe_base_dir = GraphState.get_global_value('logdir')
370
+ try:
371
+ # 安全验证:路径长度检查
372
+ if len(file_path) > FILE_PATH_MAX_LENGTH:
373
+ raise PermissionError(f"Path length exceeds limit")
374
+ if not is_dir and not os.path.exists(file_path):
375
+ return True, None
376
+ st = os.stat(file_path)
377
+ # 安全验证:禁止符号链接文件
378
+ if os.path.islink(file_path):
379
+ raise PermissionError("The target file is a symbolic link")
380
+ # 安全验证:检查目录是否存在,如果不存在则创建
381
+ if is_dir and not os.path.exists(real_path):
382
+ os.makedirs(real_path, exist_ok=True)
383
+ os.chmod(file_path, 0o640)
384
+ # 权限校验:检查是否有写权限
385
+ if not os.stat(file_path).st_mode & stat.S_IWUSR:
386
+ raise PermissionError(f"No write permission for directory\n")
387
+ # 安全验证: 非windows系统下,属主检查
388
+ if os.name != 'nt':
389
+ current_uid = os.getuid()
390
+ # 如果是root用户,跳过后续权限检查
391
+ if current_uid == 0:
392
+ logger.warning("""Security Warning: Do not run this tool as root.
393
+ Running with elevated privileges may compromise system security.
394
+ Use a regular user account.""")
395
+ return True, None
396
+ # 属主检查
397
+ if st.st_uid != current_uid:
398
+ raise PermissionError(f"Directory is not owned by the current user")
399
+ # group和其他用户不可写检查
400
+ if st.st_mode & PERM_GROUP_WRITE or st.st_mode & PERM_OTHER_WRITE:
401
+ raise PermissionError(
402
+ f"Directory has group or other write permission, there may be a risk of data tampering.")
403
+ return True, None
404
+ except Exception as e:
405
+ logger.error(e)
406
+ return False, e
407
+
408
+ @staticmethod
409
+ def safe_check_load_file_path(file_path, is_dir=False):
410
+ # 权限常量定义
411
+ file_path = os.path.normpath(file_path) # 标准化路径
412
+ real_path = os.path.realpath(file_path)
413
+ st = os.stat(real_path)
414
+ try:
415
+ # 安全验证:路径长度检查
416
+ if len(real_path) > FILE_PATH_MAX_LENGTH:
417
+ raise PermissionError(
418
+ f"Path is too long (max {FILE_PATH_MAX_LENGTH} characters). Please use a shorter path."
419
+ )
420
+ # 安全检查:文件存在性验证
421
+ if not os.path.exists(real_path):
422
+ raise FileNotFoundError(f"File or directory does not exist,please check the path and ensure it exists.")
423
+ # 安全验证:禁止符号链接文件
424
+ if os.path.islink(file_path):
425
+ raise PermissionError(f"Symbolic links are not allowed,Use a real file path instead.")
426
+ # 安全验证:文件类型检查(防御TOCTOU攻击)
427
+ # 文件类型
428
+ if not is_dir and not os.path.isfile(real_path):
429
+ raise PermissionError(
430
+ f"Path is not a regular file."
431
+ "make sure the path points to a valid file (not a directory or device)."
432
+ )
433
+ # 目录类型
434
+ if is_dir and not Path(real_path).is_dir():
435
+ raise PermissionError(
436
+ f"Expected a directory, but it does not exist or is not a directory."
437
+ "Please check the path and ensure it is a valid directory."
438
+ )
439
+ # 可读性检查
440
+ if not st.st_mode & stat.S_IRUSR:
441
+ raise PermissionError(
442
+ f"Current user lacks read permission on file or directory"
443
+ "Run 'chmod u+r \"<path>\"' to grant read access"
444
+ )
445
+ # 文件大小校验
446
+ if not is_dir and os.path.getsize(file_path) > MAX_FILE_SIZE:
447
+ file_size = GraphUtils.bytes_to_human_readable(os.path.getsize(file_path))
448
+ max_size = GraphUtils.bytes_to_human_readable(MAX_FILE_SIZE)
449
+ raise PermissionError(
450
+ f"File size exceeds limit ({file_size} > {max_size})."
451
+ "reduce file size or adjust MAX_FILE_SIZE if needed"
452
+ )
453
+ # 非windows系统下,属主检查
454
+ if os.name != 'nt':
455
+ current_uid = os.getuid()
456
+ # 如果是root用户,跳过后续权限检查
457
+ if current_uid == 0:
458
+ logger.warning("""Security Warning: Do not run this tool as root.
459
+ Running with elevated privileges may compromise system security.
460
+ Use a regular user account.""")
461
+ return True, None
462
+ # 属主检查
463
+ if st.st_uid != current_uid:
464
+ raise PermissionError(
465
+ f"File or directory is not owned by current user,"
466
+ "Run 'chown <user> \"<path>\"' to fix ownership."
467
+ )
468
+ # group和其他用户不可写检查
469
+ if st.st_mode & PERM_GROUP_WRITE or st.st_mode & PERM_OTHER_WRITE:
470
+ raise PermissionError(
471
+ f"File has insecure permissions: group or others have write access. "
472
+ "Run 'chmod go-w \"<path>\"' to remove write permissions for group and others."
473
+ )
474
+ return True, None
475
+ except Exception as e:
476
+ logger.error(e)
477
+ return False, e
478
+
479
+ @staticmethod
480
+ def find_config_files(run_name):
481
+ """
482
+ 在指定目录下查找所有以 .vis.config 结尾的文件(不包括子目录)
483
+ :param directory: 要搜索的目标目录路径
484
+ :return: 包含所有匹配文件路径的列表
485
+ """
486
+ runs = GraphState.get_global_value('runs', {})
487
+ run = runs.get(run_name)
488
+ dir_path = Path(run)
489
+ try:
490
+ if GraphUtils.safe_check_load_file_path(run, True):
491
+ return [
492
+ file.name for file in dir_path.iterdir()
493
+ if file.is_file() and file.name.endswith('.vis.config')
494
+ ]
495
+ else:
496
+ return []
497
+ except Exception as e:
498
+ logger.error(e)
499
+ return []
500
+
501
+ @staticmethod
502
+ def compare_tag_names(a: str, b: str) -> int:
503
+ """自然排序比较函数,支持路径格式和数字顺序"""
504
+
505
+ # 辅助函数:标准化路径并分割组件
506
+ def split_components(s: str) -> list:
507
+ s = s.replace('\\', '/') # 统一路径分隔符
508
+ return [c for c in re.split(r'[/_]+', s) if c] # 按 /_ 分割非空组件
509
+
510
+ # 辅助函数:将组件拆分为数字/字符串混合的 tokens
511
+ def tokenize(component: str) -> list:
512
+ tokens = []
513
+ buffer = []
514
+ is_num = False
515
+
516
+ for char in component:
517
+ if char.isdigit() == is_num and buffer:
518
+ buffer.append(char)
519
+ else:
520
+ if buffer:
521
+ tokens.append(int(''.join(buffer)) if is_num else ''.join(buffer))
522
+ buffer = [char]
523
+ is_num = char.isdigit()
524
+
525
+ if buffer:
526
+ tokens.append(int(''.join(buffer)) if is_num else ''.join(buffer))
527
+ return tokens
528
+
529
+ # 逐级比较组件
530
+ a_comps = split_components(a)
531
+ b_comps = split_components(b)
532
+
533
+ for a_part, b_part in zip(a_comps, b_comps):
534
+ a_tokens = tokenize(a_part)
535
+ b_tokens = tokenize(b_part)
536
+
537
+ # 比较 token 序列
538
+ for a_tok, b_tok in zip(a_tokens, b_tokens):
539
+ if isinstance(a_tok, int) and isinstance(b_tok, int):
540
+ if a_tok != b_tok:
541
+ return a_tok - b_tok
542
+ elif isinstance(a_tok, int):
543
+ return -1 # 数字优先于字母
544
+ elif isinstance(b_tok, int):
545
+ return 1
546
+ elif a_tok != b_tok:
547
+ return -1 if a_tok < b_tok else 1
548
+
549
+ # 处理子序列长度差异
550
+ if len(a_tokens) != len(b_tokens):
551
+ return len(a_tokens) - len(b_tokens)
552
+
553
+ # 处理组件数量差异
554
+ return len(a_comps) - len(b_comps)
555
+
556
+ @staticmethod
557
+ def sort_data(data: dict) -> dict:
558
+ """自然排序比较函数,支持路径格式和数字顺序"""
559
+ sorted_data = {}
560
+ sorted_keys = sorted(data.keys(), key=cmp_to_key(GraphUtils.compare_tag_names))
561
+ for k in sorted_keys:
562
+ # 对每个键对应的值列表进行排序
563
+ sorted_values = sorted(data.get(k, {}).get('tags'), key=cmp_to_key(GraphUtils.compare_tag_names))
564
+ sorted_data[k] = {'type': data.get(k, {}).get('type'), 'tags': sorted_values}
565
+
566
+ return sorted_data
567
+
568
+ @staticmethod
569
+ def is_safe_string(s: str) -> bool:
570
+ """
571
+ 安全检查字符串是否包含恶意内容(XSS 防护)
572
+ """
573
+ if not isinstance(s, str):
574
+ return False
575
+
576
+ # 1. 长度限制(字符数,非字节)
577
+ if len(s) > FILE_PATH_MAX_LENGTH:
578
+ return False
579
+
580
+ # 2. 转换为小写用于检测
581
+ s_lower = s.lower().strip()
582
+
583
+ # 3. 黑名单关键词(常见 XSS 向量)
584
+ dangerous_patterns = [
585
+ '<script', '<img', '<svg', '<iframe', '<video', '<audio',
586
+ 'javascript:', 'vbscript:', 'data:text/html',
587
+ 'onload=', 'onerror=', 'onmouseover=', 'onclick=',
588
+ 'eval(', 'alert(', 'document.cookie', 'document.location',
589
+ 'window.location', 'innerHTML', 'outerHTML', 'document.write'
590
+ ]
591
+
592
+ if any(pattern in s_lower for pattern in dangerous_patterns):
593
+ return False
594
+
595
+ return True
596
+
597
+ @staticmethod
598
+ def escape_html(input_str):
599
+ """
600
+ 将字符串中的特殊 HTML 字符转义为 HTML 实体。
601
+ """
602
+ html_escape_map = {
603
+ '&': '&amp;',
604
+ '<': '&lt;',
605
+ '>': '&gt;',
606
+ '"': '&quot;',
607
+ "'": '&#39;',
608
+ '/': '&#x2F;',
609
+ }
610
+ # 使用 str.translate 配合 str.maketrans 提高性能
611
+ translate_table = str.maketrans(html_escape_map)
612
+ return input_str.translate(translate_table)
613
+
614
+ @staticmethod
615
+ def validate_colors_param(colors_json: str):
616
+ # 合法颜色正则:#FFFFFF 格式,不区分大小写
617
+
618
+ """
619
+ 校验 colors 参数
620
+ 返回: (是否合法, 错误信息, 解析后的数据)
621
+ """
622
+
623
+ if not isinstance(colors_json, dict):
624
+ return False, GraphUtils.t('colorsNotObject'), {}
625
+
626
+ if len(colors_json) == 0:
627
+ return False, GraphUtils.t('colotsEmpty'), {}
628
+
629
+ for key, value in colors_json.items():
630
+ # 2. 校验颜色键
631
+ if not re.match(COLOR_PATTERN, key):
632
+ return False, f"{GraphUtils.t('illegalColorKey')}{key}", {}
633
+
634
+ if not isinstance(value, dict):
635
+ return False, f"{GraphUtils.t('colorValuesNotObject')}{key}", {}
636
+
637
+ if 'value' not in value:
638
+ return False, f"{GraphUtils.t('missingValueField')}{key}", {}
639
+
640
+ # 3. 校验 value 字段
641
+ val = value['value']
642
+ if isinstance(val, list):
643
+ if len(val) != 2:
644
+ return False, f"{GraphUtils.t('notArrayOfLength2')}{key}", {}
645
+ if not all(isinstance(x, (int, float)) for x in val):
646
+ return False, f"{GraphUtils.t('notArrayConsistNumbers')}{key}", {}
647
+ if val[0] >= val[1]:
648
+ return False, f"{GraphUtils.t('invalidValueRange')}{key}", {}
649
+ elif isinstance(val, str):
650
+ if val not in ["无匹配节点", "N/A", "No matching nodes"]:
651
+ return False, f"{GraphUtils.t('unsupportedValue')}{val}", {}
652
+ else:
653
+ return False, f"{GraphUtils.t('valueTypeError')}{key}", {}
654
+
655
+ # 4. 校验 description
656
+ desc = value.get('description', '')
657
+ if not GraphUtils.is_safe_string(desc):
658
+ return False, f"{GraphUtils.t('descriptionError')}{key}", {}
659
+ value['description'] = GraphUtils.escape_html(desc)
660
+
661
+ return True, None, colors_json