mindstudio-probe 8.2.1__py3-none-any.whl → 8.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. {mindstudio_probe-8.2.1.dist-info → mindstudio_probe-8.3.0.dist-info}/METADATA +1 -1
  2. {mindstudio_probe-8.2.1.dist-info → mindstudio_probe-8.3.0.dist-info}/RECORD +46 -37
  3. msprobe/README.md +3 -1
  4. msprobe/core/common/file_utils.py +80 -25
  5. msprobe/core/common/framework_adapter.py +7 -6
  6. msprobe/core/compare/diff_analyze/first_diff_analyze.py +4 -16
  7. msprobe/core/compare/find_first/utils.py +1 -1
  8. msprobe/core/hook_manager.py +16 -3
  9. msprobe/core/service.py +16 -5
  10. msprobe/docs/02.config_introduction.md +14 -1
  11. msprobe/docs/08.accuracy_checker_online_PyTorch.md +295 -0
  12. msprobe/docs/15.free_benchmarking_PyTorch.md +1 -1
  13. msprobe/docs/25.tool_function_introduction.md +1 -0
  14. msprobe/docs/26.data_dump_PyTorch_baseline.md +3 -3
  15. msprobe/mindspore/api_accuracy_checker/generate_op_script/op_generator.py +1 -1
  16. msprobe/mindspore/compare/utils.py +1 -2
  17. msprobe/msprobe.py +6 -4
  18. msprobe/pytorch/api_accuracy_checker/common/config.py +36 -3
  19. msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +24 -0
  20. msprobe/pytorch/api_accuracy_checker/compare/compare.py +12 -2
  21. msprobe/pytorch/api_accuracy_checker/config.yaml +6 -1
  22. msprobe/pytorch/api_accuracy_checker/generate_op_script/op_generator.py +1 -1
  23. msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +132 -12
  24. msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/__init__.py +0 -0
  25. msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py +205 -0
  26. msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py +378 -0
  27. msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py +239 -0
  28. msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py +115 -0
  29. msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py +250 -0
  30. msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/torch_ops_config.yaml +63 -0
  31. msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/utils.py +198 -0
  32. msprobe/pytorch/attl_manager.py +65 -0
  33. msprobe/pytorch/common/utils.py +22 -2
  34. msprobe/pytorch/compare/utils.py +1 -2
  35. msprobe/pytorch/debugger/debugger_config.py +10 -0
  36. msprobe/pytorch/dump/module_dump/hook_wrapper.py +24 -0
  37. msprobe/pytorch/dump/module_dump/module_processer.py +9 -3
  38. msprobe/pytorch/hook_module/api_register.py +6 -1
  39. msprobe/pytorch/pt_config.py +57 -2
  40. msprobe/pytorch/pytorch_service.py +11 -2
  41. msprobe/visualization/builder/graph_builder.py +1 -0
  42. msprobe/visualization/utils.py +11 -1
  43. msprobe/core/compare/diff_analyze/ignore_op_list.yaml +0 -3
  44. {mindstudio_probe-8.2.1.dist-info → mindstudio_probe-8.3.0.dist-info}/LICENSE +0 -0
  45. {mindstudio_probe-8.2.1.dist-info → mindstudio_probe-8.3.0.dist-info}/WHEEL +0 -0
  46. {mindstudio_probe-8.2.1.dist-info → mindstudio_probe-8.3.0.dist-info}/entry_points.txt +0 -0
  47. {mindstudio_probe-8.2.1.dist-info → mindstudio_probe-8.3.0.dist-info}/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: mindstudio-probe
3
- Version: 8.2.1
3
+ Version: 8.3.0
4
4
  Summary: Ascend Probe Utils
5
5
  Home-page: https://gitcode.com/Ascend/mstt/tree/master/debug/accuracy_tools/msprobe
6
6
  Author: Ascend Team
@@ -1,12 +1,12 @@
1
1
  msprobe/CMakeLists.txt,sha256=Cx_WsHVxbX2gWZ6Hxah2zYXz2eamSb4CgIlzKm9OQGA,125
2
- msprobe/README.md,sha256=eIoPDZlbJi14BGR1BfJrtY-prviPZdgXrIFHUOhw67w,8611
2
+ msprobe/README.md,sha256=CnXpjnrPs-Pzg4Iamw9PKBsTU7s1CVRKyRHNsqsuvo4,8736
3
3
  msprobe/__init__.py,sha256=q2S8b3VTSn1wTMrkf8Gm79OOq9EerMIK6Y26gFdu_0s,688
4
4
  msprobe/config.json,sha256=QS37x2t0D-grehkpHLLxRAEZqOrkNe40GwwOaVMx3Yc,1061
5
- msprobe/msprobe.py,sha256=pWQYbbADJ-uydeyIqga2tznOCC-spY5r_gr5m75XBC0,8146
5
+ msprobe/msprobe.py,sha256=KHFMop_Tm9hHKTd1c-_HaVZjkT8DQ6Kx6PejIkrClME,8228
6
6
  msprobe/core/__init__.py,sha256=X_OkpIz42vnqhGy6WCeuEvYK4IFx4BwCT5bQj7tuuwI,757
7
7
  msprobe/core/common_config.py,sha256=0kzhosc3FIE0_JfAnOYx8lYLbno7j0rmH0ST9On1jpQ,7182
8
- msprobe/core/hook_manager.py,sha256=Bqo9T_Wn_rCriTfs5BaKfdrW-WuRbkQVBcPLK_CG_P8,13742
9
- msprobe/core/service.py,sha256=FIYuVyHVVysFED5L3wPVZi8y8aT8ObiXtkfnhSnBGQo,13586
8
+ msprobe/core/hook_manager.py,sha256=heWLbR2spTpRRaNG3gWTKvHDdrYZQYG839ldrsusHwU,14568
9
+ msprobe/core/service.py,sha256=jNrCqBdlzS5xX0LlLGdB6yD1WB_K3lzJqKqpjjpUbgY,14062
10
10
  msprobe/core/advisor/advisor.py,sha256=WP8rBF2F9pcSphZONJDgNfz9c6GiRxbkWwPS9wRU4Tc,5900
11
11
  msprobe/core/advisor/advisor_const.py,sha256=thEEVYKPimv9WGxxaC4vBXQ-Coq6yBlZ394T_b49Ibw,2624
12
12
  msprobe/core/advisor/advisor_result.py,sha256=yM78n6NXLP1bjTsGeSddwb1bWQwtg1ZqjnPHmTzVxHE,2359
@@ -14,8 +14,8 @@ msprobe/core/common/const.py,sha256=r2wKstNjzjZOmGDw_q3jlN56hqA3tv5CtNNrdl8jZMk,
14
14
  msprobe/core/common/db_manager.py,sha256=OjC7Gr_4CKmrlLcODsLkeTv1dU-VU93fmLU0xMfiPzg,9088
15
15
  msprobe/core/common/decorator.py,sha256=lj1upkAkGYeIV3tnH0bFqp7VDIKqTkFVaE5GUOFUNUE,1785
16
16
  msprobe/core/common/exceptions.py,sha256=PJB4UTxT5VxyUE9aTCgwxiEExcuQZCcbu5V7tFdLVrY,4045
17
- msprobe/core/common/file_utils.py,sha256=WTa9u07a1IiWkWBOAWpTVW7GPQ_7r9t9BldBZ5WIaSA,38564
18
- msprobe/core/common/framework_adapter.py,sha256=xwOOc0cbDt_nIqHsHYT8G6uqhvZywLcOnB0w5UzATFk,6255
17
+ msprobe/core/common/file_utils.py,sha256=f8tOLHQcz4DQl7Qx2HVflhryWGZG1SQtq-IwVHDrl_8,40820
18
+ msprobe/core/common/framework_adapter.py,sha256=MTbJYJmTApkJme8tajFWhDfIPOo-hgWQYo3_w3DksD8,6236
19
19
  msprobe/core/common/global_lock.py,sha256=MQedbRCzJTLAYGeNeeD1T7Vmn349r7ymGKt8d-U3m2Y,2721
20
20
  msprobe/core/common/inplace_op_checker.py,sha256=gwxpddOtPS2dRGbI2JB5ZixgoPLCYyDICb_07N9XVaM,1864
21
21
  msprobe/core/common/inplace_ops.yaml,sha256=Ta1zh6eF0A_Trd345gts2xlhFsU0-ZTXKEQOqUHT-54,3178
@@ -35,13 +35,12 @@ msprobe/core/compare/npy_compare.py,sha256=aCylTUJuLUEsqqm-KySnY24egh5b36rTFh_dX
35
35
  msprobe/core/compare/utils.py,sha256=8vWFoqgqBY8XRc7eDOX2Lzaln26aPmaLF8Vs_xBQovk,35416
36
36
  msprobe/core/compare/diff_analyze/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
37
37
  msprobe/core/compare/diff_analyze/diff_analyze_threshold.yaml,sha256=77K_1RM8ICVuDSNVTFIRdeBu15HyzR2Kw9YjhHaoaT0,193
38
- msprobe/core/compare/diff_analyze/first_diff_analyze.py,sha256=806wUTMhtfgzCH_Kbx4iZcJempzMjK26KcfS-R2F4Oo,5373
39
- msprobe/core/compare/diff_analyze/ignore_op_list.yaml,sha256=OOTJqiUqvMxH1O4jD9zXIz88Tg4U9XOa1gJp_MuvYYs,33
38
+ msprobe/core/compare/diff_analyze/first_diff_analyze.py,sha256=zXfaVYuMwA-y0upjQaycuQ2Q7dc2kqH8kAgWcWqXkVs,4755
40
39
  msprobe/core/compare/find_first/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
41
40
  msprobe/core/compare/find_first/analyzer.py,sha256=pfvg4wvgdFT0KdtdiTRYhqihTBTWMaer1Q_h0rAxq1M,12803
42
41
  msprobe/core/compare/find_first/data_processor.py,sha256=C286RlYVHH_2BzFxLHni7C2V4Z3tJJ9L4ZQhMGtxqMA,1485
43
42
  msprobe/core/compare/find_first/graph.py,sha256=2TaXfRUl8YLXM0wosdgz8nwNz5OxQ7gpmdfZWJ7JyG0,8024
44
- msprobe/core/compare/find_first/utils.py,sha256=awFmw4BDzbIHMG9fx8mJn9orBET3ofd1KJiXjHR9Cn0,6979
43
+ msprobe/core/compare/find_first/utils.py,sha256=-RyNXSIs6YHX77e7GgmvUJYJmSCj2t9uAqHmqBHw1kg,6984
45
44
  msprobe/core/compare/layer_mapping/__init__.py,sha256=MLMlb-zcaZCmG7Ykh6cY28kBTw-3_waNBRKbajwQc74,776
46
45
  msprobe/core/compare/layer_mapping/data_scope_parser.py,sha256=WU9nelUpbkPa7RH_G3QAHRUhiWSHQ7Bt7TTMGFy51eM,9429
47
46
  msprobe/core/compare/layer_mapping/layer_mapping.py,sha256=h6E6V0Kyb-Z_-7FCGHH8Qiqj1KopVO0c_8zeDvqiCrU,10586
@@ -99,19 +98,20 @@ msprobe/core/single_save/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
99
98
  msprobe/core/single_save/single_comparator.py,sha256=faYvqjvZNpA0GrtpI5pXTYOlvV8x5toKFdKVv5Nzk60,10827
100
99
  msprobe/core/single_save/single_saver.py,sha256=IVS7fsG5CWmZNCeUenNwtVxhAmpIWSYHK7az2NpZ678,5562
101
100
  msprobe/docs/01.installation.md,sha256=QmJlZPTVzoC1h9Ikgg7nB_u7Ay16Aoi9ZwiirC5mfdc,10845
102
- msprobe/docs/02.config_introduction.md,sha256=zUgDHk7WkfZkQZ2QdFv1DIdcN62i18fe1AR1zf6BHP0,32190
101
+ msprobe/docs/02.config_introduction.md,sha256=zsN7EGYwlNZ9__Y8uLvEaP2d9K5XsRznuZ8afpO5R20,35057
103
102
  msprobe/docs/03.config_examples.md,sha256=lHv_HLaCxk1nlSPIUF7wQVO0RR6_Q5b9XrSp96fr1FE,4408
104
103
  msprobe/docs/04.kernel_dump_PyTorch.md,sha256=1qXNBrdkO2r8FJjHObr1r2XGy7gIzhvZ3DWC0JjDPng,3034
105
104
  msprobe/docs/05.data_dump_PyTorch.md,sha256=0ZLkESAFa_HBlo8nndc7cWXQ0O1l7BuNCCy7eARTcm8,22242
106
105
  msprobe/docs/06.data_dump_MindSpore.md,sha256=pweNRLRxNy1cu8h5cgmEf9auOBuKN3-Ezen5Et7a1s8,29284
107
106
  msprobe/docs/07.accuracy_checker_PyTorch.md,sha256=tLHA1eD0ldNv_4uQKArN-HzKcPdQbQighhK9VKuhuxQ,30993
107
+ msprobe/docs/08.accuracy_checker_online_PyTorch.md,sha256=u3kDOo3Y1YMSu2rd2QiIToYxqtpi4epfKk_ngdSc_Sw,14717
108
108
  msprobe/docs/09.accuracy_checker_MindSpore.md,sha256=XKEnEgqPJKZxL7JL3evZeIAa_LbMh8DR7sILqJ1camw,11320
109
109
  msprobe/docs/10.accuracy_compare_PyTorch.md,sha256=3H_fHJBMRweqAxEOT6trHSQPAMjP3xD9G3T_PIJPtVY,41611
110
110
  msprobe/docs/11.accuracy_compare_MindSpore.md,sha256=9zalk8EiYA_ex5kkiGEIn8yaNygLmJPplUoUEPs-C5w,40792
111
111
  msprobe/docs/12.overflow_check_PyTorch.md,sha256=VE76J_rFAFm0pw_bQEnKIV4FdXek4x7cenxTQQz2FK0,3793
112
112
  msprobe/docs/13.overflow_check_MindSpore.md,sha256=G67GjHo2VERAKsr1_uX0m67eNuG8E9aN6tmZiO1c_zg,2799
113
113
  msprobe/docs/14.data_parse_PyTorch.md,sha256=g1Hx3RFG7SIYTpTI2uySZjDIXwsF7LhVVghvITSp0L0,19125
114
- msprobe/docs/15.free_benchmarking_PyTorch.md,sha256=Dh4j8OKAqvMdNg4P7JPJnvbIrtLjku841HT_V5OrSTs,9732
114
+ msprobe/docs/15.free_benchmarking_PyTorch.md,sha256=6pIpLXx5lUu45_kdL31rhj9zV3Ew8X-yyqZXrJ_gZPk,9888
115
115
  msprobe/docs/16.free_benchmarking_MindSpore.md,sha256=swCOrnBSzU6Q5I0AHVwi2r0JfKp3VE1DgNHripyE01M,8195
116
116
  msprobe/docs/17.grad_probe.md,sha256=9g1aq6FettgpvzBxKj5C5W8bTsENPtt2itYaNOLcBk8,9624
117
117
  msprobe/docs/18.online_dispatch.md,sha256=Ae9ONIXF3wA2u0tikuDxV0nea_n5TIGug-PKTI9c7Ws,4170
@@ -121,8 +121,8 @@ msprobe/docs/21.visualization_PyTorch.md,sha256=Mds0wPfD9XjNJstj-SZshhpFvUFYjh7Z
121
121
  msprobe/docs/22.visualization_MindSpore.md,sha256=jtMgasjacyNEevhmrMT5Y8IOsAgEuDzBN7vvZYBJl7s,26605
122
122
  msprobe/docs/23.generate_operator_PyTorch.md,sha256=Tu_Jik6zWBIdf2ePOBeUAKnPMFnwcNdw1cuZg-Jfxbc,10289
123
123
  msprobe/docs/24.code_mapping_Mindspore.md,sha256=pFGY7uvNnaBqUDLQBDEay0In04nrpP2mFCzYfYYQEcs,2068
124
- msprobe/docs/25.tool_function_introduction.md,sha256=tqXbsW1xTbVBgke5bDd0B5NFxVGXmyqQkr63b9AtdFY,15668
125
- msprobe/docs/26.data_dump_PyTorch_baseline.md,sha256=nzBpiCcgvnuKKDD0CTS4k-htVnT45GFMG_rZH31NcLs,2554
124
+ msprobe/docs/25.tool_function_introduction.md,sha256=I1SFsz9RVdpjyNxLTyPPuKcLZIRDDWuwiVx2O2cQezc,16391
125
+ msprobe/docs/26.data_dump_PyTorch_baseline.md,sha256=Di-qxaT7TckS4ggAJOk3R99DGlJ67i9Vw7Jl9fIoJNA,2551
126
126
  msprobe/docs/27.dump_json_instruction.md,sha256=YW6q4U0ccoFwax_fZ5Gqr1CQo5PFlKaxr9S-4ocycZs,21025
127
127
  msprobe/docs/28.debugger_save_instruction.md,sha256=pxvLlc2GmR8tmapZ8IKZyIFGwt1tEKXU4jy8Ao4MwrI,10173
128
128
  msprobe/docs/28.kernel_dump_MindSpore.md,sha256=USVxSPdhuaiw6dlWGu02GVNMXgaryHzy9dQPBUd1FfE,2540
@@ -237,7 +237,7 @@ msprobe/mindspore/api_accuracy_checker/type_mapping.py,sha256=FL5SIQ3dJRKVmicvOO
237
237
  msprobe/mindspore/api_accuracy_checker/utils.py,sha256=uREcuxnIOy0Zeuqeo8wP4g33dcyqND_VtUIzKQBY2kQ,3951
238
238
  msprobe/mindspore/api_accuracy_checker/bench_functions/flash_attention_score.py,sha256=QJAveBVaeYN0Tmc75I-x4iA6nKpmk1Td_X3M76ClczA,23323
239
239
  msprobe/mindspore/api_accuracy_checker/bench_functions/fusion_operator.py,sha256=NXMgBWA8GW7NK7utftc5X-6aPp0c4uf_99kOUY1fwgA,1464
240
- msprobe/mindspore/api_accuracy_checker/generate_op_script/op_generator.py,sha256=lpHr1c1XesCNOfwFqTr97zqyPaE3H1cYY1SurYkzp_E,19490
240
+ msprobe/mindspore/api_accuracy_checker/generate_op_script/op_generator.py,sha256=GQxcUdKovpBbWuNtJtdvrlUQm05rj0RlV9RlpysZC7U,19491
241
241
  msprobe/mindspore/api_accuracy_checker/generate_op_script/operator_replication.template,sha256=ni56NrjDH0Qj-w13pDNw2XCqE-MOOmnq3jW2AuGm--I,81379
242
242
  msprobe/mindspore/code_mapping/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
243
243
  msprobe/mindspore/code_mapping/bind.py,sha256=oXxuSOmLN-qqcAyHC5gonw_PZMwEwGvTSW-ws5H5FiM,9549
@@ -253,7 +253,7 @@ msprobe/mindspore/compare/common_dir_compare.py,sha256=1vRos0zyadCmJefBjsvqVw5Xg
253
253
  msprobe/mindspore/compare/distributed_compare.py,sha256=H8zdjBBFyifyl3Nx21jtdoma6qN1duclPsJaArpAW1s,1556
254
254
  msprobe/mindspore/compare/ms_compare.py,sha256=yXs8wutN8fm_tcP39NQmNP5skai2CwfQScRhTTiUsM0,2237
255
255
  msprobe/mindspore/compare/ms_graph_compare.py,sha256=qph-gpAY0l5ApvQZ_xmq0SbUDVsuD8PtnD0jS7b6yRA,19949
256
- msprobe/mindspore/compare/utils.py,sha256=dNoyXNP7K5U68__flBLunmGVtog0OO21MP6aerpcwhI,1742
256
+ msprobe/mindspore/compare/utils.py,sha256=ZMJlIagUZKPv3HIhgCKG4MreRYFpsZtVMmZwUOOzOoI,1704
257
257
  msprobe/mindspore/debugger/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
258
258
  msprobe/mindspore/debugger/debugger_config.py,sha256=7DQ-g8KBOECCNXzsxvAqUU0ivCAsOcpduC6SUGs2JD0,8544
259
259
  msprobe/mindspore/debugger/precision_debugger.py,sha256=cTYWXfsZd4QbPvTqbSPkrWr7QSrMJvEgmHOz-W8fpQQ,10092
@@ -323,27 +323,28 @@ msprobe/nan_analyze/analyzer.py,sha256=EEMwQWm33JgXgvLudoMRkIYDQpQaqViFpl6BFMIrO
323
323
  msprobe/nan_analyze/graph.py,sha256=uSEYUC4I1KhGPUyi3E1BblNNpxUY3fjpf_sNnaaY7w4,8085
324
324
  msprobe/nan_analyze/utils.py,sha256=jwpnHcfluzBlytAcUuyAotj0Hw3zmi7e8ncf7rME2JU,7666
325
325
  msprobe/pytorch/__init__.py,sha256=qIvhnAk61oSpvPU0QI0YAC4zyLKYyOcfNzdXJxN7Klo,1035
326
+ msprobe/pytorch/attl_manager.py,sha256=Fe_vnjgmqZPpSHzEV7ayKweFjmBUuQV60MLSuw-fj2A,2706
326
327
  msprobe/pytorch/function_factory.py,sha256=Fi4w0zfO64Sd2IU9z45mBbDgS8k_CQEiZ9vRpC_TkVk,4031
327
- msprobe/pytorch/pt_config.py,sha256=xFge7ZvoObdvC3Bov45DgZQr4obEmmNVih1jQdQbEIY,11970
328
- msprobe/pytorch/pytorch_service.py,sha256=f_ItlE28Ruhr1GgMjHA4iqg1sAqRthi129gfsFZFVDQ,2590
328
+ msprobe/pytorch/pt_config.py,sha256=8ZUqCC3IPv831O1exVSZDqqTTreVahEQVCYYOxvdmSU,14799
329
+ msprobe/pytorch/pytorch_service.py,sha256=qtFeInAQrGSKSmf4ZMlgiR3u_ReupjNcfvix6t2nHOc,3107
329
330
  msprobe/pytorch/api_accuracy_checker/.keep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
330
331
  msprobe/pytorch/api_accuracy_checker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
331
- msprobe/pytorch/api_accuracy_checker/config.yaml,sha256=JDD7boERYCh7vd5x74ZcHXmS8afthdj6-pYGIqHinDY,67
332
+ msprobe/pytorch/api_accuracy_checker/config.yaml,sha256=2kktin5NoWkSp2MOGDoxlz3MPv0lYXXD5r5SPYArEYs,144
332
333
  msprobe/pytorch/api_accuracy_checker/common/.keep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
333
334
  msprobe/pytorch/api_accuracy_checker/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
334
- msprobe/pytorch/api_accuracy_checker/common/config.py,sha256=D4ZDCifjfjrpM-_IexazipoYS-DcBzFa3cK-r7anvKk,4070
335
+ msprobe/pytorch/api_accuracy_checker/common/config.py,sha256=7_JU9E2zT-ScgJyOTxBgGkZpQk1WB7skU_ihDcEU-J4,5400
335
336
  msprobe/pytorch/api_accuracy_checker/common/utils.py,sha256=DhAEgbMdon3NThmpZTAzpqJJmJBTNtA8c-C4ZfSPL3s,9016
336
337
  msprobe/pytorch/api_accuracy_checker/compare/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
337
338
  msprobe/pytorch/api_accuracy_checker/compare/algorithm.py,sha256=AoB1wXdOOXRqzOfpSR0Tr5_iKAF7O8zSNuIQ_ocATgo,10343
338
- msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py,sha256=5Mj9N0bkEpjWvYRtAO94E9ThHho7dxAF_ZnLvKbLxwQ,22752
339
+ msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py,sha256=F1S_ShUd0RLcjwfTA7BqRep9xOUO0zafbhxOJM4_VbQ,24030
339
340
  msprobe/pytorch/api_accuracy_checker/compare/api_precision_standard.yaml,sha256=k8J16s9to_7LTrTCj118JCdGXaMnTvnEnNsCHHtctN4,2047
340
341
  msprobe/pytorch/api_accuracy_checker/compare/api_precision_threshold.yaml,sha256=Pt5MQFazR6DwvI19SF8wUTHp2z_SLdoDz643AUdoWcY,8112
341
- msprobe/pytorch/api_accuracy_checker/compare/compare.py,sha256=MBB_ggZZOrowdYjOo8WGikqzgWx26v9HpR3shV-BGnI,21393
342
+ msprobe/pytorch/api_accuracy_checker/compare/compare.py,sha256=x_xM69LJNZfdO6AtafVEHDD0xetdAPnFQryC0XAwP-8,22125
342
343
  msprobe/pytorch/api_accuracy_checker/compare/compare_column.py,sha256=2bQBWg8qVoG3i-HhcUat310kivy7B2de4mQgJrRxWsk,7028
343
344
  msprobe/pytorch/api_accuracy_checker/compare/compare_input.py,sha256=OG83KTsDpc-NzUCjtMj5SnzuBmOHaQaJYPoyPaaxKfU,2059
344
345
  msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py,sha256=o64RKRmaMe-8E3qsASZJOxI_SBT15O9SDB_Lz59iW28,9817
345
346
  msprobe/pytorch/api_accuracy_checker/generate_op_script/config_op.json,sha256=qIwrmgYSnO08FFrfIPhTWwB5XssHfWWsoRFzKIQYHxc,195
346
- msprobe/pytorch/api_accuracy_checker/generate_op_script/op_generator.py,sha256=zD8QPEDEXmZOUfpU_5wpXd4f60mZg6grN4H7favRNBM,24049
347
+ msprobe/pytorch/api_accuracy_checker/generate_op_script/op_generator.py,sha256=FLN9gRL6vmWcaRv2pvLU6t_tZTS-oJivHtcsvNOhkhI,24050
347
348
  msprobe/pytorch/api_accuracy_checker/generate_op_script/operator_replication.template,sha256=Qbv1_maNROMVcJAZgwemw42TZPfgkLP5CAWmjBJDAlk,25286
348
349
  msprobe/pytorch/api_accuracy_checker/precision_standard/absolute_threshold.py,sha256=4vtifRSecvSeTPjX_GVLDH75_phtfbkbhXUh2Y5H4O8,5535
349
350
  msprobe/pytorch/api_accuracy_checker/precision_standard/accumulative_error_compare.py,sha256=e0qANZpdGFsiezICAkLnULA9GmP3kjdCMEq1Y1wTgxI,5662
@@ -359,9 +360,17 @@ msprobe/pytorch/api_accuracy_checker/run_ut/__init__.py,sha256=47DEQpj8HBSa-_TIm
359
360
  msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py,sha256=t8DbR13Sna4d8ovyVmrrMw0xMEI9gueEHexLPKwrv8g,17599
360
361
  msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py,sha256=vZUt-7zJ0RXSQrQFKMENFwi8Zhu8mUmHm1yifMIHLOY,10436
361
362
  msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py,sha256=cl9Avx5-_ihF4KPuofGrhPw6dzQqHJLbZslluVPC-LQ,8060
362
- msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py,sha256=4f4Vh1oXGDzYKcTDFnS-ID5jxa1_bgO7q3H-cSKeF6c,22983
363
+ msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py,sha256=shdWY9aILlygUXScXSswcShwj_0Zn6V676LDTbTZKOU,28313
363
364
  msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py,sha256=iZMGFpzQfK94Zxj7IJn0ldvcLHS5XVW63SCyK93doAo,11428
364
365
  msprobe/pytorch/api_accuracy_checker/run_ut/torch_ut_setting.json,sha256=6caXsuV9H9eDG2Zd0m_qJG_TsZ2Mk8z8RV8tS7r2m4o,92
366
+ msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
367
+ msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py,sha256=zkyT0xHTUxkMEkM-BEcs0DJqI4dcMz9zx-03jLoEfZs,7837
368
+ msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py,sha256=s8saRhYf6Cjetjw1JTKkmJnb3GDxxVL7cgbz-OL64yg,14876
369
+ msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py,sha256=L9YMol7bjXXe9MsaSEwWdmEVLtn27Y2l0PDgxB76XaE,10701
370
+ msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py,sha256=LllAmk0_PqUKDltUukKoZtcVMu9wuEqQKBW2VJyoA4I,3986
371
+ msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py,sha256=DqenaweA_kULje8J-qwiQEMadGGf4ZxlHw9I3gNhjD8,10384
372
+ msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/torch_ops_config.yaml,sha256=AAEAsbIK8R1O9c9kDb6svfeklqt2W8GHf19ewKpEROo,1057
373
+ msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/utils.py,sha256=lCK2rVmTXTTdickMoUR9VcP4QnNjWsWXuyYNRNuFEqo,7248
365
374
  msprobe/pytorch/bench_functions/__init__.py,sha256=BoY2ZOpo_mj2gQ8pcl8oLwFxfHpZidOflJ1aGYDmPrM,1157
366
375
  msprobe/pytorch/bench_functions/apply_adam.py,sha256=CNL19fQjwcMMDl1SHv3E_LKUoawkhz8_Kac5iXACsRc,6583
367
376
  msprobe/pytorch/bench_functions/apply_adam_w.py,sha256=tUkQEmmUf__kIkBTagtjphR7HaiZRpDjk_q7j6AGl00,1690
@@ -383,20 +392,20 @@ msprobe/pytorch/common/__init__.py,sha256=87H-l6y1gbEL2EQiSbPceeJwKJdcsS6zv-9Nmr
383
392
  msprobe/pytorch/common/compare_script.template,sha256=CsfmJ5IuPycAYa0-IfVPXrXVu-o6g8BeEF9v17UK_1w,286
384
393
  msprobe/pytorch/common/log.py,sha256=yGLpZ6mGfB-s9sHj_gcL5_PMq_Ai4BJxumA0F0knAso,1121
385
394
  msprobe/pytorch/common/parse_json.py,sha256=7HbrBdVCX8umYqa4u8JPHETAxUO1246jl7qhcE4FG1s,2286
386
- msprobe/pytorch/common/utils.py,sha256=KCjq5AbFoJKXH0JxTB8tCibhJbnv9A0riY7sMXC9k9I,17871
395
+ msprobe/pytorch/common/utils.py,sha256=VPIixcerXBg4GEPKFhDTcF0nPdzj806sRrPLmIGTvOk,18483
387
396
  msprobe/pytorch/compare/distributed_compare.py,sha256=uUMMB4cTsnxHUG8e8TyG5rcrqvip8-lo0DyMA0wB0zQ,916
388
397
  msprobe/pytorch/compare/mapping.yaml,sha256=Udlbf1dWDAdnTs8VcDTiSkqsMNj93o2qFUU2ocFgt2k,11697
389
398
  msprobe/pytorch/compare/match.py,sha256=j6ydfhs9zNMu6vzxsp_qMei-7-zDMuPO8Uj9i_OIhSA,1886
390
399
  msprobe/pytorch/compare/pt_compare.py,sha256=wUsLvx4lVPy3SbAeadO5Uw7kjDQhDJ7ZMx0UgBEFH38,2013
391
400
  msprobe/pytorch/compare/pt_diff_analyze.py,sha256=KG_7Diri1Nrl64FigVvAjMekSZ9SMWy3BqtqMpP75xE,893
392
- msprobe/pytorch/compare/utils.py,sha256=4F-CphKurzxAlMJezF-ELw5l-PFr8ZWPAB-htTc225g,1959
401
+ msprobe/pytorch/compare/utils.py,sha256=X_IsirhkQo1iH652WaJsnERsjYZaJ3DiP18RZr8EQ3c,1921
393
402
  msprobe/pytorch/debugger/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
394
- msprobe/pytorch/debugger/debugger_config.py,sha256=jP_xdwF1zV2YbvTrI8dB2PM1PQD0YGPOFSB9cIq_x1E,7993
403
+ msprobe/pytorch/debugger/debugger_config.py,sha256=9tdsLIKpRT3MjFX-EqN4Rvr-T0Fhbp1bvfhdDIgEVK4,8695
395
404
  msprobe/pytorch/debugger/precision_debugger.py,sha256=pKAFBu1MIgG4sM27unnRRJolyj6eFpgs1O4JLKxaUTo,6718
396
405
  msprobe/pytorch/dump/module_dump/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
397
- msprobe/pytorch/dump/module_dump/hook_wrapper.py,sha256=0L8c2gYv0V8KHiNttYZMpAhSSxFtCuDaR_yFdWMoChs,3746
406
+ msprobe/pytorch/dump/module_dump/hook_wrapper.py,sha256=gVu8COEsGDRitROcFHK5AgQrQ4MydHNkdwAptQ2PMq4,4619
398
407
  msprobe/pytorch/dump/module_dump/module_dump.py,sha256=yHKVkDaxayrbanUkxc8Wo7crHc1KNPQAXpLFpbcybjY,1767
399
- msprobe/pytorch/dump/module_dump/module_processer.py,sha256=N-FPVJGEDYJM-59fAqrc5NssIRwtQ2yiU_HFqrY6zvI,13756
408
+ msprobe/pytorch/dump/module_dump/module_processer.py,sha256=EdxNMJfonb7u6BwxpmhZjJ4okr4dFKqhtQWQjC0H7os,13849
400
409
  msprobe/pytorch/free_benchmark/__init__.py,sha256=N2xdPic80U-Rl1B_6c8j09EMuU5Xn0knJjgmKQ5uiXQ,904
401
410
  msprobe/pytorch/free_benchmark/main.py,sha256=n-Q55NNoI0o7jhyhAUri68Fq-kyBa9xNLh5az5zBbaY,4802
402
411
  msprobe/pytorch/free_benchmark/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -428,7 +437,7 @@ msprobe/pytorch/grad_probe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
428
437
  msprobe/pytorch/grad_probe/grad_monitor.py,sha256=xb3M1XguMxVsSgZuYwyB08tz_jno1-43cURs2pG1FYU,4910
429
438
  msprobe/pytorch/grad_probe/grad_stat_csv.py,sha256=k0YesT0Jy-2LsUlSYEDt8eRuhhfFhpSGv4ImtE74NH0,4940
430
439
  msprobe/pytorch/hook_module/__init__.py,sha256=ktS1W-6hBa1q8wF-DGz8kFsfVhiIPWBOj6C3kSatgno,679
431
- msprobe/pytorch/hook_module/api_register.py,sha256=PtTeEAMSoae5naegb6KjRG315b6_7UZc4gE8JiZlpSM,7037
440
+ msprobe/pytorch/hook_module/api_register.py,sha256=x3tmihtP3ghAG_ncWY8Ld882ncJXud13NpDZE1_XS-o,7142
432
441
  msprobe/pytorch/hook_module/hook_module.py,sha256=I1i_vISgVcLPd4uEYMMfr7Xfz76c3ASO0-XOcIoKxVA,3738
433
442
  msprobe/pytorch/hook_module/pt_hook_manager.py,sha256=x77ZU_YNYs5PB9snbopxSgyBmDNBEDQD5JeK1bcano4,5199
434
443
  msprobe/pytorch/hook_module/register_optimizer_hook.py,sha256=3JS0VqHAswQ7SlGXQ9gS9o4kGQYZflFAwOrJhUpB5gk,2539
@@ -470,9 +479,9 @@ msprobe/pytorch/parse_tool/lib/visualization.py,sha256=_M1R4kiyc3zdxyoHRqFoxMEfm
470
479
  msprobe/visualization/__init__.py,sha256=rEvJTAx-jTGHBM7-bB6VZ7fwfWwYA961M1eZJj4sRcY,622
471
480
  msprobe/visualization/db_utils.py,sha256=GhazcnqNQ27n-kpfxcuW8nK83_uC336S3-C0zpoNZ2I,9449
472
481
  msprobe/visualization/graph_service.py,sha256=_GfTbRtutN5Yk25cvV9kzD5kVWrgcHD5t6gkSbX5P5o,26302
473
- msprobe/visualization/utils.py,sha256=w40WEx16RrEh43h48KV2R1T48ydAZT08FNb6OUoB_3s,11899
482
+ msprobe/visualization/utils.py,sha256=P5Ds9uMRoMPRNft8XTSiqDrGAbEG7bQbIZD6qJy3Lvs,12620
474
483
  msprobe/visualization/builder/__init__.py,sha256=rEvJTAx-jTGHBM7-bB6VZ7fwfWwYA961M1eZJj4sRcY,622
475
- msprobe/visualization/builder/graph_builder.py,sha256=Sw3_weDDVJ5JT7Hs9YBNIH8xsGbIlbiLazl24IfZshk,20430
484
+ msprobe/visualization/builder/graph_builder.py,sha256=iTqXOhzfNz3uJAl3OrM3RvQhOgkTygzNZlaHpoKu3pI,20492
476
485
  msprobe/visualization/builder/graph_merger.py,sha256=RbNJ8yFPwJDmMwtTRfccskqSxBGixzOsh6wRQ-LK_40,50849
477
486
  msprobe/visualization/builder/msprobe_adapter.py,sha256=V-K90RqY02hmRJL_lxiyJjZIkwE-843hM1ruc6N2Wdc,9125
478
487
  msprobe/visualization/compare/__init__.py,sha256=vd0u6IEpGE1r-HC4CEhs3csiMaa0bxuFcHtyRlAX0kU,618
@@ -484,9 +493,9 @@ msprobe/visualization/graph/distributed_analyzer.py,sha256=IVPaA6Z3uZxuOob7fibgc
484
493
  msprobe/visualization/graph/graph.py,sha256=5AokyZE1nGZD_grRJZL8_gmpH3zkns9WOeHyuc1r75g,8607
485
494
  msprobe/visualization/graph/node_colors.py,sha256=7LpurTuE3edKGilwLVsXD7Ue4bMT7Maz63Udq_6CADM,4504
486
495
  msprobe/visualization/graph/node_op.py,sha256=qkbw3ZJkKGQH071C1CLIXi4kavvsLxBTzt0KXncz6fY,1368
487
- mindstudio_probe-8.2.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
488
- mindstudio_probe-8.2.1.dist-info/METADATA,sha256=mokGkctgy0FweYXTJ44Fs2qSncnlBKN7hod2hXFAzlY,1437
489
- mindstudio_probe-8.2.1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
490
- mindstudio_probe-8.2.1.dist-info/entry_points.txt,sha256=4ob3a9L018EBZFdlfgMW1lbgeIOhc4F-HCR8gBksaCQ,49
491
- mindstudio_probe-8.2.1.dist-info/top_level.txt,sha256=LxFEFqelENSyWmRtocCiEUF04IE8aZvwTl7ADB598Tk,8
492
- mindstudio_probe-8.2.1.dist-info/RECORD,,
496
+ mindstudio_probe-8.3.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
497
+ mindstudio_probe-8.3.0.dist-info/METADATA,sha256=fB6i8qNuYK0jfA7Dtc7hULPZTKrxXlsLUNuvjfZHoYs,1437
498
+ mindstudio_probe-8.3.0.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
499
+ mindstudio_probe-8.3.0.dist-info/entry_points.txt,sha256=4ob3a9L018EBZFdlfgMW1lbgeIOhc4F-HCR8gBksaCQ,49
500
+ mindstudio_probe-8.3.0.dist-info/top_level.txt,sha256=LxFEFqelENSyWmRtocCiEUF04IE8aZvwTl7ADB598Tk,8
501
+ mindstudio_probe-8.3.0.dist-info/RECORD,,
msprobe/README.md CHANGED
@@ -84,7 +84,9 @@ msprobe 通过在训练脚本中添加 PrecisionDebugger 接口的方式对 API
84
84
 
85
85
  精度预检旨在昇腾 NPU 上扫描训练模型中的所有 API 进行 API 复现,给出精度情况的诊断和分析。对应 config.json 中的 "run_ut" task。
86
86
 
87
- PyTorch 场景的[离线预检](./docs/07.accuracy_checker_PyTorch.md)
87
+ PyTorch 场景的[离线预检](./docs/07.accuracy_checker_PyTorch.md)和[在线预检](./docs/08.accuracy_checker_online_PyTorch.md)
88
+
89
+ 在线预检计划在 Mindstudio 8.3.0 版本正式下线。
88
90
 
89
91
  MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore.md)
90
92
 
@@ -12,29 +12,31 @@
12
12
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
13
  # See the License for the specific language governing permissions and
14
14
  # limitations under the License.
15
+
15
16
  import atexit
16
17
  import csv
17
18
  import fcntl
18
19
  import io
20
+ import json
21
+ import multiprocessing
19
22
  import os
20
23
  import pickle
21
- from multiprocessing import shared_memory
22
- import stat
23
- import json
24
24
  import re
25
25
  import shutil
26
+ import stat
26
27
  import sys
27
28
  import zipfile
28
- import multiprocessing
29
- import yaml
29
+ from multiprocessing import shared_memory
30
+
30
31
  import numpy as np
31
32
  import pandas as pd
33
+ import yaml
32
34
 
35
+ from msprobe.core.common.const import FileCheckConst, CompareConst, Const
33
36
  from msprobe.core.common.decorator import recursion_depth_decorator
34
- from msprobe.core.common.log import logger
35
37
  from msprobe.core.common.exceptions import FileCheckException
36
- from msprobe.core.common.const import FileCheckConst, CompareConst, Const
37
38
  from msprobe.core.common.global_lock import global_lock, is_main_process
39
+ from msprobe.core.common.log import logger
38
40
 
39
41
  proc_lock = multiprocessing.Lock()
40
42
 
@@ -46,16 +48,15 @@ class FileChecker:
46
48
  Attributes:
47
49
  file_path: The file or dictionary path to be verified.
48
50
  path_type: file or dictionary
49
- ability(str): FileCheckConst.WRITE_ABLE or FileCheckConst.READ_ABLE to set file has writability or readability
51
+ ability(str): one of [FileCheckConst.READ_ABLE, FileCheckConst.WRITE_ABLE, FileCheckConst.READ_WRITE_ABLE]
50
52
  file_type(str): The correct file type for file
51
53
  """
52
54
 
53
- def __init__(self, file_path, path_type, ability=None, file_type=None, is_script=True):
55
+ def __init__(self, file_path, path_type, ability=None, file_type=None):
54
56
  self.file_path = file_path
55
57
  self.path_type = self._check_path_type(path_type)
56
- self.ability = ability
58
+ self.ability = self._check_ability_type(ability)
57
59
  self.file_type = file_type
58
- self.is_script = is_script
59
60
 
60
61
  @staticmethod
61
62
  def _check_path_type(path_type):
@@ -64,9 +65,17 @@ class FileChecker:
64
65
  raise FileCheckException(FileCheckException.ILLEGAL_PARAM_ERROR)
65
66
  return path_type
66
67
 
68
+ @staticmethod
69
+ def _check_ability_type(ability):
70
+ ability_list = [FileCheckConst.READ_ABLE, FileCheckConst.WRITE_ABLE, FileCheckConst.READ_WRITE_ABLE]
71
+ if ability and ability not in ability_list:
72
+ logger.error(f'The ability must be one of {ability_list}.')
73
+ raise FileCheckException(FileCheckException.ILLEGAL_PARAM_ERROR)
74
+ return ability
75
+
67
76
  def common_check(self):
68
77
  """
69
- 功能:用户校验基本文件权限:软连接、文件长度、是否存在、读写权限、文件属组、文件特殊字符
78
+ 功能:基本文件权限校验,包括文件存在性、软连接、文件长度、文件类型、文件读写权限、文件属组、文件路径特殊字符、文件后缀等
70
79
  注意:文件后缀的合法性,非通用操作,可使用其他独立接口实现
71
80
  """
72
81
  check_path_exists(self.file_path)
@@ -75,13 +84,13 @@ class FileChecker:
75
84
  check_path_length(self.file_path)
76
85
  check_path_type(self.file_path, self.path_type)
77
86
  self.check_path_ability()
78
- if self.is_script:
79
- check_path_owner_consistent(self.file_path)
87
+ check_path_owner_consistent(self.file_path)
80
88
  check_path_pattern_valid(self.file_path)
81
89
  check_common_file_size(self.file_path)
82
90
  check_file_suffix(self.file_path, self.file_type)
91
+ check_path_no_others_write(self.file_path)
83
92
  if self.path_type == FileCheckConst.FILE:
84
- check_dirpath_before_read(self.file_path)
93
+ check_dirpath_permission(self.file_path)
85
94
  return self.file_path
86
95
 
87
96
  def check_path_ability(self):
@@ -137,7 +146,8 @@ class FileOpen:
137
146
  check_path_pattern_valid(self.file_path)
138
147
  if os.path.exists(self.file_path):
139
148
  check_common_file_size(self.file_path)
140
- check_dirpath_before_read(self.file_path)
149
+ check_path_no_others_write(self.file_path)
150
+ check_dirpath_permission(self.file_path)
141
151
 
142
152
  def check_ability_and_owner(self):
143
153
  if self.mode in self.SUPPORT_READ_MODE:
@@ -256,12 +266,15 @@ def check_path_type(file_path, file_type):
256
266
  raise FileCheckException(FileCheckException.INVALID_FILE_ERROR)
257
267
 
258
268
 
259
- def check_others_writable(directory):
260
- dir_stat = os.stat(directory)
261
- is_writable = (
262
- bool(dir_stat.st_mode & stat.S_IWGRP) or # 组可写
263
- bool(dir_stat.st_mode & stat.S_IWOTH) # 其他用户可写
264
- )
269
+ def check_group_writable(file_path):
270
+ path_stat = os.stat(file_path)
271
+ is_writable = bool(path_stat.st_mode & stat.S_IWGRP)
272
+ return is_writable
273
+
274
+
275
+ def check_others_writable(file_path):
276
+ path_stat = os.stat(file_path)
277
+ is_writable = bool(path_stat.st_mode & stat.S_IWOTH)
265
278
  return is_writable
266
279
 
267
280
 
@@ -309,7 +322,7 @@ def check_path_before_create(path):
309
322
  'The file path {} contains special characters.'.format(path))
310
323
 
311
324
 
312
- def check_dirpath_before_read(path):
325
+ def check_dirpath_permission(path):
313
326
  path = os.path.realpath(path)
314
327
  dirpath = os.path.dirname(path)
315
328
  if dedup_log('check_dirpath_before_read', dirpath):
@@ -321,13 +334,14 @@ def check_dirpath_before_read(path):
321
334
  logger.warning(f"The directory {dirpath} is not yours.")
322
335
 
323
336
 
324
- def check_file_or_directory_path(path, isdir=False):
337
+ def check_file_or_directory_path(path, isdir=False, is_strict=False):
325
338
  """
326
339
  Function Description:
327
340
  check whether the path is valid
328
341
  Parameter:
329
342
  path: the path to check
330
343
  isdir: the path is dir or file
344
+ is_strict: whether to perform stricter validation (e.g., verify group cannot write to path)
331
345
  Exception Description:
332
346
  when invalid data throw exception
333
347
  """
@@ -337,6 +351,33 @@ def check_file_or_directory_path(path, isdir=False):
337
351
  path_checker = FileChecker(path, FileCheckConst.FILE, FileCheckConst.READ_ABLE)
338
352
  path_checker.common_check()
339
353
 
354
+ if is_strict:
355
+ if check_group_writable(path):
356
+ raise FileCheckException(
357
+ FileCheckException.FILE_PERMISSION_ERROR,
358
+ f"The directory/file must not allow write access to group. Directory/File path: {path}"
359
+ )
360
+
361
+
362
+ def check_path_no_others_write(file_path):
363
+ if dedup_log('check_path_no_others_write', file_path):
364
+ if check_group_writable(file_path):
365
+ logger.warning(f"The directory/file path is writable by group: {file_path}.")
366
+
367
+ if check_others_writable(file_path):
368
+ raise FileCheckException(
369
+ FileCheckException.FILE_PERMISSION_ERROR,
370
+ f"The directory/file must not allow write access to others. Directory/File path: {file_path}"
371
+ )
372
+
373
+
374
+ def check_path_no_group_others_write(file_path):
375
+ if check_group_writable(file_path) or check_others_writable(file_path):
376
+ raise FileCheckException(
377
+ FileCheckException.FILE_PERMISSION_ERROR,
378
+ f"The directory/file must not allow write access to group or others. Directory/File path: {file_path}"
379
+ )
380
+
340
381
 
341
382
  def change_mode(path, mode):
342
383
  if not os.path.exists(path) or os.path.islink(path):
@@ -388,6 +429,14 @@ def check_file_type(path):
388
429
  raise FileCheckException(FileCheckException.INVALID_FILE_ERROR)
389
430
 
390
431
 
432
+ def root_privilege_warning():
433
+ if os.getuid() == 0:
434
+ logger.warning(
435
+ "msprobe is being run as root. "
436
+ "To avoid security risks, it is recommended to switch to a regular user to run it."
437
+ )
438
+
439
+
391
440
  def load_yaml(yaml_path):
392
441
  path_checker = FileChecker(yaml_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.YAML_SUFFIX)
393
442
  checked_path = path_checker.common_check()
@@ -968,7 +1017,13 @@ class SharedDict:
968
1017
  def _safe_load(self):
969
1018
  with io.BytesIO(self._shm.buf[:]) as buff:
970
1019
  try:
971
- self._dict = SafeUnpickler(buff).load()
1020
+ data = SafeUnpickler(buff).load()
1021
+ if not isinstance(data, dict):
1022
+ logger.debug(f"Data from shared memory is '{type(data)}' type, expected 'dict'.")
1023
+ self._dict = {}
1024
+ self._changed = True
1025
+ else:
1026
+ self._dict = data
972
1027
  except Exception as e:
973
1028
  logger.debug(f'shared dict is unreadable, reason: {e}, create new dict.')
974
1029
  self._dict = {}
@@ -12,10 +12,11 @@
12
12
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
13
  # See the License for the specific language governing permissions and
14
14
  # limitations under the License.import functools
15
+
15
16
  import functools
17
+
16
18
  from msprobe.core.common.const import Const
17
- from msprobe.core.common.file_utils import check_file_or_directory_path
18
- from msprobe.core.common.file_utils import save_npy
19
+ from msprobe.core.common.file_utils import check_file_or_directory_path, save_npy
19
20
 
20
21
 
21
22
  class FrameworkDescriptor:
@@ -103,7 +104,7 @@ class FmkAdp:
103
104
  @classmethod
104
105
  def tensor_norm(cls, tensor):
105
106
  return cls.process_tensor(tensor, lambda x: x.norm())
106
-
107
+
107
108
  @classmethod
108
109
  def save_tensor(cls, tensor, filepath):
109
110
  if cls.fmk == Const.PT_FRAMEWORK:
@@ -151,7 +152,7 @@ class FmkAdp:
151
152
 
152
153
  @classmethod
153
154
  def load_checkpoint(cls, path, to_cpu=True, weights_only=True):
154
- check_file_or_directory_path(path)
155
+ check_file_or_directory_path(path, is_strict=not weights_only)
155
156
  if cls.fmk == Const.PT_FRAMEWORK:
156
157
  try:
157
158
  if to_cpu:
@@ -161,9 +162,9 @@ class FmkAdp:
161
162
  except Exception as e:
162
163
  raise RuntimeError(f"load pt file {path} failed: {e}") from e
163
164
  return mindspore.load_checkpoint(path)
164
-
165
+
165
166
  @classmethod
166
167
  def asnumpy(cls, tensor):
167
168
  if cls.fmk == Const.PT_FRAMEWORK:
168
169
  return tensor.float().numpy()
169
- return tensor.float().asnumpy()
170
+ return tensor.float().asnumpy()
@@ -26,8 +26,6 @@ from msprobe.core.compare.utils import gen_api_batches
26
26
 
27
27
  cur_dir = os.path.dirname(os.path.realpath(__file__))
28
28
  diff_threshold_yaml_path = os.path.join(cur_dir, 'diff_analyze_threshold.yaml')
29
- ignore_op_list_yaml_path = os.path.join(cur_dir, 'ignore_op_list.yaml')
30
- ignore_list = load_yaml(ignore_op_list_yaml_path)
31
29
  thresholds = load_yaml(diff_threshold_yaml_path)
32
30
  cmp_metrics = thresholds.get('compare_metrics')
33
31
 
@@ -53,7 +51,7 @@ class FirstDiffAnalyze:
53
51
  return True
54
52
  return False
55
53
 
56
- def single_api_check(self, result_slice, header, api_name=None):
54
+ def single_api_check(self, result_slice, header):
57
55
  """
58
56
  单个api差异检查
59
57
 
@@ -67,18 +65,14 @@ class FirstDiffAnalyze:
67
65
  }
68
66
 
69
67
  column_indices = {name: idx for idx, name in enumerate(header)}
70
- output_idx = -1
68
+
71
69
  for line in result_slice:
72
70
  op_item = {
73
71
  column_name: line[column_indices[column_name]]
74
72
  for column_name in header
75
73
  }
76
74
  single_check_result['op_items'].append(op_item)
77
- if op_item['state'] != 'output':
78
- continue
79
- output_idx += 1
80
- if output_idx in ignore_list.get(api_name, []):
81
- continue
75
+
82
76
  # set is_same
83
77
  if self.mode_config.dump_mode == Const.MD5:
84
78
  if line[column_indices[CompareConst.RESULT]] == CompareConst.DIFF:
@@ -123,13 +117,7 @@ class FirstDiffAnalyze:
123
117
  with tqdm(total=len(api_batches), desc=bar_desc_add_rank, unit="api/module", ncols=100) as progress_bar:
124
118
  for api_batch in api_batches:
125
119
  result_slice = result[api_batch.start: api_batch.params_grad_end_index]
126
- api_compo = api_batch.api_name.split('.')
127
- # suppose name is Tensor.MatMul.0.forward
128
- if len(api_compo) < 4:
129
- continue
130
- # get MatMul as api_name
131
- api_name = api_compo[-3]
132
- check_result[api_batch.api_name] = self.single_api_check(result_slice, header, api_name)
120
+ check_result[api_batch.api_name] = self.single_api_check(result_slice, header)
133
121
  progress_bar.update(1)
134
122
 
135
123
  return check_result
@@ -182,7 +182,7 @@ def analyze_diff_in_group(nodes_group):
182
182
  input_diff_nodes = list(filter(lambda node: node.is_diff, src_list))
183
183
  # 如果有异常回溯计算节点找到异常来源
184
184
  # 使用cpu模拟节点进行计算,查看结果是否有问题。需要对所有计算节点录入/映射,暂不实现。
185
- get_compute_ops_from_comm_nodes(nodes_group)
185
+ get_compute_ops_from_comm_nodes(input_diff_nodes)
186
186
  # 筛选入参没问题但出参有问题的通信节点
187
187
  output_diff_nodes = list(filter(lambda node: node.data.is_diff, nodes_group))
188
188
  get_comm_ops(output_diff_nodes)
@@ -46,9 +46,10 @@ class BaseHookManager(ABC):
46
46
  hook_handle_dict = {}
47
47
  params_grad_info = {}
48
48
 
49
- def __init__(self, data_collector, config):
49
+ def __init__(self, data_collector, config, attl_manager=None):
50
50
  self.data_collector = data_collector
51
51
  self.config = config
52
+ self.attl_manager = attl_manager
52
53
 
53
54
  @property
54
55
  def _pid(self):
@@ -63,7 +64,6 @@ class BaseHookManager(ABC):
63
64
  def reset_status():
64
65
  BaseHookManager.inner_switch = defaultdict(bool)
65
66
  BaseHookManager.inner_api_count = defaultdict(int)
66
- BaseHookManager.hook_handle_dict.clear()
67
67
  BaseHookManager.params_grad_info.clear()
68
68
 
69
69
  @staticmethod
@@ -216,6 +216,10 @@ class BaseHookManager(ABC):
216
216
 
217
217
  args = self._register_backward_hook(module, full_backward_name, args)
218
218
  with self._no_grad_context():
219
+ if getattr(self.config, "online_run_ut", False):
220
+ BaseHookManager.inner_switch[tid] = False
221
+ ThreadSafe.release()
222
+ return
219
223
  self.data_collector.update_api_or_module_name(full_forward_name)
220
224
  self.data_collector.forward_input_data_collect(
221
225
  full_forward_name,
@@ -261,6 +265,13 @@ class BaseHookManager(ABC):
261
265
  output = self._register_backward_pre_hook(module, full_backward_name, output)
262
266
 
263
267
  with self._no_grad_context():
268
+ if getattr(self.config, "online_run_ut", False):
269
+ if self.data_collector.scope and not self.data_collector.scope.check(full_name):
270
+ return None
271
+ if self.attl_manager:
272
+ self.attl_manager.attl_send(full_name, args, kwargs, output)
273
+ BaseHookManager.inner_switch[tid] = False
274
+ return None
264
275
  if hook_type == Const.MODULE:
265
276
  params_dict = self._get_params_dict(module)
266
277
  setattr(module_input_output, Const.PARAMS, params_dict)
@@ -309,7 +320,9 @@ class BaseHookManager(ABC):
309
320
  original_state = self.ensure_gc_enabled()
310
321
  BaseHookManager.inner_switch[tid] = True
311
322
  self.data_collector.update_api_or_module_name(full_name)
312
-
323
+ if getattr(self.config, "online_run_ut", False):
324
+ BaseHookManager.inner_switch[tid] = False
325
+ return
313
326
  need_exchange = self._need_exchange(module) if hook_type == Const.MODULE else True
314
327
  if need_exchange:
315
328
  module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_output, grad_output=grad_input)