mindstudio-probe 8.3.0__py3-none-any.whl → 8.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mindstudio_probe-8.3.0.dist-info → mindstudio_probe-8.3.1.dist-info}/METADATA +1 -1
- {mindstudio_probe-8.3.0.dist-info → mindstudio_probe-8.3.1.dist-info}/RECORD +37 -47
- msprobe/README.md +8 -5
- msprobe/core/common/const.py +17 -3
- msprobe/core/common/file_utils.py +64 -13
- msprobe/core/common/framework_adapter.py +10 -1
- msprobe/core/common/utils.py +17 -0
- msprobe/core/config_check/ckpt_compare/ckpt_comparator.py +6 -1
- msprobe/core/hook_manager.py +2 -16
- msprobe/core/service.py +5 -16
- msprobe/docs/01.installation.md +2 -0
- msprobe/docs/02.config_introduction.md +0 -13
- msprobe/docs/14.data_parse_PyTorch.md +2 -0
- msprobe/docs/21.visualization_PyTorch.md +1 -1
- msprobe/docs/25.tool_function_introduction.md +0 -1
- msprobe/docs/32.ckpt_compare.md +5 -5
- msprobe/mindspore/monitor/module_hook.py +17 -20
- msprobe/pytorch/api_accuracy_checker/common/config.py +3 -36
- msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +0 -24
- msprobe/pytorch/api_accuracy_checker/compare/compare.py +2 -12
- msprobe/pytorch/api_accuracy_checker/config.yaml +1 -6
- msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py +34 -5
- msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +12 -132
- msprobe/pytorch/common/utils.py +0 -70
- msprobe/pytorch/debugger/debugger_config.py +0 -10
- msprobe/pytorch/dump/module_dump/module_processer.py +18 -3
- msprobe/pytorch/hook_module/api_register.py +5 -1
- msprobe/pytorch/monitor/module_hook.py +16 -34
- msprobe/pytorch/pt_config.py +2 -51
- msprobe/pytorch/pytorch_service.py +2 -11
- msprobe/visualization/builder/graph_builder.py +2 -2
- msprobe/visualization/builder/graph_merger.py +13 -0
- msprobe/visualization/graph/graph.py +13 -9
- msprobe/docs/08.accuracy_checker_online_PyTorch.md +0 -295
- msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/__init__.py +0 -0
- msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py +0 -205
- msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py +0 -378
- msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py +0 -239
- msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py +0 -115
- msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py +0 -250
- msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/torch_ops_config.yaml +0 -63
- msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/utils.py +0 -198
- msprobe/pytorch/attl_manager.py +0 -65
- {mindstudio_probe-8.3.0.dist-info → mindstudio_probe-8.3.1.dist-info}/LICENSE +0 -0
- {mindstudio_probe-8.3.0.dist-info → mindstudio_probe-8.3.1.dist-info}/WHEEL +0 -0
- {mindstudio_probe-8.3.0.dist-info → mindstudio_probe-8.3.1.dist-info}/entry_points.txt +0 -0
- {mindstudio_probe-8.3.0.dist-info → mindstudio_probe-8.3.1.dist-info}/top_level.txt +0 -0
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
msprobe/CMakeLists.txt,sha256=Cx_WsHVxbX2gWZ6Hxah2zYXz2eamSb4CgIlzKm9OQGA,125
|
|
2
|
-
msprobe/README.md,sha256=
|
|
2
|
+
msprobe/README.md,sha256=avO7tZ2lPArJMF12rlrZ0SYOD0PXR1x2eWkPE2RLcx4,9060
|
|
3
3
|
msprobe/__init__.py,sha256=q2S8b3VTSn1wTMrkf8Gm79OOq9EerMIK6Y26gFdu_0s,688
|
|
4
4
|
msprobe/config.json,sha256=QS37x2t0D-grehkpHLLxRAEZqOrkNe40GwwOaVMx3Yc,1061
|
|
5
5
|
msprobe/msprobe.py,sha256=KHFMop_Tm9hHKTd1c-_HaVZjkT8DQ6Kx6PejIkrClME,8228
|
|
6
6
|
msprobe/core/__init__.py,sha256=X_OkpIz42vnqhGy6WCeuEvYK4IFx4BwCT5bQj7tuuwI,757
|
|
7
7
|
msprobe/core/common_config.py,sha256=0kzhosc3FIE0_JfAnOYx8lYLbno7j0rmH0ST9On1jpQ,7182
|
|
8
|
-
msprobe/core/hook_manager.py,sha256=
|
|
9
|
-
msprobe/core/service.py,sha256=
|
|
8
|
+
msprobe/core/hook_manager.py,sha256=PyOZjK3UrwV-Tmn_3J_Ydjia83TasgrP3hWeEU9YL_E,13693
|
|
9
|
+
msprobe/core/service.py,sha256=FIYuVyHVVysFED5L3wPVZi8y8aT8ObiXtkfnhSnBGQo,13586
|
|
10
10
|
msprobe/core/advisor/advisor.py,sha256=WP8rBF2F9pcSphZONJDgNfz9c6GiRxbkWwPS9wRU4Tc,5900
|
|
11
11
|
msprobe/core/advisor/advisor_const.py,sha256=thEEVYKPimv9WGxxaC4vBXQ-Coq6yBlZ394T_b49Ibw,2624
|
|
12
12
|
msprobe/core/advisor/advisor_result.py,sha256=yM78n6NXLP1bjTsGeSddwb1bWQwtg1ZqjnPHmTzVxHE,2359
|
|
13
|
-
msprobe/core/common/const.py,sha256=
|
|
13
|
+
msprobe/core/common/const.py,sha256=2W9F588W134_ebIIIXpo7vup0TocUNJBELUH84Y2kKo,28099
|
|
14
14
|
msprobe/core/common/db_manager.py,sha256=OjC7Gr_4CKmrlLcODsLkeTv1dU-VU93fmLU0xMfiPzg,9088
|
|
15
15
|
msprobe/core/common/decorator.py,sha256=lj1upkAkGYeIV3tnH0bFqp7VDIKqTkFVaE5GUOFUNUE,1785
|
|
16
16
|
msprobe/core/common/exceptions.py,sha256=PJB4UTxT5VxyUE9aTCgwxiEExcuQZCcbu5V7tFdLVrY,4045
|
|
17
|
-
msprobe/core/common/file_utils.py,sha256=
|
|
18
|
-
msprobe/core/common/framework_adapter.py,sha256=
|
|
17
|
+
msprobe/core/common/file_utils.py,sha256=G3tY0xu5A4RM2skYJXdIH93F52mjjx76qoBKDp0h0_s,42180
|
|
18
|
+
msprobe/core/common/framework_adapter.py,sha256=86tGwDOvX_3UCwZhmfK3Y2EebTpSDpH9MbMkUG06__A,6842
|
|
19
19
|
msprobe/core/common/global_lock.py,sha256=MQedbRCzJTLAYGeNeeD1T7Vmn349r7ymGKt8d-U3m2Y,2721
|
|
20
20
|
msprobe/core/common/inplace_op_checker.py,sha256=gwxpddOtPS2dRGbI2JB5ZixgoPLCYyDICb_07N9XVaM,1864
|
|
21
21
|
msprobe/core/common/inplace_ops.yaml,sha256=Ta1zh6eF0A_Trd345gts2xlhFsU0-ZTXKEQOqUHT-54,3178
|
|
@@ -23,7 +23,7 @@ msprobe/core/common/log.py,sha256=VrecAUbJNfhm8tKtg-dGmLcOm3gM5XJth2tpeW6DsDo,33
|
|
|
23
23
|
msprobe/core/common/megatron_utils.py,sha256=BALTlzSdHYsQ83s9XhHKo9Ub6CQGWeOqGF4SphXEDps,1864
|
|
24
24
|
msprobe/core/common/parallel_state.py,sha256=WUfPtp4tfiV1E_oVkmQF2FZxO-sTgJtOsvDdSn4mdvY,8305
|
|
25
25
|
msprobe/core/common/runtime.py,sha256=KpPoWIvIWSGjD-xnldxtZQXcEB6Ip2qj1hVIcZG4tWg,844
|
|
26
|
-
msprobe/core/common/utils.py,sha256=
|
|
26
|
+
msprobe/core/common/utils.py,sha256=8WsZVFPSifCahQ_REEZl85up_vQflZF6Tg1hoKBdxhY,27794
|
|
27
27
|
msprobe/core/compare/acc_compare.py,sha256=M_Ie7Xtc8stbKNFVKZsLlp7R3i0RIu95vqzXYbX5sE0,38484
|
|
28
28
|
msprobe/core/compare/check.py,sha256=L4sq90oXdNfqiM7BzxsCXU6zD55K1-0zjFTDKwII9Y0,4384
|
|
29
29
|
msprobe/core/compare/compare_cli.py,sha256=B0WoZVAnWCKRAyRwDkYIKgIsTbKc4HA_8I8m2McO_OU,6940
|
|
@@ -59,7 +59,7 @@ msprobe/core/config_check/checkers/hyperparameter_checker.py,sha256=JIbEMTlfM-YN
|
|
|
59
59
|
msprobe/core/config_check/checkers/pip_checker.py,sha256=r7PMSjwCODypcxk_yEhjZy2ydvd_sj3YkP0KKwgZlSE,3441
|
|
60
60
|
msprobe/core/config_check/checkers/random_checker.py,sha256=hR6qJu18dtAcFvZokNcW6qzUxnaUz6U-XeCavyY0e-c,13158
|
|
61
61
|
msprobe/core/config_check/checkers/weights_checker.py,sha256=ObjwgiNyB4nLKPFIhL71kNmjekaWZAXJtvwoyQatsoU,5825
|
|
62
|
-
msprobe/core/config_check/ckpt_compare/ckpt_comparator.py,sha256=
|
|
62
|
+
msprobe/core/config_check/ckpt_compare/ckpt_comparator.py,sha256=NxzmK4KT836tUZX91xs6kKXmjoHarUuI4J-MqgbtnWM,3511
|
|
63
63
|
msprobe/core/config_check/ckpt_compare/megatron_loader.py,sha256=0HUEGsQQBBl-hI2CCZsPU7F-tqDwkG2miTjLYPA4zqo,12057
|
|
64
64
|
msprobe/core/config_check/ckpt_compare/metrics.py,sha256=EDRLEFA_difzZWAfD-Y9-0qlk3Veu0IyKrpixrVCamo,2010
|
|
65
65
|
msprobe/core/config_check/ckpt_compare/name_mapping.yaml,sha256=dC6SX0qYcOfgAw1wRhic_MxQFC5YNkogFajNszDIHUk,379
|
|
@@ -97,31 +97,30 @@ msprobe/core/overflow_check/utils.py,sha256=1j5p2qWBOmOoat1v_Zi8krLOWz2NMCdUjrnD
|
|
|
97
97
|
msprobe/core/single_save/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
98
98
|
msprobe/core/single_save/single_comparator.py,sha256=faYvqjvZNpA0GrtpI5pXTYOlvV8x5toKFdKVv5Nzk60,10827
|
|
99
99
|
msprobe/core/single_save/single_saver.py,sha256=IVS7fsG5CWmZNCeUenNwtVxhAmpIWSYHK7az2NpZ678,5562
|
|
100
|
-
msprobe/docs/01.installation.md,sha256=
|
|
101
|
-
msprobe/docs/02.config_introduction.md,sha256=
|
|
100
|
+
msprobe/docs/01.installation.md,sha256=I4iA214liJglmn7T6moGlipnWNy2y1aKNWAoHsU0BFE,11369
|
|
101
|
+
msprobe/docs/02.config_introduction.md,sha256=EVzpOM38ZsFde3PTQkR2LYepix-lpLbYIr9uKZuBNtU,32272
|
|
102
102
|
msprobe/docs/03.config_examples.md,sha256=lHv_HLaCxk1nlSPIUF7wQVO0RR6_Q5b9XrSp96fr1FE,4408
|
|
103
103
|
msprobe/docs/04.kernel_dump_PyTorch.md,sha256=1qXNBrdkO2r8FJjHObr1r2XGy7gIzhvZ3DWC0JjDPng,3034
|
|
104
104
|
msprobe/docs/05.data_dump_PyTorch.md,sha256=0ZLkESAFa_HBlo8nndc7cWXQ0O1l7BuNCCy7eARTcm8,22242
|
|
105
105
|
msprobe/docs/06.data_dump_MindSpore.md,sha256=pweNRLRxNy1cu8h5cgmEf9auOBuKN3-Ezen5Et7a1s8,29284
|
|
106
106
|
msprobe/docs/07.accuracy_checker_PyTorch.md,sha256=tLHA1eD0ldNv_4uQKArN-HzKcPdQbQighhK9VKuhuxQ,30993
|
|
107
|
-
msprobe/docs/08.accuracy_checker_online_PyTorch.md,sha256=u3kDOo3Y1YMSu2rd2QiIToYxqtpi4epfKk_ngdSc_Sw,14717
|
|
108
107
|
msprobe/docs/09.accuracy_checker_MindSpore.md,sha256=XKEnEgqPJKZxL7JL3evZeIAa_LbMh8DR7sILqJ1camw,11320
|
|
109
108
|
msprobe/docs/10.accuracy_compare_PyTorch.md,sha256=3H_fHJBMRweqAxEOT6trHSQPAMjP3xD9G3T_PIJPtVY,41611
|
|
110
109
|
msprobe/docs/11.accuracy_compare_MindSpore.md,sha256=9zalk8EiYA_ex5kkiGEIn8yaNygLmJPplUoUEPs-C5w,40792
|
|
111
110
|
msprobe/docs/12.overflow_check_PyTorch.md,sha256=VE76J_rFAFm0pw_bQEnKIV4FdXek4x7cenxTQQz2FK0,3793
|
|
112
111
|
msprobe/docs/13.overflow_check_MindSpore.md,sha256=G67GjHo2VERAKsr1_uX0m67eNuG8E9aN6tmZiO1c_zg,2799
|
|
113
|
-
msprobe/docs/14.data_parse_PyTorch.md,sha256=
|
|
112
|
+
msprobe/docs/14.data_parse_PyTorch.md,sha256=YWFj32k4L0n6D1-j9c-COvcUEczn-a-TxV4DJ7n2n_A,19394
|
|
114
113
|
msprobe/docs/15.free_benchmarking_PyTorch.md,sha256=6pIpLXx5lUu45_kdL31rhj9zV3Ew8X-yyqZXrJ_gZPk,9888
|
|
115
114
|
msprobe/docs/16.free_benchmarking_MindSpore.md,sha256=swCOrnBSzU6Q5I0AHVwi2r0JfKp3VE1DgNHripyE01M,8195
|
|
116
115
|
msprobe/docs/17.grad_probe.md,sha256=9g1aq6FettgpvzBxKj5C5W8bTsENPtt2itYaNOLcBk8,9624
|
|
117
116
|
msprobe/docs/18.online_dispatch.md,sha256=Ae9ONIXF3wA2u0tikuDxV0nea_n5TIGug-PKTI9c7Ws,4170
|
|
118
117
|
msprobe/docs/19.monitor.md,sha256=ogaecwFA5vpEyEEKuXaWmbaoClnPdtRedYhntppTAYo,57192
|
|
119
118
|
msprobe/docs/20.monitor_performance_baseline.md,sha256=t-aM1s7BqEE8ls47gL1wmipkhdmB6CSPI90-r4abyXY,3672
|
|
120
|
-
msprobe/docs/21.visualization_PyTorch.md,sha256=
|
|
119
|
+
msprobe/docs/21.visualization_PyTorch.md,sha256=lEt72aqsT5voRk38vAapXSJA_nzbu3DWq3-mEYKZowU,27678
|
|
121
120
|
msprobe/docs/22.visualization_MindSpore.md,sha256=jtMgasjacyNEevhmrMT5Y8IOsAgEuDzBN7vvZYBJl7s,26605
|
|
122
121
|
msprobe/docs/23.generate_operator_PyTorch.md,sha256=Tu_Jik6zWBIdf2ePOBeUAKnPMFnwcNdw1cuZg-Jfxbc,10289
|
|
123
122
|
msprobe/docs/24.code_mapping_Mindspore.md,sha256=pFGY7uvNnaBqUDLQBDEay0In04nrpP2mFCzYfYYQEcs,2068
|
|
124
|
-
msprobe/docs/25.tool_function_introduction.md,sha256=
|
|
123
|
+
msprobe/docs/25.tool_function_introduction.md,sha256=tqXbsW1xTbVBgke5bDd0B5NFxVGXmyqQkr63b9AtdFY,15668
|
|
125
124
|
msprobe/docs/26.data_dump_PyTorch_baseline.md,sha256=Di-qxaT7TckS4ggAJOk3R99DGlJ67i9Vw7Jl9fIoJNA,2551
|
|
126
125
|
msprobe/docs/27.dump_json_instruction.md,sha256=YW6q4U0ccoFwax_fZ5Gqr1CQo5PFlKaxr9S-4ocycZs,21025
|
|
127
126
|
msprobe/docs/28.debugger_save_instruction.md,sha256=pxvLlc2GmR8tmapZ8IKZyIFGwt1tEKXU4jy8Ao4MwrI,10173
|
|
@@ -129,7 +128,7 @@ msprobe/docs/28.kernel_dump_MindSpore.md,sha256=USVxSPdhuaiw6dlWGu02GVNMXgaryHzy
|
|
|
129
128
|
msprobe/docs/29.data_dump_MSAdapter.md,sha256=lWZaqR4ClAGW-YUj5jKZlJJqgOz2iIXgro3gq95VvMw,11615
|
|
130
129
|
msprobe/docs/30.overflow_check_MSAdapter.md,sha256=leiJqnUR3L93K3-kDDeEPTcG5UeIl-Gjbq9GCzFjXNw,2208
|
|
131
130
|
msprobe/docs/31.config_check.md,sha256=HsqOL1DHmnM-fR-QlrPVF0N2E5Aey9sFlLBTCG-1Us4,4171
|
|
132
|
-
msprobe/docs/32.ckpt_compare.md,sha256=
|
|
131
|
+
msprobe/docs/32.ckpt_compare.md,sha256=bne2nbNuX5Mz5UGVxOcy0VKHE-7PpIDk5qgGFstcZ8I,2956
|
|
133
132
|
msprobe/docs/33.generate_operator_MindSpore.md,sha256=Q95U1T1g1p3OGDaFHKIA6Xb7h986SBrugKy6jaCQmjU,10005
|
|
134
133
|
msprobe/docs/34.RL_collect.md,sha256=fC1nDBAFjEBnMbDOYvzujQjvpRCZK2wrRhl9odKInQ8,4430
|
|
135
134
|
msprobe/docs/35.nan_analyze.md,sha256=YaY0v_n64JfSIdUVk_MYEJgPJukGBlWUV5qE1TLC9-w,3301
|
|
@@ -308,7 +307,7 @@ msprobe/mindspore/mindtorch/mindtorch_adaptor.py,sha256=M_1uX4Yc29Owz46diRsUbtRI
|
|
|
308
307
|
msprobe/mindspore/monitor/common_func.py,sha256=L-t_w1nl---0T4W80Rtp0YijNx1GwFHkuhg9IV0wpbw,1549
|
|
309
308
|
msprobe/mindspore/monitor/data_writers.py,sha256=xBj_kv1MpFH3DpB1mdl39vH-1_M81uz4rMJiz4ptKkw,8782
|
|
310
309
|
msprobe/mindspore/monitor/features.py,sha256=l7PSxsEy7fwJJkzYW68CcUvtiW1Q4mq9N2j2d9pEAnw,4510
|
|
311
|
-
msprobe/mindspore/monitor/module_hook.py,sha256=
|
|
310
|
+
msprobe/mindspore/monitor/module_hook.py,sha256=vFBztQ0IgzQ6UZCouWTGbaoZdfNmKG09fuBWYxpbo-Y,48081
|
|
312
311
|
msprobe/mindspore/monitor/optimizer_collect.py,sha256=B_WsSnuX5LDAY2LTwuwwC8mMtVcR3X1iautlNzaxCSE,14419
|
|
313
312
|
msprobe/mindspore/monitor/utils.py,sha256=Rnb0BOf1mFdhNfIkAL1cG0IzCsznaHYLBGInZbVFjXQ,3412
|
|
314
313
|
msprobe/mindspore/monitor/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -323,23 +322,22 @@ msprobe/nan_analyze/analyzer.py,sha256=EEMwQWm33JgXgvLudoMRkIYDQpQaqViFpl6BFMIrO
|
|
|
323
322
|
msprobe/nan_analyze/graph.py,sha256=uSEYUC4I1KhGPUyi3E1BblNNpxUY3fjpf_sNnaaY7w4,8085
|
|
324
323
|
msprobe/nan_analyze/utils.py,sha256=jwpnHcfluzBlytAcUuyAotj0Hw3zmi7e8ncf7rME2JU,7666
|
|
325
324
|
msprobe/pytorch/__init__.py,sha256=qIvhnAk61oSpvPU0QI0YAC4zyLKYyOcfNzdXJxN7Klo,1035
|
|
326
|
-
msprobe/pytorch/attl_manager.py,sha256=Fe_vnjgmqZPpSHzEV7ayKweFjmBUuQV60MLSuw-fj2A,2706
|
|
327
325
|
msprobe/pytorch/function_factory.py,sha256=Fi4w0zfO64Sd2IU9z45mBbDgS8k_CQEiZ9vRpC_TkVk,4031
|
|
328
|
-
msprobe/pytorch/pt_config.py,sha256=
|
|
329
|
-
msprobe/pytorch/pytorch_service.py,sha256=
|
|
326
|
+
msprobe/pytorch/pt_config.py,sha256=1A6RhK6BBdMxXdYfLIcvtkOFUSXEaRMuNvaK6stjl7o,12284
|
|
327
|
+
msprobe/pytorch/pytorch_service.py,sha256=f_ItlE28Ruhr1GgMjHA4iqg1sAqRthi129gfsFZFVDQ,2590
|
|
330
328
|
msprobe/pytorch/api_accuracy_checker/.keep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
331
329
|
msprobe/pytorch/api_accuracy_checker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
332
|
-
msprobe/pytorch/api_accuracy_checker/config.yaml,sha256=
|
|
330
|
+
msprobe/pytorch/api_accuracy_checker/config.yaml,sha256=JDD7boERYCh7vd5x74ZcHXmS8afthdj6-pYGIqHinDY,67
|
|
333
331
|
msprobe/pytorch/api_accuracy_checker/common/.keep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
334
332
|
msprobe/pytorch/api_accuracy_checker/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
335
|
-
msprobe/pytorch/api_accuracy_checker/common/config.py,sha256=
|
|
333
|
+
msprobe/pytorch/api_accuracy_checker/common/config.py,sha256=D4ZDCifjfjrpM-_IexazipoYS-DcBzFa3cK-r7anvKk,4070
|
|
336
334
|
msprobe/pytorch/api_accuracy_checker/common/utils.py,sha256=DhAEgbMdon3NThmpZTAzpqJJmJBTNtA8c-C4ZfSPL3s,9016
|
|
337
335
|
msprobe/pytorch/api_accuracy_checker/compare/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
338
336
|
msprobe/pytorch/api_accuracy_checker/compare/algorithm.py,sha256=AoB1wXdOOXRqzOfpSR0Tr5_iKAF7O8zSNuIQ_ocATgo,10343
|
|
339
|
-
msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py,sha256=
|
|
337
|
+
msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py,sha256=5Mj9N0bkEpjWvYRtAO94E9ThHho7dxAF_ZnLvKbLxwQ,22752
|
|
340
338
|
msprobe/pytorch/api_accuracy_checker/compare/api_precision_standard.yaml,sha256=k8J16s9to_7LTrTCj118JCdGXaMnTvnEnNsCHHtctN4,2047
|
|
341
339
|
msprobe/pytorch/api_accuracy_checker/compare/api_precision_threshold.yaml,sha256=Pt5MQFazR6DwvI19SF8wUTHp2z_SLdoDz643AUdoWcY,8112
|
|
342
|
-
msprobe/pytorch/api_accuracy_checker/compare/compare.py,sha256=
|
|
340
|
+
msprobe/pytorch/api_accuracy_checker/compare/compare.py,sha256=MBB_ggZZOrowdYjOo8WGikqzgWx26v9HpR3shV-BGnI,21393
|
|
343
341
|
msprobe/pytorch/api_accuracy_checker/compare/compare_column.py,sha256=2bQBWg8qVoG3i-HhcUat310kivy7B2de4mQgJrRxWsk,7028
|
|
344
342
|
msprobe/pytorch/api_accuracy_checker/compare/compare_input.py,sha256=OG83KTsDpc-NzUCjtMj5SnzuBmOHaQaJYPoyPaaxKfU,2059
|
|
345
343
|
msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py,sha256=o64RKRmaMe-8E3qsASZJOxI_SBT15O9SDB_Lz59iW28,9817
|
|
@@ -358,19 +356,11 @@ msprobe/pytorch/api_accuracy_checker/precision_standard/ulp_compare.py,sha256=pE
|
|
|
358
356
|
msprobe/pytorch/api_accuracy_checker/run_ut/.keep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
359
357
|
msprobe/pytorch/api_accuracy_checker/run_ut/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
360
358
|
msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py,sha256=t8DbR13Sna4d8ovyVmrrMw0xMEI9gueEHexLPKwrv8g,17599
|
|
361
|
-
msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py,sha256=
|
|
359
|
+
msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py,sha256=kCssWavWUyZS7n4-mHK2JvJTng4xWUM4_LnzmOFqou8,12169
|
|
362
360
|
msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py,sha256=cl9Avx5-_ihF4KPuofGrhPw6dzQqHJLbZslluVPC-LQ,8060
|
|
363
|
-
msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py,sha256=
|
|
361
|
+
msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py,sha256=4f4Vh1oXGDzYKcTDFnS-ID5jxa1_bgO7q3H-cSKeF6c,22983
|
|
364
362
|
msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py,sha256=iZMGFpzQfK94Zxj7IJn0ldvcLHS5XVW63SCyK93doAo,11428
|
|
365
363
|
msprobe/pytorch/api_accuracy_checker/run_ut/torch_ut_setting.json,sha256=6caXsuV9H9eDG2Zd0m_qJG_TsZ2Mk8z8RV8tS7r2m4o,92
|
|
366
|
-
msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
367
|
-
msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py,sha256=zkyT0xHTUxkMEkM-BEcs0DJqI4dcMz9zx-03jLoEfZs,7837
|
|
368
|
-
msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py,sha256=s8saRhYf6Cjetjw1JTKkmJnb3GDxxVL7cgbz-OL64yg,14876
|
|
369
|
-
msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py,sha256=L9YMol7bjXXe9MsaSEwWdmEVLtn27Y2l0PDgxB76XaE,10701
|
|
370
|
-
msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py,sha256=LllAmk0_PqUKDltUukKoZtcVMu9wuEqQKBW2VJyoA4I,3986
|
|
371
|
-
msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py,sha256=DqenaweA_kULje8J-qwiQEMadGGf4ZxlHw9I3gNhjD8,10384
|
|
372
|
-
msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/torch_ops_config.yaml,sha256=AAEAsbIK8R1O9c9kDb6svfeklqt2W8GHf19ewKpEROo,1057
|
|
373
|
-
msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/utils.py,sha256=lCK2rVmTXTTdickMoUR9VcP4QnNjWsWXuyYNRNuFEqo,7248
|
|
374
364
|
msprobe/pytorch/bench_functions/__init__.py,sha256=BoY2ZOpo_mj2gQ8pcl8oLwFxfHpZidOflJ1aGYDmPrM,1157
|
|
375
365
|
msprobe/pytorch/bench_functions/apply_adam.py,sha256=CNL19fQjwcMMDl1SHv3E_LKUoawkhz8_Kac5iXACsRc,6583
|
|
376
366
|
msprobe/pytorch/bench_functions/apply_adam_w.py,sha256=tUkQEmmUf__kIkBTagtjphR7HaiZRpDjk_q7j6AGl00,1690
|
|
@@ -392,7 +382,7 @@ msprobe/pytorch/common/__init__.py,sha256=87H-l6y1gbEL2EQiSbPceeJwKJdcsS6zv-9Nmr
|
|
|
392
382
|
msprobe/pytorch/common/compare_script.template,sha256=CsfmJ5IuPycAYa0-IfVPXrXVu-o6g8BeEF9v17UK_1w,286
|
|
393
383
|
msprobe/pytorch/common/log.py,sha256=yGLpZ6mGfB-s9sHj_gcL5_PMq_Ai4BJxumA0F0knAso,1121
|
|
394
384
|
msprobe/pytorch/common/parse_json.py,sha256=7HbrBdVCX8umYqa4u8JPHETAxUO1246jl7qhcE4FG1s,2286
|
|
395
|
-
msprobe/pytorch/common/utils.py,sha256=
|
|
385
|
+
msprobe/pytorch/common/utils.py,sha256=3PShpmPAfGP4G-_CAPsLL0_XA6QMDeTwEUkkH14AKSE,16082
|
|
396
386
|
msprobe/pytorch/compare/distributed_compare.py,sha256=uUMMB4cTsnxHUG8e8TyG5rcrqvip8-lo0DyMA0wB0zQ,916
|
|
397
387
|
msprobe/pytorch/compare/mapping.yaml,sha256=Udlbf1dWDAdnTs8VcDTiSkqsMNj93o2qFUU2ocFgt2k,11697
|
|
398
388
|
msprobe/pytorch/compare/match.py,sha256=j6ydfhs9zNMu6vzxsp_qMei-7-zDMuPO8Uj9i_OIhSA,1886
|
|
@@ -400,12 +390,12 @@ msprobe/pytorch/compare/pt_compare.py,sha256=wUsLvx4lVPy3SbAeadO5Uw7kjDQhDJ7ZMx0
|
|
|
400
390
|
msprobe/pytorch/compare/pt_diff_analyze.py,sha256=KG_7Diri1Nrl64FigVvAjMekSZ9SMWy3BqtqMpP75xE,893
|
|
401
391
|
msprobe/pytorch/compare/utils.py,sha256=X_IsirhkQo1iH652WaJsnERsjYZaJ3DiP18RZr8EQ3c,1921
|
|
402
392
|
msprobe/pytorch/debugger/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
403
|
-
msprobe/pytorch/debugger/debugger_config.py,sha256=
|
|
393
|
+
msprobe/pytorch/debugger/debugger_config.py,sha256=jP_xdwF1zV2YbvTrI8dB2PM1PQD0YGPOFSB9cIq_x1E,7993
|
|
404
394
|
msprobe/pytorch/debugger/precision_debugger.py,sha256=pKAFBu1MIgG4sM27unnRRJolyj6eFpgs1O4JLKxaUTo,6718
|
|
405
395
|
msprobe/pytorch/dump/module_dump/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
406
396
|
msprobe/pytorch/dump/module_dump/hook_wrapper.py,sha256=gVu8COEsGDRitROcFHK5AgQrQ4MydHNkdwAptQ2PMq4,4619
|
|
407
397
|
msprobe/pytorch/dump/module_dump/module_dump.py,sha256=yHKVkDaxayrbanUkxc8Wo7crHc1KNPQAXpLFpbcybjY,1767
|
|
408
|
-
msprobe/pytorch/dump/module_dump/module_processer.py,sha256=
|
|
398
|
+
msprobe/pytorch/dump/module_dump/module_processer.py,sha256=i_Mevm4zlo48KB8sJtG1CVIhQMcpnuADoWB1ndsDTo4,14512
|
|
409
399
|
msprobe/pytorch/free_benchmark/__init__.py,sha256=N2xdPic80U-Rl1B_6c8j09EMuU5Xn0knJjgmKQ5uiXQ,904
|
|
410
400
|
msprobe/pytorch/free_benchmark/main.py,sha256=n-Q55NNoI0o7jhyhAUri68Fq-kyBa9xNLh5az5zBbaY,4802
|
|
411
401
|
msprobe/pytorch/free_benchmark/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -437,7 +427,7 @@ msprobe/pytorch/grad_probe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
|
|
|
437
427
|
msprobe/pytorch/grad_probe/grad_monitor.py,sha256=xb3M1XguMxVsSgZuYwyB08tz_jno1-43cURs2pG1FYU,4910
|
|
438
428
|
msprobe/pytorch/grad_probe/grad_stat_csv.py,sha256=k0YesT0Jy-2LsUlSYEDt8eRuhhfFhpSGv4ImtE74NH0,4940
|
|
439
429
|
msprobe/pytorch/hook_module/__init__.py,sha256=ktS1W-6hBa1q8wF-DGz8kFsfVhiIPWBOj6C3kSatgno,679
|
|
440
|
-
msprobe/pytorch/hook_module/api_register.py,sha256=
|
|
430
|
+
msprobe/pytorch/hook_module/api_register.py,sha256=Vfdh-t0zBWinbzuXFqZ4WCSTTKexOFpaGAuEz2gUMgU,7313
|
|
441
431
|
msprobe/pytorch/hook_module/hook_module.py,sha256=I1i_vISgVcLPd4uEYMMfr7Xfz76c3ASO0-XOcIoKxVA,3738
|
|
442
432
|
msprobe/pytorch/hook_module/pt_hook_manager.py,sha256=x77ZU_YNYs5PB9snbopxSgyBmDNBEDQD5JeK1bcano4,5199
|
|
443
433
|
msprobe/pytorch/hook_module/register_optimizer_hook.py,sha256=3JS0VqHAswQ7SlGXQ9gS9o4kGQYZflFAwOrJhUpB5gk,2539
|
|
@@ -449,7 +439,7 @@ msprobe/pytorch/monitor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG
|
|
|
449
439
|
msprobe/pytorch/monitor/csv2tb.py,sha256=OCsOtZ6G423Gu-WicF0eWijGeJ6TFY9Zefe-3SlAVI0,6557
|
|
450
440
|
msprobe/pytorch/monitor/data_writers.py,sha256=ieJTuN7hSzusUqq-Pi251Xwtq1B6iwnIkmpuNIDq41w,9713
|
|
451
441
|
msprobe/pytorch/monitor/features.py,sha256=BShZU-f5oQLc4zIfoRax_JHVMv79-hN18RS1at40IyY,6281
|
|
452
|
-
msprobe/pytorch/monitor/module_hook.py,sha256=
|
|
442
|
+
msprobe/pytorch/monitor/module_hook.py,sha256=MuBB6JgZrgAuc_-v-t09bMN2BmzJXIoT6n6uNSSWlOE,59227
|
|
453
443
|
msprobe/pytorch/monitor/module_metric.py,sha256=khxcepVffT0B7pvI0c80NCOFMwxX23tLimElyJ3Rls0,6328
|
|
454
444
|
msprobe/pytorch/monitor/optimizer_collect.py,sha256=-ai9XAXP4gTsuTQ3cecVZySWS8Zb3PABuxcJSm9XGPw,19947
|
|
455
445
|
msprobe/pytorch/monitor/utils.py,sha256=uBSJfpYRbi3YJRy0fHyIoY3bAnmIRF2kErdvZ1R2HSo,1641
|
|
@@ -481,8 +471,8 @@ msprobe/visualization/db_utils.py,sha256=GhazcnqNQ27n-kpfxcuW8nK83_uC336S3-C0zpo
|
|
|
481
471
|
msprobe/visualization/graph_service.py,sha256=_GfTbRtutN5Yk25cvV9kzD5kVWrgcHD5t6gkSbX5P5o,26302
|
|
482
472
|
msprobe/visualization/utils.py,sha256=P5Ds9uMRoMPRNft8XTSiqDrGAbEG7bQbIZD6qJy3Lvs,12620
|
|
483
473
|
msprobe/visualization/builder/__init__.py,sha256=rEvJTAx-jTGHBM7-bB6VZ7fwfWwYA961M1eZJj4sRcY,622
|
|
484
|
-
msprobe/visualization/builder/graph_builder.py,sha256=
|
|
485
|
-
msprobe/visualization/builder/graph_merger.py,sha256=
|
|
474
|
+
msprobe/visualization/builder/graph_builder.py,sha256=yL7rHq4F94zWYBYJCyQvY3gLwzhLWnV6oLsVfUZ8AZo,20535
|
|
475
|
+
msprobe/visualization/builder/graph_merger.py,sha256=Zpk0Kg54Ad2dKZqwxN5wE0A1MCdgoX31O9WIRv0Z9UU,51901
|
|
486
476
|
msprobe/visualization/builder/msprobe_adapter.py,sha256=V-K90RqY02hmRJL_lxiyJjZIkwE-843hM1ruc6N2Wdc,9125
|
|
487
477
|
msprobe/visualization/compare/__init__.py,sha256=vd0u6IEpGE1r-HC4CEhs3csiMaa0bxuFcHtyRlAX0kU,618
|
|
488
478
|
msprobe/visualization/compare/graph_comparator.py,sha256=v1YbN5oIqcmnZZ7oZBJEH_EXen1Y9bvh9Itul7erb8o,10282
|
|
@@ -490,12 +480,12 @@ msprobe/visualization/compare/mode_adapter.py,sha256=d7yjHzL1rwAg0E8RJRU0B0s7S90
|
|
|
490
480
|
msprobe/visualization/graph/__init__.py,sha256=rEvJTAx-jTGHBM7-bB6VZ7fwfWwYA961M1eZJj4sRcY,622
|
|
491
481
|
msprobe/visualization/graph/base_node.py,sha256=tai40AZ_0PXkpPgEQbnTAcf8K0XSMWfKkrnJF7VMrCc,3725
|
|
492
482
|
msprobe/visualization/graph/distributed_analyzer.py,sha256=IVPaA6Z3uZxuOob7fibgc9zGFMjtC-LaqQBzaragWbw,19046
|
|
493
|
-
msprobe/visualization/graph/graph.py,sha256=
|
|
483
|
+
msprobe/visualization/graph/graph.py,sha256=SFf_x4a3e5ejUe6PntPeEqnGCPaR6gUWpak6w_hyIBE,8776
|
|
494
484
|
msprobe/visualization/graph/node_colors.py,sha256=7LpurTuE3edKGilwLVsXD7Ue4bMT7Maz63Udq_6CADM,4504
|
|
495
485
|
msprobe/visualization/graph/node_op.py,sha256=qkbw3ZJkKGQH071C1CLIXi4kavvsLxBTzt0KXncz6fY,1368
|
|
496
|
-
mindstudio_probe-8.3.
|
|
497
|
-
mindstudio_probe-8.3.
|
|
498
|
-
mindstudio_probe-8.3.
|
|
499
|
-
mindstudio_probe-8.3.
|
|
500
|
-
mindstudio_probe-8.3.
|
|
501
|
-
mindstudio_probe-8.3.
|
|
486
|
+
mindstudio_probe-8.3.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
487
|
+
mindstudio_probe-8.3.1.dist-info/METADATA,sha256=r3DZB8reM8qZOOrmbvMGNGFRKXQaH5y86T4GWCcgQ_M,1437
|
|
488
|
+
mindstudio_probe-8.3.1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
|
|
489
|
+
mindstudio_probe-8.3.1.dist-info/entry_points.txt,sha256=4ob3a9L018EBZFdlfgMW1lbgeIOhc4F-HCR8gBksaCQ,49
|
|
490
|
+
mindstudio_probe-8.3.1.dist-info/top_level.txt,sha256=LxFEFqelENSyWmRtocCiEUF04IE8aZvwTl7ADB598Tk,8
|
|
491
|
+
mindstudio_probe-8.3.1.dist-info/RECORD,,
|
msprobe/README.md
CHANGED
|
@@ -50,9 +50,14 @@ export MSPROBE_LOG_LEVEL={x}
|
|
|
50
50
|
|
|
51
51
|
## 🚨 工具限制与注意事项
|
|
52
52
|
|
|
53
|
-
**1.
|
|
53
|
+
**1. 工具读写的所有路径,如config_path、dump_path等,只允许包含大小写字母、数字、下划线、斜杠、点和短横线。**
|
|
54
|
+
|
|
55
|
+
**2. 出于安全性及权限最小化角度考虑,msprobe工具不应使用root等高权限账户使用,建议使用普通用户权限安装执行。**
|
|
56
|
+
|
|
57
|
+
**3. 使用msprobe工具前请确保执行用户的umask值大于等于0027,否则可能会导致工具生成的精度数据文件和目录权限过大。**
|
|
58
|
+
|
|
59
|
+
**4. 用户须自行保证使用最小权限原则,如给工具输入的文件要求other用户不可写,在一些对安全要求更严格的功能场景下还需确保输入的文件group用户不可写。**
|
|
54
60
|
|
|
55
|
-
**2. 工具读写的所有路径,如config_path、dump_path等,只允许包含大小写字母、数字、下划线、斜杠、点和短横线。**
|
|
56
61
|
|
|
57
62
|
## ⚙️ 安装
|
|
58
63
|
|
|
@@ -84,9 +89,7 @@ msprobe 通过在训练脚本中添加 PrecisionDebugger 接口的方式对 API
|
|
|
84
89
|
|
|
85
90
|
精度预检旨在昇腾 NPU 上扫描训练模型中的所有 API 进行 API 复现,给出精度情况的诊断和分析。对应 config.json 中的 "run_ut" task。
|
|
86
91
|
|
|
87
|
-
PyTorch 场景的[离线预检](./docs/07.accuracy_checker_PyTorch.md)
|
|
88
|
-
|
|
89
|
-
在线预检计划在 Mindstudio 8.3.0 版本正式下线。
|
|
92
|
+
PyTorch 场景的[离线预检](./docs/07.accuracy_checker_PyTorch.md)
|
|
90
93
|
|
|
91
94
|
MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore.md)
|
|
92
95
|
|
msprobe/core/common/const.py
CHANGED
|
@@ -691,9 +691,23 @@ class FileCheckConst:
|
|
|
691
691
|
"""
|
|
692
692
|
Class for file check const
|
|
693
693
|
"""
|
|
694
|
-
READ_ABLE = "
|
|
695
|
-
WRITE_ABLE = "
|
|
696
|
-
|
|
694
|
+
READ_ABLE = "r"
|
|
695
|
+
WRITE_ABLE = "w"
|
|
696
|
+
EXECUTE_ABLE = "x"
|
|
697
|
+
READ_WRITE_ABLE = "rw"
|
|
698
|
+
READ_EXECUTE_ABLE = "rx"
|
|
699
|
+
WRITE_EXECUTE_ABLE = "wx"
|
|
700
|
+
READ_WRITE_EXECUTE_ABLE = "rwx"
|
|
701
|
+
PERM_OPTIONS = [
|
|
702
|
+
READ_ABLE,
|
|
703
|
+
WRITE_ABLE,
|
|
704
|
+
EXECUTE_ABLE,
|
|
705
|
+
READ_WRITE_ABLE,
|
|
706
|
+
READ_EXECUTE_ABLE,
|
|
707
|
+
WRITE_EXECUTE_ABLE,
|
|
708
|
+
READ_WRITE_EXECUTE_ABLE
|
|
709
|
+
]
|
|
710
|
+
|
|
697
711
|
DIRECTORY_LENGTH = 4096
|
|
698
712
|
FILE_NAME_LENGTH = 255
|
|
699
713
|
FILE_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$"
|
|
@@ -46,13 +46,19 @@ class FileChecker:
|
|
|
46
46
|
The class for check file.
|
|
47
47
|
|
|
48
48
|
Attributes:
|
|
49
|
-
file_path: The file or dictionary path to be verified
|
|
50
|
-
path_type: file or dictionary
|
|
51
|
-
ability(str): one of [
|
|
49
|
+
file_path: The file or dictionary path to be verified
|
|
50
|
+
path_type: file or dictionary type
|
|
51
|
+
ability(str): one of ["r", "w", "x", "rw", "rx", "wx", "rwx"], r: read, w: write, x: execute
|
|
52
52
|
file_type(str): The correct file type for file
|
|
53
53
|
"""
|
|
54
54
|
|
|
55
|
-
def __init__(
|
|
55
|
+
def __init__(
|
|
56
|
+
self,
|
|
57
|
+
file_path,
|
|
58
|
+
path_type,
|
|
59
|
+
ability=None,
|
|
60
|
+
file_type=None
|
|
61
|
+
):
|
|
56
62
|
self.file_path = file_path
|
|
57
63
|
self.path_type = self._check_path_type(path_type)
|
|
58
64
|
self.ability = self._check_ability_type(ability)
|
|
@@ -67,15 +73,14 @@ class FileChecker:
|
|
|
67
73
|
|
|
68
74
|
@staticmethod
|
|
69
75
|
def _check_ability_type(ability):
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
logger.error(f'The ability must be one of {ability_list}.')
|
|
76
|
+
if ability and ability not in FileCheckConst.PERM_OPTIONS:
|
|
77
|
+
logger.error(f'The ability must be one of {FileCheckConst.PERM_OPTIONS}.')
|
|
73
78
|
raise FileCheckException(FileCheckException.ILLEGAL_PARAM_ERROR)
|
|
74
79
|
return ability
|
|
75
80
|
|
|
76
81
|
def common_check(self):
|
|
77
82
|
"""
|
|
78
|
-
|
|
83
|
+
功能:基本文件权限校验,包括文件存在性、软连接、文件长度、文件类型、文件读写权限、文件属组、文件路径特殊字符、文件后缀名等
|
|
79
84
|
注意:文件后缀的合法性,非通用操作,可使用其他独立接口实现
|
|
80
85
|
"""
|
|
81
86
|
check_path_exists(self.file_path)
|
|
@@ -94,13 +99,15 @@ class FileChecker:
|
|
|
94
99
|
return self.file_path
|
|
95
100
|
|
|
96
101
|
def check_path_ability(self):
|
|
97
|
-
if self.ability
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
if self.ability == FileCheckConst.READ_WRITE_ABLE:
|
|
102
|
+
if not self.ability:
|
|
103
|
+
return
|
|
104
|
+
|
|
105
|
+
if FileCheckConst.READ_ABLE in self.ability:
|
|
102
106
|
check_path_readability(self.file_path)
|
|
107
|
+
if FileCheckConst.WRITE_ABLE in self.ability:
|
|
103
108
|
check_path_writability(self.file_path)
|
|
109
|
+
if FileCheckConst.EXECUTE_ABLE in self.ability:
|
|
110
|
+
check_path_executable(self.file_path)
|
|
104
111
|
|
|
105
112
|
|
|
106
113
|
class FileOpen:
|
|
@@ -1040,4 +1047,48 @@ class SafeUnpickler(pickle.Unpickler):
|
|
|
1040
1047
|
raise pickle.PicklingError(f'Unpickling {module}.{name} is illegal!')
|
|
1041
1048
|
|
|
1042
1049
|
|
|
1050
|
+
class DeserializationScanner:
|
|
1051
|
+
"""反序列化风险扫描器"""
|
|
1052
|
+
|
|
1053
|
+
DANGEROUS_METHODS = {
|
|
1054
|
+
'__reduce__', '__reduce_ex__', '__setstate__', '__getstate__',
|
|
1055
|
+
'__new__', '__init__', '__del__',
|
|
1056
|
+
'__call__', '__enter__', '__exit__',
|
|
1057
|
+
'eval', 'exec', 'compile', '__import__', 'open'
|
|
1058
|
+
'os.system', 'os.popen', 'subprocess.call', 'subprocess.Popen',
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
DANGEROUS_MODULES = {
|
|
1062
|
+
'os', 'sys', 'subprocess', 'shutil', 'socket',
|
|
1063
|
+
'requests', 'urllib', 'ftplib', 'smtplib',
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
@classmethod
|
|
1067
|
+
def scan_pickle_content(cls, filepath: str) -> bool:
|
|
1068
|
+
with FileOpen(filepath, 'rb') as f:
|
|
1069
|
+
content = f.read()
|
|
1070
|
+
|
|
1071
|
+
try:
|
|
1072
|
+
text_content = content.decode('latin-1')
|
|
1073
|
+
except Exception as e:
|
|
1074
|
+
text_content = str(content)
|
|
1075
|
+
|
|
1076
|
+
for method in cls.DANGEROUS_METHODS:
|
|
1077
|
+
if re.fullmatch(method, text_content):
|
|
1078
|
+
logger.warning(f"Insecure method found: {method}")
|
|
1079
|
+
return False
|
|
1080
|
+
|
|
1081
|
+
for module in cls.DANGEROUS_MODULES:
|
|
1082
|
+
patterns = [
|
|
1083
|
+
f"import {module}",
|
|
1084
|
+
f"from {module} import",
|
|
1085
|
+
f"{module}.",
|
|
1086
|
+
]
|
|
1087
|
+
for pattern in patterns:
|
|
1088
|
+
if pattern in text_content:
|
|
1089
|
+
logger.warning(f"Insecure module found: {module}")
|
|
1090
|
+
return False
|
|
1091
|
+
return True
|
|
1092
|
+
|
|
1093
|
+
|
|
1043
1094
|
atexit.register(SharedDict.destroy_shared_memory)
|
|
@@ -16,7 +16,9 @@
|
|
|
16
16
|
import functools
|
|
17
17
|
|
|
18
18
|
from msprobe.core.common.const import Const
|
|
19
|
-
from msprobe.core.common.file_utils import check_file_or_directory_path, save_npy
|
|
19
|
+
from msprobe.core.common.file_utils import check_file_or_directory_path, save_npy, DeserializationScanner
|
|
20
|
+
from msprobe.core.common.log import logger
|
|
21
|
+
from msprobe.core.common.utils import confirm
|
|
20
22
|
|
|
21
23
|
|
|
22
24
|
class FrameworkDescriptor:
|
|
@@ -155,6 +157,13 @@ class FmkAdp:
|
|
|
155
157
|
check_file_or_directory_path(path, is_strict=not weights_only)
|
|
156
158
|
if cls.fmk == Const.PT_FRAMEWORK:
|
|
157
159
|
try:
|
|
160
|
+
if not weights_only:
|
|
161
|
+
if not DeserializationScanner.scan_pickle_content(path):
|
|
162
|
+
if not confirm(
|
|
163
|
+
f"Some insecure methods or modules are detected in {path}, "
|
|
164
|
+
f"input yes to ignore and continue, otherwise exit", False):
|
|
165
|
+
logger.error("Insecure risks found and exit!")
|
|
166
|
+
raise Exception("Insecure risks found and exit!")
|
|
158
167
|
if to_cpu:
|
|
159
168
|
return cls.framework.load(path, map_location=cls.framework.device("cpu"), weights_only=weights_only)
|
|
160
169
|
else:
|
msprobe/core/common/utils.py
CHANGED
|
@@ -708,3 +708,20 @@ def check_process_num(process_num):
|
|
|
708
708
|
raise ValueError(f"process_num({process_num}) is not a positive integer")
|
|
709
709
|
if process_num > Const.MAX_PROCESS_NUM:
|
|
710
710
|
raise ValueError(f"The maximum supported process_num is {Const.MAX_PROCESS_NUM}, current value: {process_num}.")
|
|
711
|
+
|
|
712
|
+
|
|
713
|
+
def confirm(prompt, default=False):
|
|
714
|
+
if default is True:
|
|
715
|
+
prompt_suffix = " [Y/n] "
|
|
716
|
+
elif default is False:
|
|
717
|
+
prompt_suffix = " [y/N] "
|
|
718
|
+
else:
|
|
719
|
+
prompt_suffix = " [y/n] "
|
|
720
|
+
|
|
721
|
+
full_prompt = prompt + prompt_suffix
|
|
722
|
+
|
|
723
|
+
user_input = input(full_prompt).strip().lower()
|
|
724
|
+
if user_input in ['y', 'yes']:
|
|
725
|
+
return True
|
|
726
|
+
else:
|
|
727
|
+
return default
|
|
@@ -19,11 +19,11 @@ from tqdm import tqdm
|
|
|
19
19
|
from msprobe.core.common.file_utils import save_json, check_path_before_create, check_path_not_exists, \
|
|
20
20
|
check_file_or_directory_path
|
|
21
21
|
from msprobe.core.common.log import logger
|
|
22
|
+
from msprobe.core.common.utils import confirm
|
|
22
23
|
from msprobe.core.config_check.ckpt_compare.megatron_loader import load_megatron_weights
|
|
23
24
|
from msprobe.core.config_check.ckpt_compare.metrics import METRIC_FUNC
|
|
24
25
|
|
|
25
26
|
|
|
26
|
-
|
|
27
27
|
def compare_checkpoints(ckpt_path1, ckpt_path2, output_path) -> Dict:
|
|
28
28
|
"""Compare weights between two checkpoints using cosine similarity and L2 distance.
|
|
29
29
|
|
|
@@ -45,6 +45,11 @@ def compare_checkpoints(ckpt_path1, ckpt_path2, output_path) -> Dict:
|
|
|
45
45
|
"""
|
|
46
46
|
|
|
47
47
|
# Load both checkpoints
|
|
48
|
+
if not confirm("You are using torch.load with weights_only is False, it may cause arbitrary code "
|
|
49
|
+
"execution. Do it only if you get the file from a trusted source. Input yes to continue, "
|
|
50
|
+
"otherwise exit", False):
|
|
51
|
+
logger.error("Insecure risks found and exit!")
|
|
52
|
+
raise Exception("Insecure risks found and exit!")
|
|
48
53
|
check_file_or_directory_path(ckpt_path1, isdir=True)
|
|
49
54
|
check_file_or_directory_path(ckpt_path2, isdir=True)
|
|
50
55
|
check_path_before_create(output_path)
|
msprobe/core/hook_manager.py
CHANGED
|
@@ -46,10 +46,9 @@ class BaseHookManager(ABC):
|
|
|
46
46
|
hook_handle_dict = {}
|
|
47
47
|
params_grad_info = {}
|
|
48
48
|
|
|
49
|
-
def __init__(self, data_collector, config
|
|
49
|
+
def __init__(self, data_collector, config):
|
|
50
50
|
self.data_collector = data_collector
|
|
51
51
|
self.config = config
|
|
52
|
-
self.attl_manager = attl_manager
|
|
53
52
|
|
|
54
53
|
@property
|
|
55
54
|
def _pid(self):
|
|
@@ -216,10 +215,6 @@ class BaseHookManager(ABC):
|
|
|
216
215
|
|
|
217
216
|
args = self._register_backward_hook(module, full_backward_name, args)
|
|
218
217
|
with self._no_grad_context():
|
|
219
|
-
if getattr(self.config, "online_run_ut", False):
|
|
220
|
-
BaseHookManager.inner_switch[tid] = False
|
|
221
|
-
ThreadSafe.release()
|
|
222
|
-
return
|
|
223
218
|
self.data_collector.update_api_or_module_name(full_forward_name)
|
|
224
219
|
self.data_collector.forward_input_data_collect(
|
|
225
220
|
full_forward_name,
|
|
@@ -265,13 +260,6 @@ class BaseHookManager(ABC):
|
|
|
265
260
|
output = self._register_backward_pre_hook(module, full_backward_name, output)
|
|
266
261
|
|
|
267
262
|
with self._no_grad_context():
|
|
268
|
-
if getattr(self.config, "online_run_ut", False):
|
|
269
|
-
if self.data_collector.scope and not self.data_collector.scope.check(full_name):
|
|
270
|
-
return None
|
|
271
|
-
if self.attl_manager:
|
|
272
|
-
self.attl_manager.attl_send(full_name, args, kwargs, output)
|
|
273
|
-
BaseHookManager.inner_switch[tid] = False
|
|
274
|
-
return None
|
|
275
263
|
if hook_type == Const.MODULE:
|
|
276
264
|
params_dict = self._get_params_dict(module)
|
|
277
265
|
setattr(module_input_output, Const.PARAMS, params_dict)
|
|
@@ -320,9 +308,7 @@ class BaseHookManager(ABC):
|
|
|
320
308
|
original_state = self.ensure_gc_enabled()
|
|
321
309
|
BaseHookManager.inner_switch[tid] = True
|
|
322
310
|
self.data_collector.update_api_or_module_name(full_name)
|
|
323
|
-
|
|
324
|
-
BaseHookManager.inner_switch[tid] = False
|
|
325
|
-
return
|
|
311
|
+
|
|
326
312
|
need_exchange = self._need_exchange(module) if hook_type == Const.MODULE else True
|
|
327
313
|
if need_exchange:
|
|
328
314
|
module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_output, grad_output=grad_input)
|
msprobe/core/service.py
CHANGED
|
@@ -35,7 +35,6 @@ class BaseService(ABC):
|
|
|
35
35
|
self.config.level = getattr(config, 'level_ori', config.level) # 兼容MindSpore配置
|
|
36
36
|
self.model = None
|
|
37
37
|
self.data_collector = build_data_collector(self.config)
|
|
38
|
-
self.attl_manager = None
|
|
39
38
|
self.current_iter = 0
|
|
40
39
|
self.loop = 0
|
|
41
40
|
self.init_step = 0
|
|
@@ -91,10 +90,6 @@ class BaseService(ABC):
|
|
|
91
90
|
self.config.task in self.data_collector.tasks_need_tensor_data or
|
|
92
91
|
(self.config.task == Const.STATISTICS and self.config.tensor_list)
|
|
93
92
|
)
|
|
94
|
-
|
|
95
|
-
@property
|
|
96
|
-
def _is_online_run_ut(self):
|
|
97
|
-
return getattr(self.config, "online_run_ut", False)
|
|
98
93
|
|
|
99
94
|
@property
|
|
100
95
|
@abstractmethod
|
|
@@ -146,11 +141,9 @@ class BaseService(ABC):
|
|
|
146
141
|
self.primitive_switch = True
|
|
147
142
|
self._change_jit_switch(True)
|
|
148
143
|
self.logger.info(f"Dump switch is turned on at step {self.current_iter}. ")
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
self.create_dirs()
|
|
153
|
-
self.logger.info(f"Dump data will be saved in {self.dump_iter_dir}.")
|
|
144
|
+
|
|
145
|
+
self.create_dirs()
|
|
146
|
+
self.logger.info(f"Dump data will be saved in {self.dump_iter_dir}.")
|
|
154
147
|
|
|
155
148
|
def stop(self):
|
|
156
149
|
"""通用stop模板"""
|
|
@@ -165,8 +158,7 @@ class BaseService(ABC):
|
|
|
165
158
|
self._change_jit_switch(False)
|
|
166
159
|
if self._is_l2_level:
|
|
167
160
|
return
|
|
168
|
-
|
|
169
|
-
self._run_ut_dispatch(False)
|
|
161
|
+
|
|
170
162
|
self._process_async_dump()
|
|
171
163
|
self.data_collector.write_json()
|
|
172
164
|
|
|
@@ -266,8 +258,6 @@ class BaseService(ABC):
|
|
|
266
258
|
end_service = self.config.step and self.current_iter > max(self.config.step) or \
|
|
267
259
|
self.data_collector and self.data_collector.data_processor.is_terminated
|
|
268
260
|
if end_service:
|
|
269
|
-
if self._is_online_run_ut and self.attl_manager:
|
|
270
|
-
self.attl_manager.attl_stop()
|
|
271
261
|
self.primitive_switch = False
|
|
272
262
|
self._change_jit_switch(False)
|
|
273
263
|
Runtime.is_running = False
|
|
@@ -310,8 +300,7 @@ class BaseService(ABC):
|
|
|
310
300
|
if root_model and isinstance(root_model, list):
|
|
311
301
|
root_model = root_model[0]
|
|
312
302
|
self.logger.warning("Infer model can only input one to support token_range, choose the first one.")
|
|
313
|
-
|
|
314
|
-
return
|
|
303
|
+
|
|
315
304
|
root_model.register_forward_pre_hook(infer_hook)
|
|
316
305
|
|
|
317
306
|
def _create_l2_dirs(self, cur_rank):
|
msprobe/docs/01.installation.md
CHANGED
|
@@ -16,6 +16,8 @@ pip install mindstudio-probe
|
|
|
16
16
|
|
|
17
17
|
| 版本 | 发布日期 |支持 PyTorch 版本|支持 MindSpore 版本| 下载链接 |校验码|
|
|
18
18
|
|:-----:|:----------:|:--:|:--:|:----------------------------------------------------------------------------------------------------------------------------------:|:--:|
|
|
19
|
+
| 8.3.0 | 2025.10.30 |1.11/2.0/2.1/2.2/2.5/2.6/2.7|2.4.0/2.5.0/2.6.0| [mindstudio_probe-8.3.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/8.3/mindstudio_probe-8.3.0-py3-none-any.whl) |e933657b8ceb20774f924865d1f47978bd49cd1d9cec5fb60dec4f18802afbcc|
|
|
20
|
+
| 8.2.1 | 2025.9.29 |1.11/2.0/2.1/2.2|2.4.0/2.5.0/2.6.0| [mindstudio_probe-8.2.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/8.2/mindstudio_probe-8.2.1-py3-none-any.whl) |2152fadefec3d70148a910a54f2cfb6fccfa60b5ccd59f835f20289a7a5f4764|
|
|
19
21
|
| 8.2.0 | 2025.9.03 |1.11/2.0/2.1/2.2|2.4.0/2.5.0/2.6.0| [mindstudio_probe-8.2.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/8.2/mindstudio_probe-8.2.0-py3-none-any.whl) |bbc1577d76754adf987069308177d3e0a04e36de9c7f22e75c34cf4ad0ce1af2|
|
|
20
22
|
| 8.1.2 | 2025.8.01 |1.11/2.0/2.1/2.2|2.4.0/2.5.0/2.6.0| [mindstudio_probe-8.1.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/8.1/mindstudio_probe-8.1.2-py3-none-any.whl) |ff07bb81fddd3b8f3096d119ca1481bde8fdb24f10644def5250caad727448ab|
|
|
21
23
|
| 8.1.1 | 2025.6.20 |1.11/2.0/2.1/2.2|2.4.0/2.5.0/2.6.0| [mindstudio_probe-8.1.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/8.1/mindstudio_probe-8.1.1-py3-none-any.whl) |2aad10a243575544d7feef552caf4d06aa93028488ebd0bbc9aa350379da859d|
|