mindstudio-probe 8.3.3__py3-none-any.whl → 26.0.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mindstudio_probe-8.3.3.dist-info → mindstudio_probe-26.0.0a1.dist-info}/METADATA +26 -14
- mindstudio_probe-26.0.0a1.dist-info/RECORD +498 -0
- {mindstudio_probe-8.3.3.dist-info → mindstudio_probe-26.0.0a1.dist-info}/WHEEL +1 -1
- mindstudio_probe-26.0.0a1.dist-info/entry_points.txt +5 -0
- mindstudio_probe-26.0.0a1.dist-info/licenses/LICENSE +124 -0
- mindstudio_probe-26.0.0a1.dist-info/top_level.txt +2 -0
- msprobe/__init__.py +12 -13
- msprobe/config.json +9 -31
- msprobe/core/__init__.py +12 -11
- msprobe/core/acc_check/acc_check_cli.py +145 -0
- msprobe/core/common/const.py +97 -38
- msprobe/core/common/db_manager.py +133 -12
- msprobe/core/common/decorator.py +12 -11
- msprobe/core/common/exceptions.py +12 -11
- msprobe/core/common/file_utils.py +101 -25
- msprobe/core/common/framework_adapter.py +36 -25
- msprobe/core/common/global_lock.py +12 -11
- msprobe/core/common/inplace_op_checker.py +12 -11
- msprobe/core/common/log.py +22 -11
- msprobe/core/common/megatron_utils.py +566 -11
- msprobe/core/common/parallel_state.py +12 -11
- msprobe/core/common/runtime.py +12 -11
- msprobe/core/common/utils.py +41 -41
- msprobe/core/compare/acc_compare.py +361 -104
- msprobe/core/compare/atb_data_compare.py +422 -0
- msprobe/core/compare/auto_compare.py +134 -0
- msprobe/core/compare/check.py +14 -17
- msprobe/core/compare/compare_cli.py +72 -149
- msprobe/core/compare/config.py +12 -13
- msprobe/core/compare/diff_analyze/first_diff_analyze.py +28 -15
- msprobe/core/compare/diff_analyze/ignore_op_list.yaml +3 -0
- msprobe/core/compare/find_first/analyzer.py +18 -18
- msprobe/core/compare/find_first/graph.py +12 -11
- msprobe/core/compare/find_first/utils.py +13 -12
- msprobe/core/compare/indicator_analysis/__init__.py +15 -0
- msprobe/core/compare/indicator_analysis/algorithm.py +363 -0
- msprobe/core/compare/indicator_analysis/api_data.py +141 -0
- msprobe/core/compare/indicator_analysis/calculator.py +181 -0
- msprobe/core/compare/indicator_analysis/utils.py +116 -0
- msprobe/core/compare/layer_mapping/__init__.py +12 -11
- msprobe/core/compare/layer_mapping/data_scope_parser.py +20 -11
- msprobe/core/compare/layer_mapping/layer_mapping.py +14 -13
- msprobe/core/compare/layer_mapping/postprocess_pass.py +13 -11
- msprobe/core/compare/merge_result/merge_result.py +12 -11
- msprobe/core/compare/merge_result/merge_result_cli.py +12 -11
- msprobe/core/compare/merge_result/utils.py +12 -11
- msprobe/core/compare/multiprocessing_compute.py +13 -14
- msprobe/core/compare/npy_compare.py +13 -11
- msprobe/core/compare/offline_data_compare.py +160 -0
- msprobe/core/compare/stats_diff_calc.py +39 -0
- msprobe/core/compare/torchair_acc_cmp.py +764 -0
- msprobe/core/compare/torchair_cmp_utils.py +338 -0
- msprobe/core/compare/utils.py +140 -49
- msprobe/core/config_check/__init__.py +12 -11
- msprobe/core/config_check/checkers/__init__.py +12 -11
- msprobe/core/config_check/checkers/base_checker.py +15 -14
- msprobe/core/config_check/checkers/dataset_checker.py +13 -12
- msprobe/core/config_check/checkers/env_args_checker.py +13 -12
- msprobe/core/config_check/checkers/hyperparameter_checker.py +16 -15
- msprobe/core/config_check/checkers/pip_checker.py +15 -15
- msprobe/core/config_check/checkers/random_checker.py +13 -12
- msprobe/core/config_check/checkers/weights_checker.py +14 -12
- msprobe/core/config_check/ckpt_compare/ckpt_comparator.py +13 -17
- msprobe/core/config_check/ckpt_compare/megatron_loader.py +13 -12
- msprobe/core/config_check/ckpt_compare/metrics.py +12 -11
- msprobe/core/config_check/config_check_cli.py +18 -17
- msprobe/core/config_check/config_checker.py +16 -14
- msprobe/core/config_check/resource/dependency.yaml +15 -12
- msprobe/core/config_check/resource/env.yaml +12 -11
- msprobe/core/config_check/utils/hyperparameter_parser.py +12 -11
- msprobe/core/config_check/utils/utils.py +12 -11
- msprobe/core/{data_dump → dump/api_dump}/api_registry.py +12 -11
- msprobe/core/{common_config.py → dump/common_config.py} +13 -24
- msprobe/core/dump/data_dump/data_collector.py +257 -0
- msprobe/core/{data_dump → dump/data_dump}/data_processor/base.py +45 -36
- msprobe/core/{data_dump → dump/data_dump}/data_processor/factory.py +33 -25
- msprobe/core/{data_dump → dump/data_dump}/data_processor/mindspore_processor.py +37 -113
- msprobe/core/{data_dump → dump/data_dump}/data_processor/pytorch_processor.py +364 -131
- msprobe/core/{data_dump → dump/data_dump}/json_writer.py +24 -31
- msprobe/core/{data_dump → dump/data_dump}/scope.py +12 -13
- msprobe/core/{debugger → dump/debugger}/precision_debugger.py +15 -23
- msprobe/core/dump/dump2db/db_utils.py +215 -0
- msprobe/core/dump/dump2db/dump2db.py +409 -0
- msprobe/core/{hook_manager.py → dump/hook_manager.py} +38 -87
- msprobe/core/dump/kernel_dump/kernel_config.py +34 -0
- msprobe/core/{service.py → dump/service.py} +43 -27
- msprobe/core/install_deps/install_deps.py +51 -0
- msprobe/core/monitor/anomaly_processor.py +13 -11
- msprobe/core/monitor/csv2db.py +73 -93
- msprobe/core/monitor/db_utils.py +140 -205
- msprobe/core/monitor/utils.py +18 -17
- msprobe/core/monitor_v2/__init__.py +20 -0
- msprobe/core/monitor_v2/base.py +83 -0
- msprobe/core/monitor_v2/cc.py +287 -0
- msprobe/core/monitor_v2/factory.py +81 -0
- msprobe/core/monitor_v2/module.py +201 -0
- msprobe/core/monitor_v2/optimizer.py +245 -0
- msprobe/core/monitor_v2/param.py +154 -0
- msprobe/core/monitor_v2/trainer.py +326 -0
- msprobe/core/monitor_v2/utils.py +122 -0
- msprobe/core/monitor_v2/weight_grad.py +419 -0
- msprobe/core/monitor_v2/writer.py +162 -0
- msprobe/core/overflow_check/abnormal_scene.py +12 -11
- msprobe/core/overflow_check/api_info.py +12 -11
- msprobe/core/overflow_check/checker.py +12 -11
- msprobe/core/overflow_check/filter.py +13 -11
- msprobe/core/overflow_check/level.py +12 -11
- msprobe/core/overflow_check/utils.py +12 -11
- msprobe/core/single_save/single_comparator.py +12 -11
- msprobe/core/single_save/single_saver.py +12 -11
- msprobe/infer/__init__.py +16 -0
- msprobe/infer/offline/__init__.py +16 -0
- msprobe/infer/offline/compare/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/adapter_cli/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/adapter_cli/args_adapter.py +46 -0
- msprobe/infer/offline/compare/msquickcmp/atc/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/atc/atc_utils.py +98 -0
- msprobe/infer/offline/compare/msquickcmp/cmp_process.py +328 -0
- msprobe/infer/offline/compare/msquickcmp/common/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/common/args_check.py +112 -0
- msprobe/infer/offline/compare/msquickcmp/common/convert.py +74 -0
- msprobe/infer/offline/compare/msquickcmp/common/dump_data.py +121 -0
- msprobe/infer/offline/compare/msquickcmp/common/dynamic_argument_bean.py +39 -0
- msprobe/infer/offline/compare/msquickcmp/common/utils.py +669 -0
- msprobe/infer/offline/compare/msquickcmp/config.ini +6 -0
- msprobe/infer/offline/compare/msquickcmp/dump/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/dump/args_adapter.py +50 -0
- msprobe/infer/offline/compare/msquickcmp/dump/dump_process.py +91 -0
- msprobe/infer/offline/compare/msquickcmp/install_aclruntime_aisbench.sh +180 -0
- msprobe/infer/offline/compare/msquickcmp/main.py +199 -0
- msprobe/infer/offline/compare/msquickcmp/net_compare/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/net_compare/net_compare.py +277 -0
- msprobe/infer/offline/compare/msquickcmp/npu/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/npu/npu_dump_data.py +558 -0
- msprobe/infer/offline/compare/msquickcmp/npu/om_parser.py +416 -0
- msprobe/infer/offline/compare/msquickcmp/onnx_model/__init__.py +16 -0
- msprobe/infer/offline/compare/msquickcmp/onnx_model/onnx_dump_data.py +374 -0
- msprobe/infer/utils/__init__.py +15 -0
- msprobe/infer/utils/acc_cmp.py +94 -0
- msprobe/infer/utils/check/__init__.py +37 -0
- msprobe/infer/utils/check/args_checker.py +35 -0
- msprobe/infer/utils/check/checker.py +227 -0
- msprobe/infer/utils/check/dict_checker.py +78 -0
- msprobe/infer/utils/check/func_wrapper.py +96 -0
- msprobe/infer/utils/check/list_checker.py +56 -0
- msprobe/infer/utils/check/number_checker.py +64 -0
- msprobe/infer/utils/check/obj_checker.py +41 -0
- msprobe/infer/utils/check/path_checker.py +249 -0
- msprobe/infer/utils/check/rule.py +126 -0
- msprobe/infer/utils/check/string_checker.py +66 -0
- msprobe/infer/utils/cmp_algorithm.py +261 -0
- msprobe/infer/utils/constants.py +112 -0
- msprobe/infer/utils/file_open_check.py +337 -0
- msprobe/infer/utils/util.py +177 -0
- msprobe/mindspore/__init__.py +14 -13
- msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py +14 -13
- msprobe/mindspore/api_accuracy_checker/api_info.py +12 -11
- msprobe/mindspore/api_accuracy_checker/api_runner.py +12 -11
- msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py +12 -11
- msprobe/mindspore/api_accuracy_checker/bench_functions/flash_attention_score.py +12 -11
- msprobe/mindspore/api_accuracy_checker/bench_functions/fusion_operator.py +12 -11
- msprobe/mindspore/api_accuracy_checker/checker_support_api.yaml +12 -11
- msprobe/mindspore/api_accuracy_checker/cmd_parser.py +15 -14
- msprobe/mindspore/api_accuracy_checker/compute_element.py +12 -11
- msprobe/mindspore/api_accuracy_checker/data_manager.py +13 -11
- msprobe/mindspore/api_accuracy_checker/main.py +12 -11
- msprobe/mindspore/api_accuracy_checker/multi_api_accuracy_checker.py +14 -12
- msprobe/mindspore/api_accuracy_checker/multi_data_manager.py +13 -11
- msprobe/mindspore/api_accuracy_checker/torch_mindtorch_importer.py +12 -11
- msprobe/mindspore/api_accuracy_checker/type_mapping.py +12 -11
- msprobe/mindspore/api_accuracy_checker/utils.py +12 -11
- msprobe/mindspore/common/const.py +15 -74
- msprobe/mindspore/common/log.py +12 -11
- msprobe/mindspore/common/utils.py +30 -15
- msprobe/mindspore/compare/common_dir_compare.py +21 -23
- msprobe/mindspore/compare/distributed_compare.py +18 -16
- msprobe/mindspore/compare/ms_compare.py +14 -14
- msprobe/mindspore/compare/ms_graph_compare.py +26 -20
- msprobe/mindspore/compare/utils.py +14 -12
- msprobe/mindspore/{cell_processor.py → dump/cell_processor.py} +15 -14
- msprobe/mindspore/{debugger → dump/debugger}/debugger_config.py +12 -30
- msprobe/mindspore/{debugger → dump/debugger}/precision_debugger.py +43 -45
- msprobe/mindspore/dump/{cell_dump_process.py → dump_processor/cell_dump_process.py} +31 -17
- msprobe/mindspore/dump/{cell_dump_with_insert_gradient.py → dump_processor/cell_dump_with_insert_gradient.py} +18 -14
- msprobe/mindspore/dump/{dump_tool_factory.py → dump_processor/dump_tool_factory.py} +16 -15
- msprobe/mindspore/dump/{graph_mode_cell_dump.py → dump_processor/graph_mode_cell_dump.py} +16 -15
- msprobe/mindspore/dump/{graph_tensor_dump.py → dump_processor/graph_tensor_dump.py} +134 -133
- msprobe/mindspore/dump/{hook_cell → dump_processor/hook_cell}/api_register.py +15 -14
- msprobe/mindspore/dump/{hook_cell → dump_processor/hook_cell}/hook_cell.py +12 -11
- msprobe/mindspore/dump/{hook_cell → dump_processor/hook_cell}/ms_hook_manager.py +47 -20
- msprobe/mindspore/dump/{hook_cell → dump_processor/hook_cell}/primitive_hooks.py +14 -13
- msprobe/mindspore/dump/{hook_cell → dump_processor/hook_cell}/support_wrap_ops.yaml +13 -11
- msprobe/mindspore/dump/{jit_dump.py → dump_processor/jit_dump.py} +14 -13
- msprobe/mindspore/dump/{kernel_graph_dump.py → dump_processor/kernel_graph_dump.py} +13 -12
- msprobe/mindspore/dump/{kernel_kbyk_dump.py → dump_processor/kernel_kbyk_dump.py} +13 -12
- msprobe/mindspore/{exception_dump → dump/exception_dump}/exception_dump_tool_factory.py +14 -13
- msprobe/mindspore/{exception_dump → dump/exception_dump}/kernel_graph_exception_dump.py +13 -12
- msprobe/mindspore/{mindspore_service.py → dump/mindspore_service.py} +18 -17
- msprobe/mindspore/dump/mindtorch/__init__.py +19 -0
- msprobe/mindspore/dump/ms_config.py +105 -0
- msprobe/mindspore/{overflow_check → dump/overflow_check}/kernel_graph_overflow_check.py +13 -12
- msprobe/mindspore/{overflow_check → dump/overflow_check}/overflow_check_tool_factory.py +14 -13
- msprobe/mindspore/dump/task_handler_factory.py +43 -0
- msprobe/mindspore/monitor/common_func.py +12 -11
- msprobe/mindspore/monitor/data_writers.py +12 -11
- msprobe/mindspore/monitor/distributed/wrap_distributed.py +93 -39
- msprobe/mindspore/monitor/features.py +12 -11
- msprobe/mindspore/monitor/module_hook.py +19 -22
- msprobe/mindspore/monitor/optimizer_collect.py +29 -25
- msprobe/mindspore/monitor/utils.py +13 -11
- msprobe/msaccucmp/advisor/__init__.py +16 -0
- msprobe/msaccucmp/advisor/advisor_const.py +65 -0
- msprobe/msaccucmp/advisor/advisor_result.py +73 -0
- msprobe/msaccucmp/advisor/compare_advisor.py +99 -0
- msprobe/msaccucmp/advisor/input_advisor.py +66 -0
- msprobe/msaccucmp/advisor/node_advisor.py +68 -0
- msprobe/msaccucmp/advisor/overflow_advisor.py +58 -0
- msprobe/msaccucmp/algorithm_manager/__init__.py +16 -0
- msprobe/msaccucmp/algorithm_manager/algorithm_manager.py +464 -0
- msprobe/msaccucmp/algorithm_manager/algorithm_parameter.py +42 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_AccumulatedRelativeError.py +46 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_CosineSimilarity.py +58 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_KullbackLeiblerDivergence.py +84 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_MaxAbsoluteError.py +41 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_MaxRelativeError.py +46 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_MeanAbsoluteError.py +41 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_MeanRelativeError.py +46 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_RelativeEuclideanDistance.py +46 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_RootMeanSquareError.py +40 -0
- msprobe/msaccucmp/algorithm_manager/builtin_algorithm/alg_StandardDeviation.py +47 -0
- msprobe/msaccucmp/cmp_utils/__init__.py +16 -0
- msprobe/msaccucmp/cmp_utils/common.py +113 -0
- msprobe/msaccucmp/cmp_utils/constant/__init__.py +16 -0
- msprobe/msaccucmp/cmp_utils/constant/compare_error.py +81 -0
- msprobe/msaccucmp/cmp_utils/constant/const_manager.py +530 -0
- msprobe/msaccucmp/cmp_utils/file_utils.py +497 -0
- msprobe/msaccucmp/cmp_utils/log.py +257 -0
- msprobe/msaccucmp/cmp_utils/multi_process/__init__.py +16 -0
- msprobe/msaccucmp/cmp_utils/multi_process/multi_convert_process.py +140 -0
- msprobe/msaccucmp/cmp_utils/multi_process/progress.py +78 -0
- msprobe/msaccucmp/cmp_utils/path_check.py +274 -0
- msprobe/msaccucmp/cmp_utils/reg_manager.py +98 -0
- msprobe/msaccucmp/cmp_utils/tlv_parse.py +279 -0
- msprobe/msaccucmp/cmp_utils/utils.py +356 -0
- msprobe/msaccucmp/cmp_utils/utils_type.py +63 -0
- msprobe/msaccucmp/compare_vector.py +48 -0
- msprobe/msaccucmp/conversion/__init__.py +16 -0
- msprobe/msaccucmp/conversion/data_conversion.py +277 -0
- msprobe/msaccucmp/conversion/dtype_conversion.py +99 -0
- msprobe/msaccucmp/conversion/shape_format_conversion.py +477 -0
- msprobe/msaccucmp/conversion/tensor_conversion.py +369 -0
- msprobe/msaccucmp/dump_data_conversion.py +46 -0
- msprobe/msaccucmp/dump_parse/__init__.py +16 -0
- msprobe/msaccucmp/dump_parse/big_dump_data.py +317 -0
- msprobe/msaccucmp/dump_parse/dump.py +423 -0
- msprobe/msaccucmp/dump_parse/dump_data_object.py +322 -0
- msprobe/msaccucmp/dump_parse/dump_data_parser.py +436 -0
- msprobe/msaccucmp/dump_parse/dump_utils.py +246 -0
- msprobe/msaccucmp/dump_parse/ffts_parser.py +137 -0
- msprobe/msaccucmp/dump_parse/mapping.py +62 -0
- msprobe/msaccucmp/dump_parse/nano_dump_data.py +392 -0
- msprobe/msaccucmp/dump_parse/proto_dump_data.py +308 -0
- msprobe/msaccucmp/dump_parser.py +90 -0
- msprobe/msaccucmp/format_manager/__init__.py +16 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_FRACTAL_NZ_to_NCHW.py +53 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_FRACTAL_NZ_to_ND.py +52 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_FRACTAL_NZ_to_NHWC.py +53 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_FRACTAL_Z_to_HWCN.py +47 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_FRACTAL_Z_to_NCHW.py +47 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_HWCN_to_FRACTAL_Z.py +89 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_HWCN_to_NCHW.py +37 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_HWCN_to_NHWC.py +37 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NC1HWC0_to_HWCN.py +43 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NC1HWC0_to_NCHW.py +48 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NC1HWC0_to_NHWC.py +43 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NCHW_to_FRACTAL_Z.py +87 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NCHW_to_NHWC.py +37 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NDC1HWC0_to_NCDHW.py +48 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NDC1HWC0_to_ND.py +44 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NHWC_to_FRACTAL_Z.py +87 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NHWC_to_HWCN.py +37 -0
- msprobe/msaccucmp/format_manager/builtin_format_convert/convert_NHWC_to_NCHW.py +37 -0
- msprobe/msaccucmp/format_manager/format_manager.py +307 -0
- msprobe/msaccucmp/inplace_layer_process.py +186 -0
- msprobe/msaccucmp/msaccucmp.py +532 -0
- msprobe/msaccucmp/mscmp_advisor.py +128 -0
- msprobe/msaccucmp/overflow/__init__.py +16 -0
- msprobe/msaccucmp/overflow/overflow_analyse.py +305 -0
- msprobe/msaccucmp/overflow/overflow_detection.py +143 -0
- msprobe/msaccucmp/pytorch_cmp/__init__.py +16 -0
- msprobe/msaccucmp/pytorch_cmp/compare_pytorch.py +389 -0
- msprobe/msaccucmp/pytorch_cmp/hdf5_parser.py +377 -0
- msprobe/msaccucmp/pytorch_cmp/pytorch_dump_data.py +461 -0
- msprobe/msaccucmp/shape_conversion.py +41 -0
- msprobe/msaccucmp/vector_cmp/__init__.py +16 -0
- msprobe/msaccucmp/vector_cmp/batch_compare.py +197 -0
- msprobe/msaccucmp/vector_cmp/compare_detail/__init__.py +16 -0
- msprobe/msaccucmp/vector_cmp/compare_detail/compare_detail.py +245 -0
- msprobe/msaccucmp/vector_cmp/compare_detail/detail.py +182 -0
- msprobe/msaccucmp/vector_cmp/compare_detail/detail_writer.py +580 -0
- msprobe/msaccucmp/vector_cmp/fusion_manager/__init__.py +16 -0
- msprobe/msaccucmp/vector_cmp/fusion_manager/compare_fusion_op.py +588 -0
- msprobe/msaccucmp/vector_cmp/fusion_manager/compare_npu_vs_npu.py +339 -0
- msprobe/msaccucmp/vector_cmp/fusion_manager/compare_result.py +326 -0
- msprobe/msaccucmp/vector_cmp/fusion_manager/compare_rule.py +156 -0
- msprobe/msaccucmp/vector_cmp/fusion_manager/fusion_op.py +204 -0
- msprobe/msaccucmp/vector_cmp/fusion_manager/fusion_rule_parser.py +635 -0
- msprobe/msaccucmp/vector_cmp/fusion_manager/quant_filter.py +187 -0
- msprobe/msaccucmp/vector_cmp/range_manager/__init__.py +16 -0
- msprobe/msaccucmp/vector_cmp/range_manager/range_manager.py +100 -0
- msprobe/msaccucmp/vector_cmp/range_manager/range_mode.py +94 -0
- msprobe/msaccucmp/vector_cmp/range_manager/select_mode.py +86 -0
- msprobe/msaccucmp/vector_cmp/vector_comparison.py +535 -0
- msprobe/msprobe.py +101 -130
- msprobe/overflow_check/__init__.py +15 -0
- msprobe/{nan_analyze → overflow_check}/analyzer.py +38 -27
- msprobe/{nan_analyze → overflow_check}/graph.py +28 -27
- msprobe/{nan_analyze → overflow_check}/utils.py +15 -14
- msprobe/pytorch/__init__.py +20 -14
- msprobe/pytorch/aclgraph_dump/__init__.py +45 -0
- msprobe/pytorch/aclgraph_dump/_meta.py +26 -0
- msprobe/pytorch/api_accuracy_checker/{run_ut/run_ut.py → acc_check/acc_check.py} +50 -45
- msprobe/pytorch/api_accuracy_checker/{run_ut/run_ut_utils.py → acc_check/acc_check_utils.py} +201 -30
- msprobe/pytorch/api_accuracy_checker/{run_ut → acc_check}/data_generate.py +56 -16
- msprobe/pytorch/api_accuracy_checker/{run_ut/multi_run_ut.py → acc_check/multi_acc_check.py} +32 -47
- msprobe/pytorch/api_accuracy_checker/{run_ut → acc_check}/run_overflow_check.py +19 -18
- msprobe/pytorch/api_accuracy_checker/common/config.py +22 -20
- msprobe/pytorch/api_accuracy_checker/common/utils.py +72 -13
- msprobe/pytorch/api_accuracy_checker/compare/algorithm.py +41 -11
- msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +23 -14
- msprobe/pytorch/api_accuracy_checker/compare/compare.py +45 -32
- msprobe/pytorch/api_accuracy_checker/compare/compare_column.py +12 -11
- msprobe/pytorch/api_accuracy_checker/compare/compare_input.py +14 -12
- msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py +14 -12
- msprobe/pytorch/api_accuracy_checker/precision_standard/absolute_threshold.py +12 -11
- msprobe/pytorch/api_accuracy_checker/precision_standard/accumulative_error_compare.py +12 -11
- msprobe/pytorch/api_accuracy_checker/precision_standard/base_standard.py +21 -19
- msprobe/pytorch/api_accuracy_checker/precision_standard/benchmark_compare.py +14 -13
- msprobe/pytorch/api_accuracy_checker/precision_standard/binary_consistency.py +12 -11
- msprobe/pytorch/api_accuracy_checker/precision_standard/standard_config.py +60 -11
- msprobe/pytorch/api_accuracy_checker/precision_standard/standard_register.py +27 -16
- msprobe/pytorch/api_accuracy_checker/precision_standard/thousandth_standard.py +13 -11
- msprobe/pytorch/api_accuracy_checker/precision_standard/ulp_compare.py +39 -18
- msprobe/pytorch/bench_functions/__init__.py +12 -11
- msprobe/pytorch/bench_functions/apply_adam.py +12 -11
- msprobe/pytorch/bench_functions/apply_adam_w.py +12 -11
- msprobe/pytorch/bench_functions/confusion_transpose.py +12 -11
- msprobe/pytorch/bench_functions/fast_gelu.py +12 -11
- msprobe/pytorch/bench_functions/group_norm_silu.py +12 -11
- msprobe/pytorch/bench_functions/layer_norm_eval.py +12 -11
- msprobe/pytorch/bench_functions/linear.py +12 -11
- msprobe/pytorch/bench_functions/matmul_backward.py +12 -11
- msprobe/pytorch/bench_functions/mish.py +12 -11
- msprobe/pytorch/bench_functions/moe_gating_top_k_softmax.py +12 -11
- msprobe/pytorch/bench_functions/npu_fusion_attention.py +12 -11
- msprobe/pytorch/bench_functions/rms_norm.py +12 -11
- msprobe/pytorch/bench_functions/rotary_mul.py +12 -11
- msprobe/pytorch/bench_functions/scaled_mask_softmax.py +12 -11
- msprobe/pytorch/bench_functions/sort_v2.py +12 -11
- msprobe/pytorch/bench_functions/swiglu.py +12 -11
- msprobe/pytorch/common/__init__.py +12 -11
- msprobe/pytorch/common/log.py +12 -11
- msprobe/pytorch/common/parse_json.py +12 -11
- msprobe/pytorch/common/utils.py +52 -19
- msprobe/pytorch/compare/distributed_compare.py +13 -13
- msprobe/pytorch/compare/match.py +12 -11
- msprobe/pytorch/compare/pt_compare.py +14 -20
- msprobe/pytorch/compare/pt_diff_analyze.py +12 -11
- msprobe/pytorch/compare/utils.py +12 -11
- msprobe/pytorch/{hook_module → dump/api_dump}/api_register.py +18 -16
- msprobe/pytorch/{hook_module → dump/api_dump}/hook_module.py +14 -13
- msprobe/pytorch/{hook_module → dump/api_dump}/pt_hook_manager.py +68 -23
- msprobe/pytorch/{hook_module → dump/api_dump}/register_optimizer_hook.py +13 -11
- msprobe/pytorch/{hook_module → dump/api_dump}/script_wrapper.py +17 -14
- msprobe/pytorch/{hook_module → dump/api_dump}/utils.py +12 -11
- msprobe/pytorch/{debugger → dump/debugger}/debugger_config.py +23 -38
- msprobe/pytorch/dump/debugger/precision_debugger.py +130 -0
- msprobe/pytorch/{function_factory.py → dump/function_factory.py} +12 -11
- msprobe/pytorch/dump/module_dump/hook_wrapper.py +17 -13
- msprobe/pytorch/dump/module_dump/module_dump.py +16 -15
- msprobe/pytorch/dump/module_dump/{module_processer.py → module_processor.py} +54 -42
- msprobe/pytorch/dump/pt_config.py +128 -0
- msprobe/pytorch/{pytorch_service.py → dump/pytorch_service.py} +22 -21
- msprobe/pytorch/monitor/csv2tb.py +13 -11
- msprobe/pytorch/monitor/data_writers.py +13 -11
- msprobe/pytorch/monitor/distributed/wrap_distributed.py +13 -11
- msprobe/pytorch/monitor/features.py +12 -11
- msprobe/pytorch/monitor/module_hook.py +67 -59
- msprobe/pytorch/monitor/module_metric.py +13 -11
- msprobe/pytorch/monitor/optimizer_collect.py +37 -35
- msprobe/pytorch/monitor/utils.py +13 -11
- msprobe/pytorch/monitor/visualizer.py +12 -11
- msprobe/pytorch/torchair_dump/__init__.py +17 -0
- msprobe/pytorch/torchair_dump/torchair_dump.py +114 -0
- msprobe/scripts/atb/config_example.json +10 -0
- msprobe/scripts/atb/load_atb_probe.sh +101 -0
- msprobe/scripts/atb/unload_atb_probe.sh +27 -0
- msprobe/scripts/build_msaccucmp.sh +186 -0
- msprobe/scripts/conf/help.info +6 -0
- msprobe/scripts/conf/version.info +3 -0
- msprobe/scripts/run_script/common.sh +538 -0
- msprobe/scripts/run_script/main_msaccucmp.sh +232 -0
- msprobe/visualization/__init__.py +12 -11
- msprobe/visualization/builder/__init__.py +12 -11
- msprobe/visualization/builder/graph_builder.py +45 -30
- msprobe/visualization/builder/graph_merger.py +53 -32
- msprobe/visualization/builder/msprobe_adapter.py +34 -44
- msprobe/visualization/compare/__init__.py +12 -11
- msprobe/visualization/compare/graph_comparator.py +63 -51
- msprobe/visualization/compare/mode_adapter.py +28 -113
- msprobe/visualization/db_utils.py +133 -22
- msprobe/visualization/graph/__init__.py +12 -11
- msprobe/visualization/graph/base_node.py +15 -27
- msprobe/visualization/graph/distributed_analyzer.py +97 -40
- msprobe/visualization/graph/graph.py +14 -16
- msprobe/visualization/graph/node_colors.py +34 -31
- msprobe/visualization/graph/node_op.py +12 -11
- msprobe/visualization/graph_service.py +580 -205
- msprobe/visualization/utils.py +278 -31
- tb_graph_ascend/secure_build.py +175 -0
- tb_graph_ascend/server/__init__.py +15 -0
- tb_graph_ascend/server/app/__init__.py +15 -0
- tb_graph_ascend/server/app/model/__init__.py +15 -0
- tb_graph_ascend/server/app/model/hierarchy.py +348 -0
- tb_graph_ascend/server/app/model/layout_hierarchy_model.py +69 -0
- tb_graph_ascend/server/app/model/match_nodes_model.py +573 -0
- tb_graph_ascend/server/app/repositories/__init__.py +15 -0
- tb_graph_ascend/server/app/repositories/graph_repo_base.py +32 -0
- tb_graph_ascend/server/app/repositories/graph_repo_db.py +879 -0
- tb_graph_ascend/server/app/repositories/graph_repo_vis.py +83 -0
- tb_graph_ascend/server/app/service/__init__.py +18 -0
- tb_graph_ascend/server/app/service/graph_service_base.py +158 -0
- tb_graph_ascend/server/app/service/graph_service_db.py +438 -0
- tb_graph_ascend/server/app/service/graph_service_factory.py +54 -0
- tb_graph_ascend/server/app/service/graph_service_vis.py +480 -0
- tb_graph_ascend/server/app/utils/__init__.py +15 -0
- tb_graph_ascend/server/app/utils/constant.py +80 -0
- tb_graph_ascend/server/app/utils/file_check_wrapper.py +46 -0
- tb_graph_ascend/server/app/utils/global_state.py +95 -0
- tb_graph_ascend/server/app/utils/graph_utils.py +661 -0
- tb_graph_ascend/server/app/utils/i18n.py +153 -0
- tb_graph_ascend/server/app/utils/request_method.py +46 -0
- tb_graph_ascend/server/app/views/__init__.py +15 -0
- tb_graph_ascend/server/app/views/graph_views.py +304 -0
- tb_graph_ascend/server/plugin.py +108 -0
- tb_graph_ascend/server/static/index.html +9250 -0
- tb_graph_ascend/server/static/index.js +21 -0
- tb_graph_ascend/setup.py +57 -0
- mindstudio_probe-8.3.3.dist-info/LICENSE +0 -201
- mindstudio_probe-8.3.3.dist-info/RECORD +0 -491
- mindstudio_probe-8.3.3.dist-info/entry_points.txt +0 -2
- mindstudio_probe-8.3.3.dist-info/top_level.txt +0 -1
- msprobe/CMakeLists.txt +0 -5
- msprobe/README.md +0 -203
- msprobe/core/advisor/advisor.py +0 -129
- msprobe/core/advisor/advisor_const.py +0 -58
- msprobe/core/advisor/advisor_result.py +0 -58
- msprobe/core/compare/find_first/data_processor.py +0 -35
- msprobe/core/compare/highlight.py +0 -390
- msprobe/core/data_dump/data_collector.py +0 -356
- msprobe/core/grad_probe/constant.py +0 -90
- msprobe/core/grad_probe/grad_compare.py +0 -187
- msprobe/core/grad_probe/utils.py +0 -105
- msprobe/core/kernel_dump/kernel_config.py +0 -33
- msprobe/docs/01.installation.md +0 -250
- msprobe/docs/02.config_introduction.md +0 -221
- msprobe/docs/03.config_examples.md +0 -281
- msprobe/docs/04.kernel_dump_PyTorch.md +0 -73
- msprobe/docs/05.data_dump_PyTorch.md +0 -518
- msprobe/docs/06.data_dump_MindSpore.md +0 -618
- msprobe/docs/07.accuracy_checker_PyTorch.md +0 -310
- msprobe/docs/09.accuracy_checker_MindSpore.md +0 -120
- msprobe/docs/10.accuracy_compare_PyTorch.md +0 -637
- msprobe/docs/11.accuracy_compare_MindSpore.md +0 -769
- msprobe/docs/12.overflow_check_PyTorch.md +0 -82
- msprobe/docs/13.overflow_check_MindSpore.md +0 -33
- msprobe/docs/14.data_parse_PyTorch.md +0 -282
- msprobe/docs/15.free_benchmarking_PyTorch.md +0 -169
- msprobe/docs/16.free_benchmarking_MindSpore.md +0 -159
- msprobe/docs/17.grad_probe.md +0 -205
- msprobe/docs/18.online_dispatch.md +0 -89
- msprobe/docs/19.monitor.md +0 -753
- msprobe/docs/20.monitor_performance_baseline.md +0 -52
- msprobe/docs/21.visualization_PyTorch.md +0 -519
- msprobe/docs/22.visualization_MindSpore.md +0 -515
- msprobe/docs/23.generate_operator_PyTorch.md +0 -107
- msprobe/docs/24.code_mapping_Mindspore.md +0 -29
- msprobe/docs/25.tool_function_introduction.md +0 -29
- msprobe/docs/26.data_dump_PyTorch_baseline.md +0 -48
- msprobe/docs/27.dump_json_instruction.md +0 -795
- msprobe/docs/28.debugger_save_instruction.md +0 -288
- msprobe/docs/28.kernel_dump_MindSpore.md +0 -69
- msprobe/docs/29.data_dump_MSAdapter.md +0 -235
- msprobe/docs/30.overflow_check_MSAdapter.md +0 -31
- msprobe/docs/31.config_check.md +0 -107
- msprobe/docs/32.ckpt_compare.md +0 -69
- msprobe/docs/33.generate_operator_MindSpore.md +0 -181
- msprobe/docs/34.RL_collect.md +0 -101
- msprobe/docs/35.nan_analyze.md +0 -73
- msprobe/docs/36.calculation_result_change.md +0 -75
- msprobe/docs/FAQ.md +0 -232
- msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md +0 -146
- msprobe/docs/accuracy_checker_MindSpore/accuracy_checker_MindSpore_baseline.md +0 -14
- msprobe/docs/data_dump_MindSpore/data_dump_MindSpore_baseline.md +0 -33
- msprobe/docs/data_dump_MindSpore/dynamic_graph_quick_start_example.md +0 -217
- msprobe/docs/img/BLOOM-7B_1.png +0 -0
- msprobe/docs/img/BLOOM-7B_2.png +0 -0
- msprobe/docs/img/BLOOM-7B_3.png +0 -0
- msprobe/docs/img/BLOOM-7B_4.png +0 -0
- msprobe/docs/img/GPT-3_1.png +0 -0
- msprobe/docs/img/GPT-3_2.png +0 -0
- msprobe/docs/img/GPT-3_3.png +0 -0
- msprobe/docs/img/GPT-3_4.png +0 -0
- msprobe/docs/img/GPT-3_5.png +0 -0
- msprobe/docs/img/GPT-3_6.png +0 -0
- msprobe/docs/img/GPT-3_7.png +0 -0
- msprobe/docs/img/GPT-3_8.png +0 -0
- msprobe/docs/img/YOLOV5S_1.png +0 -0
- msprobe/docs/img/YOLOV5S_2.png +0 -0
- msprobe/docs/img/accuracy_checking_details.png +0 -0
- msprobe/docs/img/accuracy_checking_result.png +0 -0
- msprobe/docs/img/api_precision_compare_details.png +0 -0
- msprobe/docs/img/api_precision_compare_result.png +0 -0
- msprobe/docs/img/auto_analyze_log.png +0 -0
- msprobe/docs/img/compare_result.png +0 -0
- msprobe/docs/img/compare_result_pkl.png +0 -0
- msprobe/docs/img/compare_result_pkl_md5.png.png +0 -0
- msprobe/docs/img/cpu_info.png +0 -0
- msprobe/docs/img/free_benchmark.png +0 -0
- msprobe/docs/img/free_benchmark_framework.png +0 -0
- msprobe/docs/img/grad_probe_image-1.png +0 -0
- msprobe/docs/img/grad_probe_image-2.png +0 -0
- msprobe/docs/img/grad_probe_image-3.png +0 -0
- msprobe/docs/img/grad_probe_image-4.png +0 -0
- msprobe/docs/img/grad_probe_image.png +0 -0
- msprobe/docs/img/merge_result.png +0 -0
- msprobe/docs/img/module_compare.png +0 -0
- msprobe/docs/img/monitor/cpu_info.png +0 -0
- msprobe/docs/img/monitor/step_count_per_record.png +0 -0
- msprobe/docs/img/ms_dump.png +0 -0
- msprobe/docs/img/ms_layer.png +0 -0
- msprobe/docs/img/pt_dump.png +0 -0
- msprobe/docs/img/save_compare_result_sample.png +0 -0
- msprobe/docs/img/visualization/fuzzy_match_ms.png +0 -0
- msprobe/docs/img/visualization/fuzzy_match_pt.png +0 -0
- msprobe/docs/img/visualization/proxy.png +0 -0
- msprobe/docs/img/visualization/tensorboard_1.png +0 -0
- msprobe/docs/img/visualization/tensorboard_2.png +0 -0
- msprobe/docs/img/visualization/vis_browser_1.png +0 -0
- msprobe/docs/img/visualization/vis_browser_2.png +0 -0
- msprobe/docs/img/visualization/vis_match_info.png +0 -0
- msprobe/docs/img/visualization/vis_precision_info.png +0 -0
- msprobe/docs/img/visualization/vis_search_info.png +0 -0
- msprobe/docs/img/visualization/vis_show_info.png +0 -0
- msprobe/docs/img/visualization/vis_showcase.png +0 -0
- msprobe/docs/img/visualization/vis_unmatch_info.png +0 -0
- msprobe/docs/visualization/GPTModel.png +0 -0
- msprobe/docs/visualization/ParallelMLP.png +0 -0
- msprobe/docs/visualization/layer_mapping_example.md +0 -132
- msprobe/docs/visualization/mapping.png +0 -0
- msprobe/docs/visualization/mapping1.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/1.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/2.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/3.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/4.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/5.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/6.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/7.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/llamafactory-qwen25vl.txt +0 -59
- msprobe/docs/visualization/mindspeed_llamafactoary_img/llamafactory1.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/llamafactory2.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/mindspeed-mm-qwen25vl.txt +0 -80
- msprobe/docs/visualization/mindspeed_llamafactoary_img/mindspeed1.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactoary_img/mindspeed2.png +0 -0
- msprobe/docs/visualization/mindspeed_llamafactory_mapping.md +0 -330
- msprobe/docs/visualization/module_name.png +0 -0
- msprobe/docs/visualization/module_name1.png +0 -0
- msprobe/docs/visualization/no_mapping.png +0 -0
- msprobe/docs/visualization/no_mapping1.png +0 -0
- msprobe/docs/visualization/no_mapping_analyze.png +0 -0
- msprobe/docs/visualization/top_layer.png +0 -0
- msprobe/mindspore/api_accuracy_checker/generate_op_script/op_generator.py +0 -460
- msprobe/mindspore/api_accuracy_checker/generate_op_script/operator_replication.template +0 -2081
- msprobe/mindspore/code_mapping/bind.py +0 -283
- msprobe/mindspore/code_mapping/cmd_parser.py +0 -40
- msprobe/mindspore/code_mapping/graph.py +0 -49
- msprobe/mindspore/code_mapping/graph_parser.py +0 -211
- msprobe/mindspore/code_mapping/main.py +0 -24
- msprobe/mindspore/code_mapping/processor.py +0 -34
- msprobe/mindspore/dym_loader/hook_dynamic_loader.cpp +0 -111
- msprobe/mindspore/dym_loader/hook_dynamic_loader.h +0 -52
- msprobe/mindspore/free_benchmark/api_pynative_self_check.py +0 -257
- msprobe/mindspore/free_benchmark/common/config.py +0 -27
- msprobe/mindspore/free_benchmark/common/handler_params.py +0 -31
- msprobe/mindspore/free_benchmark/common/utils.py +0 -100
- msprobe/mindspore/free_benchmark/data/support_wrap_ops.yaml +0 -638
- msprobe/mindspore/free_benchmark/handler/base_handler.py +0 -105
- msprobe/mindspore/free_benchmark/handler/check_handler.py +0 -55
- msprobe/mindspore/free_benchmark/handler/fix_handler.py +0 -51
- msprobe/mindspore/free_benchmark/handler/handler_factory.py +0 -36
- msprobe/mindspore/free_benchmark/perturbation/add_noise.py +0 -82
- msprobe/mindspore/free_benchmark/perturbation/base_perturbation.py +0 -45
- msprobe/mindspore/free_benchmark/perturbation/bit_noise.py +0 -78
- msprobe/mindspore/free_benchmark/perturbation/exchange_value.py +0 -77
- msprobe/mindspore/free_benchmark/perturbation/improve_precision.py +0 -56
- msprobe/mindspore/free_benchmark/perturbation/no_change.py +0 -27
- msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py +0 -46
- msprobe/mindspore/free_benchmark/self_check_tool_factory.py +0 -51
- msprobe/mindspore/grad_probe/global_context.py +0 -127
- msprobe/mindspore/grad_probe/grad_analyzer.py +0 -260
- msprobe/mindspore/grad_probe/grad_monitor.py +0 -42
- msprobe/mindspore/grad_probe/grad_stat_csv.py +0 -161
- msprobe/mindspore/grad_probe/hook.py +0 -115
- msprobe/mindspore/grad_probe/utils.py +0 -43
- msprobe/mindspore/mindtorch/__init__.py +0 -18
- msprobe/mindspore/ms_config.py +0 -153
- msprobe/mindspore/task_handler_factory.py +0 -44
- msprobe/nan_analyze/__init__.py +0 -14
- msprobe/pytorch/api_accuracy_checker/generate_op_script/config_op.json +0 -9
- msprobe/pytorch/api_accuracy_checker/generate_op_script/op_generator.py +0 -480
- msprobe/pytorch/api_accuracy_checker/generate_op_script/operator_replication.template +0 -567
- msprobe/pytorch/debugger/precision_debugger.py +0 -181
- msprobe/pytorch/free_benchmark/__init__.py +0 -23
- msprobe/pytorch/free_benchmark/common/constant.py +0 -85
- msprobe/pytorch/free_benchmark/common/counter.py +0 -87
- msprobe/pytorch/free_benchmark/common/enums.py +0 -80
- msprobe/pytorch/free_benchmark/common/params.py +0 -152
- msprobe/pytorch/free_benchmark/common/utils.py +0 -143
- msprobe/pytorch/free_benchmark/compare/grad_saver.py +0 -215
- msprobe/pytorch/free_benchmark/compare/single_benchmark.py +0 -121
- msprobe/pytorch/free_benchmark/main.py +0 -123
- msprobe/pytorch/free_benchmark/perturbed_layers/base_layer.py +0 -28
- msprobe/pytorch/free_benchmark/perturbed_layers/layer_factory.py +0 -56
- msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py +0 -107
- msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py +0 -121
- msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py +0 -89
- msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py +0 -87
- msprobe/pytorch/free_benchmark/perturbed_layers/npu/no_change.py +0 -43
- msprobe/pytorch/free_benchmark/perturbed_layers/npu/npu_base_layser.py +0 -60
- msprobe/pytorch/free_benchmark/perturbed_layers/run_cpu.py +0 -34
- msprobe/pytorch/free_benchmark/result_handlers/base_handler.py +0 -252
- msprobe/pytorch/free_benchmark/result_handlers/check_handler.py +0 -54
- msprobe/pytorch/free_benchmark/result_handlers/fix_handler.py +0 -40
- msprobe/pytorch/free_benchmark/result_handlers/handler_factory.py +0 -45
- msprobe/pytorch/free_benchmark/result_handlers/preheat_handler.py +0 -181
- msprobe/pytorch/grad_probe/__init__.py +0 -0
- msprobe/pytorch/grad_probe/grad_monitor.py +0 -108
- msprobe/pytorch/grad_probe/grad_stat_csv.py +0 -160
- msprobe/pytorch/hook_module/__init__.py +0 -16
- msprobe/pytorch/hook_module/wrap_aten.py +0 -111
- msprobe/pytorch/online_dispatch/__init__.py +0 -19
- msprobe/pytorch/online_dispatch/compare.py +0 -224
- msprobe/pytorch/online_dispatch/dispatch.py +0 -332
- msprobe/pytorch/online_dispatch/dump_compare.py +0 -179
- msprobe/pytorch/online_dispatch/single_compare.py +0 -412
- msprobe/pytorch/online_dispatch/torch_ops_config.yaml +0 -58
- msprobe/pytorch/online_dispatch/utils.py +0 -158
- msprobe/pytorch/parse_tool/__init__.py +0 -0
- msprobe/pytorch/parse_tool/cli.py +0 -31
- msprobe/pytorch/parse_tool/lib/__init__.py +0 -0
- msprobe/pytorch/parse_tool/lib/compare.py +0 -253
- msprobe/pytorch/parse_tool/lib/config.py +0 -50
- msprobe/pytorch/parse_tool/lib/file_desc.py +0 -45
- msprobe/pytorch/parse_tool/lib/interactive_cli.py +0 -97
- msprobe/pytorch/parse_tool/lib/parse_exception.py +0 -54
- msprobe/pytorch/parse_tool/lib/parse_tool.py +0 -161
- msprobe/pytorch/parse_tool/lib/utils.py +0 -299
- msprobe/pytorch/parse_tool/lib/visualization.py +0 -85
- msprobe/pytorch/pt_config.py +0 -299
- /msprobe/core/{grad_probe → dump}/__init__.py +0 -0
- /msprobe/{mindspore/code_mapping → core/dump/api_dump}/__init__.py +0 -0
- /msprobe/{mindspore/debugger → core/dump/data_dump}/__init__.py +0 -0
- /msprobe/{mindspore/exception_dump → core/dump/data_dump/data_processor}/__init__.py +0 -0
- /msprobe/{mindspore/free_benchmark → core/dump/debugger}/__init__.py +0 -0
- /msprobe/{mindspore/free_benchmark/common → core/dump/kernel_dump}/__init__.py +0 -0
- /msprobe/mindspore/{free_benchmark/handler → dump/debugger}/__init__.py +0 -0
- /msprobe/mindspore/{grad_probe → dump/dump_processor}/__init__.py +0 -0
- /msprobe/mindspore/{overflow_check → dump/exception_dump}/__init__.py +0 -0
- /msprobe/mindspore/{mindtorch → dump/mindtorch}/mindtorch_adaptor.py +0 -0
- /msprobe/{pytorch/api_accuracy_checker/run_ut → mindspore/dump/overflow_check}/__init__.py +0 -0
- /msprobe/{pytorch/debugger → mindspore/monitor}/__init__.py +0 -0
- /msprobe/{pytorch/free_benchmark/common → msaccucmp}/__init__.py +0 -0
- /msprobe/pytorch/api_accuracy_checker/{run_ut → acc_check}/.keep +0 -0
- /msprobe/pytorch/{free_benchmark/perturbed_layers → api_accuracy_checker/acc_check}/__init__.py +0 -0
- /msprobe/pytorch/api_accuracy_checker/{run_ut → acc_check}/torch_ut_setting.json +0 -0
- /msprobe/pytorch/{free_benchmark/perturbed_layers/npu → dump/api_dump}/__init__.py +0 -0
- /msprobe/pytorch/{hook_module → dump/api_dump}/support_wrap_ops.yaml +0 -0
- /msprobe/pytorch/{free_benchmark/result_handlers → dump/debugger}/__init__.py +0 -0
|
@@ -0,0 +1,661 @@
|
|
|
1
|
+
# -------------------------------------------------------------------------
|
|
2
|
+
# This file is part of the MindStudio project.
|
|
3
|
+
# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
|
|
4
|
+
#
|
|
5
|
+
# MindStudio is licensed under Mulan PSL v2.
|
|
6
|
+
# You can use this software according to the terms and conditions of the Mulan PSL v2.
|
|
7
|
+
# You may obtain a copy of Mulan PSL v2 at:
|
|
8
|
+
#
|
|
9
|
+
# http://license.coscl.org.cn/MulanPSL2
|
|
10
|
+
#
|
|
11
|
+
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
|
|
12
|
+
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
|
|
13
|
+
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
|
14
|
+
# See the Mulan PSL v2 for more details.
|
|
15
|
+
# -------------------------------------------------------------------------
|
|
16
|
+
|
|
17
|
+
import math
|
|
18
|
+
import os
|
|
19
|
+
import json
|
|
20
|
+
import re
|
|
21
|
+
import stat
|
|
22
|
+
import threading
|
|
23
|
+
from functools import cmp_to_key
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from tensorboard.util import tb_logging
|
|
26
|
+
from .global_state import GraphState
|
|
27
|
+
from .constant import DataType, FILE_NAME_REGEX, MAX_FILE_SIZE, PERM_GROUP_WRITE, PERM_OTHER_WRITE, COLOR_PATTERN
|
|
28
|
+
from .i18n import language, ZH_CN
|
|
29
|
+
# 创建一个全局锁
|
|
30
|
+
_thread_local_lock = threading.Lock()
|
|
31
|
+
logger = tb_logging.get_logger()
|
|
32
|
+
FILE_PATH_MAX_LENGTH = 4096
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class GraphUtils:
|
|
36
|
+
|
|
37
|
+
@staticmethod
|
|
38
|
+
def t(key):
|
|
39
|
+
lang = GraphState.get_global_value('lang', ZH_CN)
|
|
40
|
+
return language.get(lang).get(key, '')
|
|
41
|
+
|
|
42
|
+
@staticmethod
|
|
43
|
+
def get_graph_data(meta_data):
|
|
44
|
+
if not meta_data:
|
|
45
|
+
return None, 'Error: no query parameters provided'
|
|
46
|
+
try:
|
|
47
|
+
run_name = meta_data.get('run')
|
|
48
|
+
runs = GraphState.get_global_value('runs', {})
|
|
49
|
+
run = runs.get(run_name) or run_name
|
|
50
|
+
tag = meta_data.get('tag')
|
|
51
|
+
current_tag = GraphState.get_global_value('current_tag')
|
|
52
|
+
current_run = GraphState.get_global_value('current_run')
|
|
53
|
+
if current_tag == tag and current_run == run:
|
|
54
|
+
return GraphState.get_global_value('current_file_data'), None # 直接返回获取结果
|
|
55
|
+
else:
|
|
56
|
+
json_data, error_message = GraphUtils.safe_load_data(run, f"{tag}.vis", False)
|
|
57
|
+
if error_message:
|
|
58
|
+
return None, 'Error: fail to get graph data'
|
|
59
|
+
GraphState.set_global_value('current_file_data', json_data)
|
|
60
|
+
GraphState.set_global_value('current_tag', tag)
|
|
61
|
+
GraphState.set_global_value('current_run', run)
|
|
62
|
+
return json_data, error_message
|
|
63
|
+
except Exception as e:
|
|
64
|
+
logger.error(f'Error: fail to get graph data by {meta_data}, error: {e}')
|
|
65
|
+
return None, 'Error: fail to get graph data'
|
|
66
|
+
|
|
67
|
+
@staticmethod
|
|
68
|
+
def get_opposite_node_name(node_name):
|
|
69
|
+
opposite_node_name = ''
|
|
70
|
+
# 如果npu_node_name包含forward,则opposite_npu_node_name为npu_node_name替换forward为backward
|
|
71
|
+
if 'forward' in node_name:
|
|
72
|
+
opposite_node_name = node_name.replace('forward', 'backward')
|
|
73
|
+
else:
|
|
74
|
+
opposite_node_name = node_name.replace('backward', 'forward')
|
|
75
|
+
return opposite_node_name
|
|
76
|
+
|
|
77
|
+
@staticmethod
|
|
78
|
+
def get_parent_node_list(graph_data, node_name):
|
|
79
|
+
"""获取父节点列表"""
|
|
80
|
+
# 如果 graph_data 为空或 node_name 不存在,直接返回空列表
|
|
81
|
+
if not graph_data or not node_name:
|
|
82
|
+
return []
|
|
83
|
+
|
|
84
|
+
node_list = [] # 存储结果的列表
|
|
85
|
+
current_node = node_name # 当前节点名称
|
|
86
|
+
nodes = graph_data['node'] # 提取节点数据,避免重复访问
|
|
87
|
+
|
|
88
|
+
while current_node:
|
|
89
|
+
node_list.append(current_node) # 将当前节点加入列表
|
|
90
|
+
# 获取当前节点的 upnode
|
|
91
|
+
current_node = nodes.get(current_node, {}).get('upnode')
|
|
92
|
+
|
|
93
|
+
# 检测循环引用(防止死循环)
|
|
94
|
+
if current_node in node_list:
|
|
95
|
+
raise ValueError(
|
|
96
|
+
f"{GraphUtils.t('circularReferenceError1')}"
|
|
97
|
+
f"{current_node}{GraphUtils.t('circularReferenceError2')}"
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
return list(reversed(node_list)) # 返回结果列表
|
|
101
|
+
|
|
102
|
+
@staticmethod
|
|
103
|
+
def split_graph_data_by_microstep(graph_data, micro_step):
|
|
104
|
+
graph_nodes = graph_data.get('node', {})
|
|
105
|
+
if str(micro_step) == str(-1):
|
|
106
|
+
return graph_nodes
|
|
107
|
+
splited_graph_data = {}
|
|
108
|
+
node_list = [] # 存储已遍历的列表
|
|
109
|
+
|
|
110
|
+
def traverse_npu(graph_nodes, subnodes):
|
|
111
|
+
for node_name in subnodes:
|
|
112
|
+
node_data = graph_nodes.get(node_name, {})
|
|
113
|
+
micro_step_id = node_data.get('micro_step_id')
|
|
114
|
+
if str(micro_step_id) == str(micro_step) or micro_step_id is None:
|
|
115
|
+
splited_graph_data[node_name] = (node_data)
|
|
116
|
+
if node_name in node_list:
|
|
117
|
+
raise ValueError(
|
|
118
|
+
f"{GraphUtils.t('circularReferenceError1')}"
|
|
119
|
+
f"{node_name}{GraphUtils.t('circularReferenceError2')}"
|
|
120
|
+
)
|
|
121
|
+
node_list.append(node_name)
|
|
122
|
+
traverse_npu(graph_nodes, node_data.get('subnodes', []))
|
|
123
|
+
|
|
124
|
+
root = graph_data.get('root')
|
|
125
|
+
root_subnodes = graph_nodes.get(root, {}).get('subnodes', [])
|
|
126
|
+
node_list.append(root)
|
|
127
|
+
traverse_npu(graph_nodes, root_subnodes)
|
|
128
|
+
return splited_graph_data
|
|
129
|
+
|
|
130
|
+
@staticmethod
|
|
131
|
+
def walk_with_max_depth(logdir, max_depth):
|
|
132
|
+
for root, dirs, files in os.walk(logdir):
|
|
133
|
+
# 计算当前 root 相对于 top 的深度
|
|
134
|
+
depth = root[len(logdir):].count(os.sep) + 1
|
|
135
|
+
if depth >= max_depth:
|
|
136
|
+
del dirs[:]
|
|
137
|
+
yield root, dirs, files
|
|
138
|
+
|
|
139
|
+
@staticmethod
|
|
140
|
+
def safe_json_loads(json_str, default_value=None):
|
|
141
|
+
"""
|
|
142
|
+
安全地解析 JSON 字符串,带长度限制和异常处理。
|
|
143
|
+
:param json_str: 要解析的 JSON 字符串
|
|
144
|
+
:param default_value: 如果解析失败返回的默认值
|
|
145
|
+
:return: 解析后的 Python 对象 或 default_value
|
|
146
|
+
"""
|
|
147
|
+
# 类型检查
|
|
148
|
+
if not isinstance(json_str, str):
|
|
149
|
+
return default_value
|
|
150
|
+
# 长度限制
|
|
151
|
+
if len(json_str) > MAX_FILE_SIZE:
|
|
152
|
+
return default_value
|
|
153
|
+
try:
|
|
154
|
+
result = json.loads(json_str)
|
|
155
|
+
return result
|
|
156
|
+
except json.JSONDecodeError as e:
|
|
157
|
+
logger.error(f"JSON decode error: {e}")
|
|
158
|
+
return default_value
|
|
159
|
+
except Exception as e:
|
|
160
|
+
logger.error(f"Unexpected error: {e}")
|
|
161
|
+
return default_value
|
|
162
|
+
|
|
163
|
+
@staticmethod
|
|
164
|
+
def safe_get_node_info(data, default_value=None):
|
|
165
|
+
node_info = data.get('nodeInfo')
|
|
166
|
+
try:
|
|
167
|
+
# 长度限制 - 检查字典转为字符串后的长度
|
|
168
|
+
node_info_str = str(node_info)
|
|
169
|
+
if len(node_info_str) > MAX_FILE_SIZE:
|
|
170
|
+
logger.error(f"Input length exceeds {MAX_FILE_SIZE} characters.")
|
|
171
|
+
return default_value
|
|
172
|
+
# 验证必要字段是否存在
|
|
173
|
+
required_fields = ["nodeName", "nodeType"]
|
|
174
|
+
for field in required_fields:
|
|
175
|
+
if field not in node_info:
|
|
176
|
+
logger.error(f"Field {field} is missing in metadata.")
|
|
177
|
+
return default_value
|
|
178
|
+
return node_info
|
|
179
|
+
except json.JSONDecodeError:
|
|
180
|
+
logger.error("NodeInfo parameter is not in valid JSON format.")
|
|
181
|
+
return default_value
|
|
182
|
+
except Exception as e:
|
|
183
|
+
logger.error(f"An error occurred while parsing the nodeInfo parameter: {str(e)}")
|
|
184
|
+
return default_value
|
|
185
|
+
|
|
186
|
+
@staticmethod
|
|
187
|
+
def safe_get_meta_data(data, default_value=None):
|
|
188
|
+
meta_data = data.get('metaData')
|
|
189
|
+
try:
|
|
190
|
+
# 长度限制
|
|
191
|
+
meta_data_str = str(meta_data)
|
|
192
|
+
if len(meta_data_str) > MAX_FILE_SIZE:
|
|
193
|
+
logger.error(f"Input length exceeds {MAX_FILE_SIZE} characters.")
|
|
194
|
+
return default_value
|
|
195
|
+
# 验证必要字段是否存在
|
|
196
|
+
required_fields = ["tag", "microStep", "run", "type", 'lang']
|
|
197
|
+
for field in required_fields:
|
|
198
|
+
if field not in meta_data:
|
|
199
|
+
logger.error(f"Field {field} is missing in metadata.")
|
|
200
|
+
return default_value
|
|
201
|
+
config_info = GraphState.get_global_value('config_info', {})
|
|
202
|
+
micro_step = meta_data.get('microStep')
|
|
203
|
+
# 验证非必要字段如果存在,进行范围限制
|
|
204
|
+
if config_info.get('microSteps'):
|
|
205
|
+
if int(micro_step) != -1 and int(micro_step) not in list(range(config_info.get('microSteps'))):
|
|
206
|
+
logger.error(f"Field microStep {micro_step} is not in config_info.")
|
|
207
|
+
return default_value
|
|
208
|
+
if meta_data.get('type') == DataType.DB.value and config_info:
|
|
209
|
+
rank = meta_data.get('rank')
|
|
210
|
+
step = meta_data.get('step')
|
|
211
|
+
if config_info.get('ranks') and rank:
|
|
212
|
+
if rank not in config_info.get('ranks'):
|
|
213
|
+
logger.error(f"Field rank {rank} is not in config_info.")
|
|
214
|
+
return default_value
|
|
215
|
+
if config_info.get('steps') and step:
|
|
216
|
+
if step not in config_info.get('steps'):
|
|
217
|
+
logger.error(f"Field step {step} is not in config_info.")
|
|
218
|
+
return default_value
|
|
219
|
+
return meta_data
|
|
220
|
+
|
|
221
|
+
except json.JSONDecodeError:
|
|
222
|
+
logger.error("MetaData parameter is not in valid JSON format.")
|
|
223
|
+
return default_value
|
|
224
|
+
except Exception as e:
|
|
225
|
+
logger.error(f"An error occurred while parsing the metatdata parameter: {str(e)}")
|
|
226
|
+
return default_value
|
|
227
|
+
|
|
228
|
+
@staticmethod
|
|
229
|
+
def remove_prefix(node_data, prefix):
|
|
230
|
+
if node_data is None:
|
|
231
|
+
return {}
|
|
232
|
+
return {k[len(prefix):] if k.startswith(prefix) else k: v for k, v in node_data.items()}
|
|
233
|
+
|
|
234
|
+
@staticmethod
|
|
235
|
+
def convert_to_float(value):
|
|
236
|
+
try:
|
|
237
|
+
if isinstance(value, str):
|
|
238
|
+
# 处理'0.0%, 由于Mean小于1e-06, 建议不参考此相对误差,请参考绝对误差'和'0.0%'的情况
|
|
239
|
+
value = value.split(',')[0]
|
|
240
|
+
if value.endswith('%'):
|
|
241
|
+
value = value.replace('%', '').strip()
|
|
242
|
+
return float(value) / 100.0
|
|
243
|
+
return float(value)
|
|
244
|
+
except ValueError:
|
|
245
|
+
return float('nan')
|
|
246
|
+
|
|
247
|
+
@staticmethod
|
|
248
|
+
def format_relative_err(value):
|
|
249
|
+
"""格式化相对误差为百分比,保留四位小数"""
|
|
250
|
+
if value is None or math.isnan(value):
|
|
251
|
+
return "NaN"
|
|
252
|
+
else:
|
|
253
|
+
return "{:.4%}".format(value)
|
|
254
|
+
|
|
255
|
+
@staticmethod
|
|
256
|
+
def nan_to_str(value):
|
|
257
|
+
"""将 NaN 转换为 'NaN' 字符串"""
|
|
258
|
+
return "NaN" if math.isnan(value) else value
|
|
259
|
+
|
|
260
|
+
@staticmethod
|
|
261
|
+
def is_relative_to(path, base):
|
|
262
|
+
abs_path = os.path.abspath(path)
|
|
263
|
+
abs_base = os.path.abspath(base)
|
|
264
|
+
return os.path.commonpath([abs_path, abs_base]) == str(abs_base)
|
|
265
|
+
|
|
266
|
+
@staticmethod
|
|
267
|
+
def bytes_to_human_readable(size_bytes, decimal_places=2):
|
|
268
|
+
"""
|
|
269
|
+
将字节大小转换为更易读的格式(如 KB、MB、GB 等)。
|
|
270
|
+
|
|
271
|
+
:param size_bytes: int 或 float,表示字节大小
|
|
272
|
+
:param decimal_places: 保留的小数位数,默认为 2
|
|
273
|
+
:return: str,人类可读的大小表示
|
|
274
|
+
"""
|
|
275
|
+
if size_bytes == 0:
|
|
276
|
+
return "0 B"
|
|
277
|
+
|
|
278
|
+
units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
|
|
279
|
+
unit_index = 0
|
|
280
|
+
|
|
281
|
+
while size_bytes >= 1024 and unit_index < len(units) - 1:
|
|
282
|
+
size_bytes /= 1024.0
|
|
283
|
+
unit_index += 1
|
|
284
|
+
|
|
285
|
+
return f"{size_bytes:.{decimal_places}f} {units[unit_index]}"
|
|
286
|
+
|
|
287
|
+
@staticmethod
|
|
288
|
+
def safe_save_data(data, run_name, tag):
|
|
289
|
+
runs = GraphState.get_global_value('runs', {})
|
|
290
|
+
run = runs.get(run_name) or run_name
|
|
291
|
+
if not run or not tag:
|
|
292
|
+
error_message = 'The query parameters "run" and "tag" are required'
|
|
293
|
+
return None, error_message
|
|
294
|
+
try:
|
|
295
|
+
# 检查 tag 是否为合法文件名
|
|
296
|
+
if not re.match(FILE_NAME_REGEX, tag):
|
|
297
|
+
raise ValueError(f"{GraphUtils.t('invalidTag')}{tag}.")
|
|
298
|
+
# 构建文件路径并标准化
|
|
299
|
+
file_path = os.path.join(run, tag)
|
|
300
|
+
# 目录安全校验
|
|
301
|
+
success, error = GraphUtils.safe_check_save_file_path(run, True)
|
|
302
|
+
if not success:
|
|
303
|
+
raise PermissionError(error)
|
|
304
|
+
# 文件安全校验
|
|
305
|
+
success, error = GraphUtils.safe_check_save_file_path(file_path)
|
|
306
|
+
if not success:
|
|
307
|
+
raise PermissionError(error)
|
|
308
|
+
# 尝试写入文件
|
|
309
|
+
with _thread_local_lock:
|
|
310
|
+
with open(file_path, "w", encoding="utf-8") as f:
|
|
311
|
+
json.dump(data, f, ensure_ascii=False, indent=4)
|
|
312
|
+
f.flush() # 强制将缓冲区内容写入操作系统
|
|
313
|
+
os.fsync(f.fileno()) # 强制将缓冲区内容写入磁盘
|
|
314
|
+
os.chmod(file_path, 0o640)
|
|
315
|
+
# 最终校验(防御TOCTOU攻击)
|
|
316
|
+
if os.path.islink(file_path):
|
|
317
|
+
raise RuntimeError("The file has been replaced with a symbolic link")
|
|
318
|
+
return True, None
|
|
319
|
+
except (TypeError, ValueError) as e:
|
|
320
|
+
logger.error(f"Invalid data: {e}")
|
|
321
|
+
return None, 'Invalid data'
|
|
322
|
+
except OSError as e:
|
|
323
|
+
logger.error(f"Failed to create directory: {run}. Error: {e}\n")
|
|
324
|
+
return None, 'failed to create directory '
|
|
325
|
+
except Exception as e:
|
|
326
|
+
logger.error(f'Error: File "{file_path}" is not accessible. Error: {e}')
|
|
327
|
+
return None, 'failed to save file'
|
|
328
|
+
|
|
329
|
+
@staticmethod
|
|
330
|
+
def safe_load_data(run_name, tag, only_check=False):
|
|
331
|
+
runs = GraphState.get_global_value('runs', {})
|
|
332
|
+
run_dir = runs.get(str(run_name)) or run_name
|
|
333
|
+
safe_base_dir = GraphState.get_global_value('logdir')
|
|
334
|
+
"""Load a single .vis file from a given directory based on the tag."""
|
|
335
|
+
if run_dir is None or tag is None:
|
|
336
|
+
error_message = 'The query parameters "run" and "tag" are required'
|
|
337
|
+
return None, error_message
|
|
338
|
+
try:
|
|
339
|
+
file_path = os.path.join(run_dir, tag)
|
|
340
|
+
# 安全验证:基础路径校验
|
|
341
|
+
if not GraphUtils.is_relative_to(file_path, safe_base_dir):
|
|
342
|
+
raise ValueError(GraphUtils.t('pathMayNotInSecureDirectory'))
|
|
343
|
+
# 目录安全校验
|
|
344
|
+
success, error = GraphUtils.safe_check_load_file_path(run_dir, True)
|
|
345
|
+
if not success:
|
|
346
|
+
raise PermissionError(error)
|
|
347
|
+
# 文件安全校验
|
|
348
|
+
success, error = GraphUtils.safe_check_load_file_path(file_path)
|
|
349
|
+
if not success:
|
|
350
|
+
raise PermissionError(error)
|
|
351
|
+
# 读取文件比较耗时,支持onlyCheck参数,仅进行安全校验
|
|
352
|
+
if only_check:
|
|
353
|
+
return True, None
|
|
354
|
+
# 尝试解析 JSON 文件,校验文件内容是否合理
|
|
355
|
+
with _thread_local_lock:
|
|
356
|
+
with open(file_path, 'r', encoding='utf-8') as f:
|
|
357
|
+
return json.load(f), None
|
|
358
|
+
except json.JSONDecodeError:
|
|
359
|
+
logger.error(f'Error: File "{file_path}" is not a valid JSON file!')
|
|
360
|
+
return None, "File is not a valid JSON file!"
|
|
361
|
+
except Exception as e:
|
|
362
|
+
logger.error(f'Error: File "{file_path}" is not accessible. Error: {e}')
|
|
363
|
+
return None, e
|
|
364
|
+
|
|
365
|
+
@staticmethod
|
|
366
|
+
def safe_check_save_file_path(file_path, is_dir=False):
|
|
367
|
+
file_path = os.path.normpath(file_path) # 标准化路径
|
|
368
|
+
real_path = os.path.realpath(file_path)
|
|
369
|
+
safe_base_dir = GraphState.get_global_value('logdir')
|
|
370
|
+
try:
|
|
371
|
+
# 安全验证:路径长度检查
|
|
372
|
+
if len(file_path) > FILE_PATH_MAX_LENGTH:
|
|
373
|
+
raise PermissionError(f"Path length exceeds limit")
|
|
374
|
+
if not is_dir and not os.path.exists(file_path):
|
|
375
|
+
return True, None
|
|
376
|
+
st = os.stat(file_path)
|
|
377
|
+
# 安全验证:禁止符号链接文件
|
|
378
|
+
if os.path.islink(file_path):
|
|
379
|
+
raise PermissionError("The target file is a symbolic link")
|
|
380
|
+
# 安全验证:检查目录是否存在,如果不存在则创建
|
|
381
|
+
if is_dir and not os.path.exists(real_path):
|
|
382
|
+
os.makedirs(real_path, exist_ok=True)
|
|
383
|
+
os.chmod(file_path, 0o640)
|
|
384
|
+
# 权限校验:检查是否有写权限
|
|
385
|
+
if not os.stat(file_path).st_mode & stat.S_IWUSR:
|
|
386
|
+
raise PermissionError(f"No write permission for directory\n")
|
|
387
|
+
# 安全验证: 非windows系统下,属主检查
|
|
388
|
+
if os.name != 'nt':
|
|
389
|
+
current_uid = os.getuid()
|
|
390
|
+
# 如果是root用户,跳过后续权限检查
|
|
391
|
+
if current_uid == 0:
|
|
392
|
+
logger.warning("""Security Warning: Do not run this tool as root.
|
|
393
|
+
Running with elevated privileges may compromise system security.
|
|
394
|
+
Use a regular user account.""")
|
|
395
|
+
return True, None
|
|
396
|
+
# 属主检查
|
|
397
|
+
if st.st_uid != current_uid:
|
|
398
|
+
raise PermissionError(f"Directory is not owned by the current user")
|
|
399
|
+
# group和其他用户不可写检查
|
|
400
|
+
if st.st_mode & PERM_GROUP_WRITE or st.st_mode & PERM_OTHER_WRITE:
|
|
401
|
+
raise PermissionError(
|
|
402
|
+
f"Directory has group or other write permission, there may be a risk of data tampering.")
|
|
403
|
+
return True, None
|
|
404
|
+
except Exception as e:
|
|
405
|
+
logger.error(e)
|
|
406
|
+
return False, e
|
|
407
|
+
|
|
408
|
+
@staticmethod
|
|
409
|
+
def safe_check_load_file_path(file_path, is_dir=False):
|
|
410
|
+
# 权限常量定义
|
|
411
|
+
file_path = os.path.normpath(file_path) # 标准化路径
|
|
412
|
+
real_path = os.path.realpath(file_path)
|
|
413
|
+
st = os.stat(real_path)
|
|
414
|
+
try:
|
|
415
|
+
# 安全验证:路径长度检查
|
|
416
|
+
if len(real_path) > FILE_PATH_MAX_LENGTH:
|
|
417
|
+
raise PermissionError(
|
|
418
|
+
f"Path is too long (max {FILE_PATH_MAX_LENGTH} characters). Please use a shorter path."
|
|
419
|
+
)
|
|
420
|
+
# 安全检查:文件存在性验证
|
|
421
|
+
if not os.path.exists(real_path):
|
|
422
|
+
raise FileNotFoundError(f"File or directory does not exist,please check the path and ensure it exists.")
|
|
423
|
+
# 安全验证:禁止符号链接文件
|
|
424
|
+
if os.path.islink(file_path):
|
|
425
|
+
raise PermissionError(f"Symbolic links are not allowed,Use a real file path instead.")
|
|
426
|
+
# 安全验证:文件类型检查(防御TOCTOU攻击)
|
|
427
|
+
# 文件类型
|
|
428
|
+
if not is_dir and not os.path.isfile(real_path):
|
|
429
|
+
raise PermissionError(
|
|
430
|
+
f"Path is not a regular file."
|
|
431
|
+
"make sure the path points to a valid file (not a directory or device)."
|
|
432
|
+
)
|
|
433
|
+
# 目录类型
|
|
434
|
+
if is_dir and not Path(real_path).is_dir():
|
|
435
|
+
raise PermissionError(
|
|
436
|
+
f"Expected a directory, but it does not exist or is not a directory."
|
|
437
|
+
"Please check the path and ensure it is a valid directory."
|
|
438
|
+
)
|
|
439
|
+
# 可读性检查
|
|
440
|
+
if not st.st_mode & stat.S_IRUSR:
|
|
441
|
+
raise PermissionError(
|
|
442
|
+
f"Current user lacks read permission on file or directory"
|
|
443
|
+
"Run 'chmod u+r \"<path>\"' to grant read access"
|
|
444
|
+
)
|
|
445
|
+
# 文件大小校验
|
|
446
|
+
if not is_dir and os.path.getsize(file_path) > MAX_FILE_SIZE:
|
|
447
|
+
file_size = GraphUtils.bytes_to_human_readable(os.path.getsize(file_path))
|
|
448
|
+
max_size = GraphUtils.bytes_to_human_readable(MAX_FILE_SIZE)
|
|
449
|
+
raise PermissionError(
|
|
450
|
+
f"File size exceeds limit ({file_size} > {max_size})."
|
|
451
|
+
"reduce file size or adjust MAX_FILE_SIZE if needed"
|
|
452
|
+
)
|
|
453
|
+
# 非windows系统下,属主检查
|
|
454
|
+
if os.name != 'nt':
|
|
455
|
+
current_uid = os.getuid()
|
|
456
|
+
# 如果是root用户,跳过后续权限检查
|
|
457
|
+
if current_uid == 0:
|
|
458
|
+
logger.warning("""Security Warning: Do not run this tool as root.
|
|
459
|
+
Running with elevated privileges may compromise system security.
|
|
460
|
+
Use a regular user account.""")
|
|
461
|
+
return True, None
|
|
462
|
+
# 属主检查
|
|
463
|
+
if st.st_uid != current_uid:
|
|
464
|
+
raise PermissionError(
|
|
465
|
+
f"File or directory is not owned by current user,"
|
|
466
|
+
"Run 'chown <user> \"<path>\"' to fix ownership."
|
|
467
|
+
)
|
|
468
|
+
# group和其他用户不可写检查
|
|
469
|
+
if st.st_mode & PERM_GROUP_WRITE or st.st_mode & PERM_OTHER_WRITE:
|
|
470
|
+
raise PermissionError(
|
|
471
|
+
f"File has insecure permissions: group or others have write access. "
|
|
472
|
+
"Run 'chmod go-w \"<path>\"' to remove write permissions for group and others."
|
|
473
|
+
)
|
|
474
|
+
return True, None
|
|
475
|
+
except Exception as e:
|
|
476
|
+
logger.error(e)
|
|
477
|
+
return False, e
|
|
478
|
+
|
|
479
|
+
@staticmethod
|
|
480
|
+
def find_config_files(run_name):
|
|
481
|
+
"""
|
|
482
|
+
在指定目录下查找所有以 .vis.config 结尾的文件(不包括子目录)
|
|
483
|
+
:param directory: 要搜索的目标目录路径
|
|
484
|
+
:return: 包含所有匹配文件路径的列表
|
|
485
|
+
"""
|
|
486
|
+
runs = GraphState.get_global_value('runs', {})
|
|
487
|
+
run = runs.get(run_name)
|
|
488
|
+
dir_path = Path(run)
|
|
489
|
+
try:
|
|
490
|
+
if GraphUtils.safe_check_load_file_path(run, True):
|
|
491
|
+
return [
|
|
492
|
+
file.name for file in dir_path.iterdir()
|
|
493
|
+
if file.is_file() and file.name.endswith('.vis.config')
|
|
494
|
+
]
|
|
495
|
+
else:
|
|
496
|
+
return []
|
|
497
|
+
except Exception as e:
|
|
498
|
+
logger.error(e)
|
|
499
|
+
return []
|
|
500
|
+
|
|
501
|
+
@staticmethod
|
|
502
|
+
def compare_tag_names(a: str, b: str) -> int:
|
|
503
|
+
"""自然排序比较函数,支持路径格式和数字顺序"""
|
|
504
|
+
|
|
505
|
+
# 辅助函数:标准化路径并分割组件
|
|
506
|
+
def split_components(s: str) -> list:
|
|
507
|
+
s = s.replace('\\', '/') # 统一路径分隔符
|
|
508
|
+
return [c for c in re.split(r'[/_]+', s) if c] # 按 /_ 分割非空组件
|
|
509
|
+
|
|
510
|
+
# 辅助函数:将组件拆分为数字/字符串混合的 tokens
|
|
511
|
+
def tokenize(component: str) -> list:
|
|
512
|
+
tokens = []
|
|
513
|
+
buffer = []
|
|
514
|
+
is_num = False
|
|
515
|
+
|
|
516
|
+
for char in component:
|
|
517
|
+
if char.isdigit() == is_num and buffer:
|
|
518
|
+
buffer.append(char)
|
|
519
|
+
else:
|
|
520
|
+
if buffer:
|
|
521
|
+
tokens.append(int(''.join(buffer)) if is_num else ''.join(buffer))
|
|
522
|
+
buffer = [char]
|
|
523
|
+
is_num = char.isdigit()
|
|
524
|
+
|
|
525
|
+
if buffer:
|
|
526
|
+
tokens.append(int(''.join(buffer)) if is_num else ''.join(buffer))
|
|
527
|
+
return tokens
|
|
528
|
+
|
|
529
|
+
# 逐级比较组件
|
|
530
|
+
a_comps = split_components(a)
|
|
531
|
+
b_comps = split_components(b)
|
|
532
|
+
|
|
533
|
+
for a_part, b_part in zip(a_comps, b_comps):
|
|
534
|
+
a_tokens = tokenize(a_part)
|
|
535
|
+
b_tokens = tokenize(b_part)
|
|
536
|
+
|
|
537
|
+
# 比较 token 序列
|
|
538
|
+
for a_tok, b_tok in zip(a_tokens, b_tokens):
|
|
539
|
+
if isinstance(a_tok, int) and isinstance(b_tok, int):
|
|
540
|
+
if a_tok != b_tok:
|
|
541
|
+
return a_tok - b_tok
|
|
542
|
+
elif isinstance(a_tok, int):
|
|
543
|
+
return -1 # 数字优先于字母
|
|
544
|
+
elif isinstance(b_tok, int):
|
|
545
|
+
return 1
|
|
546
|
+
elif a_tok != b_tok:
|
|
547
|
+
return -1 if a_tok < b_tok else 1
|
|
548
|
+
|
|
549
|
+
# 处理子序列长度差异
|
|
550
|
+
if len(a_tokens) != len(b_tokens):
|
|
551
|
+
return len(a_tokens) - len(b_tokens)
|
|
552
|
+
|
|
553
|
+
# 处理组件数量差异
|
|
554
|
+
return len(a_comps) - len(b_comps)
|
|
555
|
+
|
|
556
|
+
@staticmethod
|
|
557
|
+
def sort_data(data: dict) -> dict:
|
|
558
|
+
"""自然排序比较函数,支持路径格式和数字顺序"""
|
|
559
|
+
sorted_data = {}
|
|
560
|
+
sorted_keys = sorted(data.keys(), key=cmp_to_key(GraphUtils.compare_tag_names))
|
|
561
|
+
for k in sorted_keys:
|
|
562
|
+
# 对每个键对应的值列表进行排序
|
|
563
|
+
sorted_values = sorted(data.get(k, {}).get('tags'), key=cmp_to_key(GraphUtils.compare_tag_names))
|
|
564
|
+
sorted_data[k] = {'type': data.get(k, {}).get('type'), 'tags': sorted_values}
|
|
565
|
+
|
|
566
|
+
return sorted_data
|
|
567
|
+
|
|
568
|
+
@staticmethod
|
|
569
|
+
def is_safe_string(s: str) -> bool:
|
|
570
|
+
"""
|
|
571
|
+
安全检查字符串是否包含恶意内容(XSS 防护)
|
|
572
|
+
"""
|
|
573
|
+
if not isinstance(s, str):
|
|
574
|
+
return False
|
|
575
|
+
|
|
576
|
+
# 1. 长度限制(字符数,非字节)
|
|
577
|
+
if len(s) > FILE_PATH_MAX_LENGTH:
|
|
578
|
+
return False
|
|
579
|
+
|
|
580
|
+
# 2. 转换为小写用于检测
|
|
581
|
+
s_lower = s.lower().strip()
|
|
582
|
+
|
|
583
|
+
# 3. 黑名单关键词(常见 XSS 向量)
|
|
584
|
+
dangerous_patterns = [
|
|
585
|
+
'<script', '<img', '<svg', '<iframe', '<video', '<audio',
|
|
586
|
+
'javascript:', 'vbscript:', 'data:text/html',
|
|
587
|
+
'onload=', 'onerror=', 'onmouseover=', 'onclick=',
|
|
588
|
+
'eval(', 'alert(', 'document.cookie', 'document.location',
|
|
589
|
+
'window.location', 'innerHTML', 'outerHTML', 'document.write'
|
|
590
|
+
]
|
|
591
|
+
|
|
592
|
+
if any(pattern in s_lower for pattern in dangerous_patterns):
|
|
593
|
+
return False
|
|
594
|
+
|
|
595
|
+
return True
|
|
596
|
+
|
|
597
|
+
@staticmethod
|
|
598
|
+
def escape_html(input_str):
|
|
599
|
+
"""
|
|
600
|
+
将字符串中的特殊 HTML 字符转义为 HTML 实体。
|
|
601
|
+
"""
|
|
602
|
+
html_escape_map = {
|
|
603
|
+
'&': '&',
|
|
604
|
+
'<': '<',
|
|
605
|
+
'>': '>',
|
|
606
|
+
'"': '"',
|
|
607
|
+
"'": ''',
|
|
608
|
+
'/': '/',
|
|
609
|
+
}
|
|
610
|
+
# 使用 str.translate 配合 str.maketrans 提高性能
|
|
611
|
+
translate_table = str.maketrans(html_escape_map)
|
|
612
|
+
return input_str.translate(translate_table)
|
|
613
|
+
|
|
614
|
+
@staticmethod
|
|
615
|
+
def validate_colors_param(colors_json: str):
|
|
616
|
+
# 合法颜色正则:#FFFFFF 格式,不区分大小写
|
|
617
|
+
|
|
618
|
+
"""
|
|
619
|
+
校验 colors 参数
|
|
620
|
+
返回: (是否合法, 错误信息, 解析后的数据)
|
|
621
|
+
"""
|
|
622
|
+
|
|
623
|
+
if not isinstance(colors_json, dict):
|
|
624
|
+
return False, GraphUtils.t('colorsNotObject'), {}
|
|
625
|
+
|
|
626
|
+
if len(colors_json) == 0:
|
|
627
|
+
return False, GraphUtils.t('colotsEmpty'), {}
|
|
628
|
+
|
|
629
|
+
for key, value in colors_json.items():
|
|
630
|
+
# 2. 校验颜色键
|
|
631
|
+
if not re.match(COLOR_PATTERN, key):
|
|
632
|
+
return False, f"{GraphUtils.t('illegalColorKey')}{key}", {}
|
|
633
|
+
|
|
634
|
+
if not isinstance(value, dict):
|
|
635
|
+
return False, f"{GraphUtils.t('colorValuesNotObject')}{key}", {}
|
|
636
|
+
|
|
637
|
+
if 'value' not in value:
|
|
638
|
+
return False, f"{GraphUtils.t('missingValueField')}{key}", {}
|
|
639
|
+
|
|
640
|
+
# 3. 校验 value 字段
|
|
641
|
+
val = value['value']
|
|
642
|
+
if isinstance(val, list):
|
|
643
|
+
if len(val) != 2:
|
|
644
|
+
return False, f"{GraphUtils.t('notArrayOfLength2')}{key}", {}
|
|
645
|
+
if not all(isinstance(x, (int, float)) for x in val):
|
|
646
|
+
return False, f"{GraphUtils.t('notArrayConsistNumbers')}{key}", {}
|
|
647
|
+
if val[0] >= val[1]:
|
|
648
|
+
return False, f"{GraphUtils.t('invalidValueRange')}{key}", {}
|
|
649
|
+
elif isinstance(val, str):
|
|
650
|
+
if val not in ["无匹配节点", "N/A", "No matching nodes"]:
|
|
651
|
+
return False, f"{GraphUtils.t('unsupportedValue')}{val}", {}
|
|
652
|
+
else:
|
|
653
|
+
return False, f"{GraphUtils.t('valueTypeError')}{key}", {}
|
|
654
|
+
|
|
655
|
+
# 4. 校验 description
|
|
656
|
+
desc = value.get('description', '')
|
|
657
|
+
if not GraphUtils.is_safe_string(desc):
|
|
658
|
+
return False, f"{GraphUtils.t('descriptionError')}{key}", {}
|
|
659
|
+
value['description'] = GraphUtils.escape_html(desc)
|
|
660
|
+
|
|
661
|
+
return True, None, colors_json
|